tests/xe: Add system_allocator test
PATCH 1/1 tests/xe: Add system_allocator test
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbuf
Running on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)
--- include/drm-uapi/xe_drm.h | 57 +- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1832 ++++++++++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1896 insertions(+), 7 deletions(-) create mode 100644 tests/intel/xe_exec_system_allocator.c
include/drm-uapi/xe_drm.h +50 -7
@@ -3,8 +3,8 @@* Copyright © 2023 Intel Corporation */ -#ifndef _XE_DRM_H_ -#define _XE_DRM_H_ +#ifndef _UAPI_XE_DRM_H_ +#define _UAPI_XE_DRM_H_#include "drm.h"
@@ -134,7 +134,7 @@ extern "C" {* redefine the interface more easily than an ever growing struct of * increasing complexity, and for large parts of that interface to be * entirely optional. The downside is more pointer chasing; chasing across -* the boundary with pointers encapsulated inside u64. +* the __user boundary with pointers encapsulated inside u64. * * Example chaining: *
@@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address
@@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4
@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these
@@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */
@@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;
@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/**
@@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;
@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0
@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */
@@ -1926,4 +1969,4 @@ struct drm_xe_query_eu_stall { }#endif -#endif /* _XE_DRM_H_ */ +#endif /* _UAPI_XE_DRM_H_ */
lib/xe/xe_ioctl.c +12 -0
@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);
lib/xe/xe_ioctl.h +1 -0
@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);
tests/intel/xe_exec_system_allocator.c new file +1832 -0
1,832 lines not on this page · raw message
tests/meson.build +1 -0
@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',
Signature · 1 line
2.34.1
Hi, Matt,
Quoted 23 lines · Matthew Brost
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbuf
Quoted 7 lines
Running on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)Signed-off-by Matthew Brost <[email protected]>
Did you have a chance to look at the memory (or rather virtual adddress) leaks on the tests as well?
With the old version it would exceed 1GB after some time, which I believe caused some slowdown.
Unfortunately the igt library funcs also leaks memory so running the test under valgrind was a bit noisy, but helped to some extent.
/Thomas
Quoted 32 lines · Thomas Hellström
Hi, Matt,
Quoted 29 lines · Matthew Brost
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)
Quoted 7 lines
Quoted 1 line
Signed-off-by Matthew Brost <[email protected]>Did you have a chance to look at the memory (or rather virtual adddress) leaks on the tests as well?
With the old version it would exceed 1GB after some time, which I believe caused some slowdown.
I have not looked at that - rather just fixed mmaping mallocs which would eventually cause the test to fail with -ENOSPC (e.g. if just ran xe_exec_system_alloc eventually everything started to fail whereas now I can run xe_exec_system_alloc to completion).
I wonder if the way I changed memory allocation fixed the leaks, let me try running under valgrind and see if anything pops.
Matt
Quoted 5 lines
Unfortunately the igt library funcs also leaks memory so running the test under valgrind was a bit noisy, but helped to some extent.
/Thomas
Hi Matt,
I am still going through your patch but sharing some comments already.
The sequence is neither complex nor too different from existing tests but as it is a lot of multi-thread / multi-process code, I am trying to come up with a suggestion to break it down. Might not be possible though.
Quoted 44 lines · Matthew Brost
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)Signed-off-by Matthew Brost <[email protected]>--- include/drm-uapi/xe_drm.h | 57 +- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1832 ++++++++++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1896 insertions(+), 7 deletions(-) create mode 100644 tests/intel/xe_exec_system_allocator.cinclude/drm-uapi/xe_drm.h +0
-0
include/drm-uapi/xe_drm.h line 7
Quoted 7 lines
include/drm-uapi/xe_drm.h +2
-2@@ -3,8 +3,8 @@* Copyright © 2023 Intel Corporation */ -#ifndef _XE_DRM_H_ -#define _XE_DRM_H_ +#ifndef _UAPI_XE_DRM_H_ +#define _UAPI_XE_DRM_H_
Nit: The header seems to have been copied directly from the kernel tree, instead it should be generated with:
make headers_install
include/drm-uapi/xe_drm.h extern "C" { · line 137
Quoted 9 lines
include/drm-uapi/xe_drm.h +1
-1#include "drm.h" @@ -134,7 +134,7 @@ extern "C" {* redefine the interface more easily than an ever growing struct of * increasing complexity, and for large parts of that interface to be * entirely optional. The downside is more pointer chasing; chasing across -* the boundary with pointers encapsulated inside u64. +* the __user boundary with pointers encapsulated inside u64.
See above comment on make headers_install.
Quoted 212 lines
include/drm-uapi/xe_drm.h +47
-4* * Example chaining: * @@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these @@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */ @@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/** @@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */ @@ -1926,4 +1969,4 @@ struct drm_xe_query_eu_stall { }#endif -#endif /* _XE_DRM_H_ */ +#endif /* _UAPI_XE_DRM_H_ */ lib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +50
-0@@ -0,0 +1,1832 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright © 2024 Intel Corporation +*/ + +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Hardware building block +* Mega feature: Shared virtual memory +* Sub-category: execbuf +* Functionality: fault mode, system allocator +* GPU requirements: GPU needs support for DRM_XE_VM_CREATE_FLAG_FAULT_MODE +*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data; +}; + +#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__, i__) ((data__)->expected_data) + +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr;
tests/intel/xe_exec_system_allocator.c line 60
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+ batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +}
Slightly out of scope for this review but the 2 functions above might be helpful under lib/ to prevent adding more duplications of the dword write batch sequence.
Quoted 7 lines
tests/intel/xe_exec_system_allocator.c new file +7
-0+ +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) {
tests/intel/xe_exec_system_allocator.c line 77
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data, i)); + +if (barrier) +pthread_barrier_wait (barrier); + } +} + +#define SYNC_FILE "/tmp/xe_exec_system_allocator_sync"
Might be worth creating and propagating a unique file name at runtime, for example with mkstemp(), in order to avoid potential concurrent accesses from multiple instances of the test.
Quoted 261 lines
tests/intel/xe_exec_system_allocator.c new file +261
-0+ +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL ); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags) +{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec); + } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +printf ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +printf ("FAIL EXEC_UFENCE: EXPEXCTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; + +printf ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +printf ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +printf ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +printf ("FAIL DARA: EXPEXCTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data); + } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \
tests/intel/xe_exec_system_allocator.c line 348
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+__xe_vm_bind_assert (fd, vm,0 , \ +0 ,0 ,0 ,0x1ull << va_bits, \ +DRM_XE_VM_BIND_OP_MAP , \ +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , \ + (__sync), (__num_sync), 0, 0) + +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0)
Is there a reason here to favor static variable + macros over helper function with parameters?
Quoted 690 lines
tests/intel/xe_exec_system_allocator.c new file +690
-0+ +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else if (flags &CPU_FAULT_THREADS ) +check_all_pages_threads (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else +check_all_pages (allocs[i].ptr, alloc_size, stride,NULL ); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); +printf ("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n" , +1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), +1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i], i)); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11) +#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21) + +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; +
tests/intel/xe_exec_system_allocator.c line 1048
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) { +char name[1024 ]; + +igt_assert (!(flags &NEW )); + +sprintf (name,"/tmp/xe_exec_system_allocator_dat%d\n" , +getpid ());
Might be another candidate to use mkstemp() or similar.
Quoted 256 lines
tests/intel/xe_exec_system_allocator.c new file +256
-0+ file_fd =open (name,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx], idx)); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx], idx)); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE ) +munmap (data, bo_size); +else + pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP && (i %2 )) { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + + xe_vm_bind_async(fd, vm, 0, bo, 0, + to_user_pointer(data),
tests/intel/xe_exec_system_allocator.c line 1314
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+ bo_size, 0, 0); + }else { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size);
Large memory leaks come from this ^ aligned_alloc(), see below.
Quoted 41 lines
tests/intel/xe_exec_system_allocator.c new file +41
-0+igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + } + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ; + +if (flags &MMAP )
tests/intel/xe_exec_system_allocator.c line 1365
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+munmap (pending_free[i], bo_size); + else +free (pending_free [i]); + } +free (pending_free); + }else { +if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data);
Something seems wrong with the flags logic when skipping this ^ free() for the allocation pointed above.
Francois
Quoted 482 lines
tests/intel/xe_exec_system_allocator.c new file +467
-0+ } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL ); + +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1
Quoted 9 lines · Francois Dugast
Hi Matt,
I am still going through your patch but sharing some comments already.
The sequence is neither complex nor too different from existing tests but as it is a lot of multi-thread / multi-process code, I am trying to come up with a suggestion to break it down. Might not be possible though.
Good to hear you don't find it too complex, open spliting if it makes sense.
Quoted 56 lines · Matthew Brost
Quoted 44 lines
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)Signed-off-by Matthew Brost <[email protected]>--- include/drm-uapi/xe_drm.h | 57 +- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1832 ++++++++++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1896 insertions(+), 7 deletions(-) create mode 100644 tests/intel/xe_exec_system_allocator.cinclude/drm-uapi/xe_drm.h +0
-0include/drm-uapi/xe_drm.h line 7
Quoted 7 lines
include/drm-uapi/xe_drm.h +2
-2@@ -3,8 +3,8 @@* Copyright © 2023 Intel Corporation */ -#ifndef _XE_DRM_H_ -#define _XE_DRM_H_ +#ifndef _UAPI_XE_DRM_H_ +#define _UAPI_XE_DRM_H_Nit: The header seems to have been copied directly from the kernel tree, instead it should be generated with:
Quoted 4 lines
make
headers_install
Let me split out the uAPI update into its own patch + use the proper flow.
Quoted 12 lines
include/drm-uapi/xe_drm.h extern "C" { · line 137
Quoted 9 lines
include/drm-uapi/xe_drm.h +1
-1#include "drm.h" @@ -134,7 +134,7 @@ extern "C" {* redefine the interface more easily than an ever growing struct of * increasing complexity, and for large parts of that interface to be * entirely optional. The downside is more pointer chasing; chasing across -* the boundary with pointers encapsulated inside u64. +* the __user boundary with pointers encapsulated inside u64. See above comment on make
headers_install.
+1
Quoted 218 lines
Quoted 218 lines
include/drm-uapi/xe_drm.h +47
-4* * Example chaining: * @@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these @@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */ @@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/** @@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */ @@ -1926,4 +1969,4 @@ struct drm_xe_query_eu_stall { }#endif -#endif /* _XE_DRM_H_ */ +#endif /* _UAPI_XE_DRM_H_ */ lib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +56
-0@@ -0,0 +1,1832 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright © 2024 Intel Corporation +*/ + +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Hardware building block +* Mega feature: Shared virtual memory +* Sub-category: execbuf +* Functionality: fault mode, system allocator +* GPU requirements: GPU needs support for DRM_XE_VM_CREATE_FLAG_FAULT_MODE +*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data; +}; + +#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__, i__) ((data__)->expected_data) + +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr; + batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx)
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 60
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +}Slightly out of scope for this review but the 2 functions above might be helpful under
lib/to prevent adding more duplications of the dword write batch sequence.
Yea we could split out generic batch writing functions into a library at some point but agree is probably out of scope for this series.
Quoted 13 lines
Quoted 13 lines
tests/intel/xe_exec_system_allocator.c new file +13
-0+ +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) { +struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data, i)); + +if (barrier) +pthread_barrier_wait (barrier);
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 77
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+ } +} + +#define SYNC_FILE "/tmp/xe_exec_system_allocator_sync"Might be worth creating and propagating a unique file name at runtime, for example with
mkstemp(), in order to avoid potential concurrent accesses from multiple instances of the test.
Let me look into that.
Quoted 266 lines
Quoted 266 lines
tests/intel/xe_exec_system_allocator.c new file +266
-0+ +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL ); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags) +{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec); + } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +printf ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +printf ("FAIL EXEC_UFENCE: EXPEXCTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; + +printf ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +printf ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +printf ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +printf ("FAIL DARA: EXPEXCTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data); + } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \ + __xe_vm_bind_assert(fd, vm, 0, \ + 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_MAP, \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR, \ + (__sync), (__num_sync), 0, 0)
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 348
Quoted 5 lines
tests/intel/xe_exec_system_allocator.c new file +5
-0+ +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0)Is there a reason here to favor static variable + macros over helper function with parameters?
va_bits is static as it looked up exactly once when the test loads.
I could change these helpers to static functions rather than macros if that is preferred.
Quoted 694 lines
Quoted 694 lines
tests/intel/xe_exec_system_allocator.c new file +694
-0+ +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else if (flags &CPU_FAULT_THREADS ) +check_all_pages_threads (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else +check_all_pages (allocs[i].ptr, alloc_size, stride,NULL ); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); +printf ("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n" , +1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), +1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i], i)); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11) +#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21) + +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; + +if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) {
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 1048
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+char name[1024 ]; + +igt_assert (!(flags &NEW )); + +sprintf (name,"/tmp/xe_exec_system_allocator_dat%d\n" , +getpid ());Might be another candidate to use
mkstemp()or similar.
Will look into that.
Quoted 260 lines
Quoted 260 lines
tests/intel/xe_exec_system_allocator.c new file +260
-0+ file_fd =open (name,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx], idx)); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx], idx)); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE ) +munmap (data, bo_size); +else + pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP && (i %2 )) { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + +xe_vm_bind_async (fd, vm,0 , bo,0 , +to_user_pointer (data), + bo_size,0 ,0 ); + }else { +if (!bo) { +if (flags &FREE )
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 1314
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+free (data); + else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size);Large memory leaks come from this ^
aligned_alloc(), see below.
Yea this will leak, thanks for pointing this out. Thomas also raised memory leaks as a concern.
Inline below with the changes to fix this...
Quoted 9 lines
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c new file +9
-0+igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + }
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1269
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size);
data = NULL;
Quoted 22 lines
Quoted 22 lines
tests/intel/xe_exec_system_allocator.c new file +22
-0+gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ;
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1361
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +if (flags &MMAP ) +munmap (pending_free[i], bo_size); +else +free (pending_free[i]); + } +free (pending_free); + }else {
s/ else {/\nif (data) {
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1365
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data);Something seems wrong with the flags logic when skipping this ^
free()for the allocation pointed above.
Yep, see above,
Matt
Quoted 484 lines
Francois
Quoted 482 lines
tests/intel/xe_exec_system_allocator.c new file +467
-0+ } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL ); + +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1
Hi,
Quoted 70 lines · Matthew Brost
Quoted 9 lines · Francois Dugast
Hi Matt,
I am still going through your patch but sharing some comments already.
The sequence is neither complex nor too different from existing tests but as it is a lot of multi-thread / multi-process code, I am trying to come up with a suggestion to break it down. Might not be possible though.
Good to hear you don't find it too complex, open spliting if it makes sense.
Quoted 56 lines · Matthew Brost
Quoted 44 lines
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)Signed-off-by Matthew Brost <[email protected]>--- include/drm-uapi/xe_drm.h | 57 +- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1832 ++++++++++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1896 insertions(+), 7 deletions(-) create mode 100644 tests/intel/xe_exec_system_allocator.cinclude/drm-uapi/xe_drm.h +0
-0include/drm-uapi/xe_drm.h line 7
Quoted 7 lines
include/drm-uapi/xe_drm.h +2
-2@@ -3,8 +3,8 @@* Copyright © 2023 Intel Corporation */ -#ifndef _XE_DRM_H_ -#define _XE_DRM_H_ +#ifndef _UAPI_XE_DRM_H_ +#define _UAPI_XE_DRM_H_Nit: The header seems to have been copied directly from the kernel tree, instead it should be generated with:
Quoted 7 lines
Quoted 4 lines
make
headers_installLet me split out the uAPI update into its own patch + use the proper flow.
Sounds good.
Quoted 182 lines
Quoted 4 lines
Quoted 4 lines
include/drm-uapi/xe_drm.h +0
-0#include "drm.h" Quoted 9 lines
include/drm-uapi/xe_drm.h extern "C" { · line 137
Quoted 6 lines
include/drm-uapi/xe_drm.h +1
-1@@ -134,7 +134,7 @@ extern "C" {* redefine the interface more easily than an ever growing struct of * increasing complexity, and for large parts of that interface to be * entirely optional. The downside is more pointer chasing; chasing across -* the boundary with pointers encapsulated inside u64. +* the __user boundary with pointers encapsulated inside u64. See above comment on make
headers_install.+1
Quoted 166 lines
Quoted 166 lines
include/drm-uapi/xe_drm.h +47
-4* * Example chaining: * @@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these @@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */ @@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/** @@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */ @@ -1926,4 +1969,4 @@ struct drm_xe_query_eu_stall { }#endif -#endif /* _XE_DRM_H_ */ +#endif /* _UAPI_XE_DRM_H_ */ lib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +4
-0@@ -0,0 +1,1832 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright © 2024 Intel Corporation +*/
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 12
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Hardware building block +* Mega feature: Shared virtual memory +* Sub-category: execbuf +* Functionality: fault mode, system allocator +* GPU requirements: GPU needs support for DRM_XE_VM_CREATE_FLAG_FAULT_MODE
Using this below will help test automation and reporting:
* TEST: Basic tests for execbuf functionality using system allocator
* Category: Core
* Mega feature: USM
* Sub-category: System allocator
* Functionality: fault mode, system allocator
* GPU: LNL, BMG, PVC
Quoted 24 lines
Quoted 24 lines
Quoted 24 lines
tests/intel/xe_exec_system_allocator.c new file +24
-0+*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data;
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 44
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+}; + +#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__, i__) ((data__)->expected_data)
Argument i__ is not used.
Quoted 117 lines
Quoted 12 lines
Quoted 12 lines
tests/intel/xe_exec_system_allocator.c new file +12
-0+ +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr; + batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx)Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 60
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +}Slightly out of scope for this review but the 2 functions above might be helpful under
lib/to prevent adding more duplications of the dword write batch sequence.Yea we could split out generic batch writing functions into a library at some point but agree is probably out of scope for this series.
Quoted 13 lines
Quoted 13 lines
tests/intel/xe_exec_system_allocator.c new file +13
-0+ +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) { +struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data, i)); + +if (barrier) +pthread_barrier_wait (barrier);Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 77
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+ } +} + +#define SYNC_FILE "/tmp/xe_exec_system_allocator_sync"Might be worth creating and propagating a unique file name at runtime, for example with
mkstemp(), in order to avoid potential concurrent accesses from multiple instances of the test.Let me look into that.
Quoted 67 lines
Quoted 67 lines
tests/intel/xe_exec_system_allocator.c new file +67
-0+ +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL );
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 152
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags)
It would be helpful to add here a comment similar to the one below for check_all_pages_threads().
Quoted 154 lines
Quoted 154 lines
Quoted 154 lines
tests/intel/xe_exec_system_allocator.c new file +154
-0+{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec);
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 314
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +printf ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +printf ("FAIL EXEC_UFENCE: EXPEXCTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]);
s/EXPEXCTED/EXPECTED/
Also, we should probably use IGT's log functions such as igt_info() instead of printf().
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
tests/intel/xe_exec_system_allocator.c new file +7
-0+ + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data;
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 329
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+uint64_t sdi_addr = addr + sdi_offset; + +printf ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +printf ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +printf ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +printf ("FAIL DARA: EXPEXCTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data);
s/DARA/DATA/ and s/EXPEXCTED/EXPECTED/ and same as above for printf().
Quoted 24 lines
Quoted 14 lines
Quoted 14 lines
tests/intel/xe_exec_system_allocator.c new file +14
-0+ } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \ + __xe_vm_bind_assert(fd, vm, 0, \ + 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_MAP, \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR, \ + (__sync), (__num_sync), 0, 0)Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 348
Quoted 5 lines
tests/intel/xe_exec_system_allocator.c new file +5
-0+ +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0)Is there a reason here to favor static variable + macros over helper function with parameters?
Quoted 4 lines
va_bitsis static as it looked up exactly once when the test loads.I could change these helpers to static functions rather than macros if that is preferred.
It forces variable names in the caller but that is just a minor issue.
Quoted 152 lines
Quoted 152 lines
Quoted 152 lines
tests/intel/xe_exec_system_allocator.c new file +151
-0+ +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags);
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 507
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+else if (flags &CPU_FAULT_THREADS ) +check_all_pages_threads (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else +check_all_pages (allocs[i].ptr, alloc_size, stride,NULL ); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); + printf("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n",
Also here, prefer IGT's log functions.
Quoted 774 lines
Quoted 535 lines
Quoted 535 lines
tests/intel/xe_exec_system_allocator.c new file +535
-0+1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), + 1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i], i)); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11) +#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21) + +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; + +if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) {Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 1048
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+char name[1024 ]; + +igt_assert (!(flags &NEW )); + +sprintf (name,"/tmp/xe_exec_system_allocator_dat%d\n" , +getpid ());Might be another candidate to use
mkstemp()or similar.Will look into that.
Quoted 227 lines
Quoted 227 lines
tests/intel/xe_exec_system_allocator.c new file +227
-0+ file_fd =open (name,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx], idx)); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx], idx)); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE )
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1283
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+munmap (data, bo_size); + else + pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP && (i %2 )) {
The odd() macro defined above can be used here.
Quoted 213 lines
Quoted 25 lines
Quoted 25 lines
tests/intel/xe_exec_system_allocator.c new file +25
-0+if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + +xe_vm_bind_async (fd, vm,0 , bo,0 , +to_user_pointer (data), + bo_size,0 ,0 ); + }else { +if (!bo) { +if (flags &FREE )Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 1314
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+free (data); + else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size);Large memory leaks come from this ^
aligned_alloc(), see below.Yea this will leak, thanks for pointing this out. Thomas also raised memory leaks as a concern.
Inline below with the changes to fix this...
Quoted 9 lines
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c new file +9
-0+igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + }Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1269
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size);data = NULL;Quoted 22 lines
Quoted 22 lines
tests/intel/xe_exec_system_allocator.c new file +22
-0+gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ;Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1361
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +if (flags &MMAP ) +munmap (pending_free[i], bo_size); +else +free (pending_free[i]); + } +free (pending_free); + }else {s/ else {/\nif (data) {
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1365
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data);Something seems wrong with the flags logic when skipping this ^
free()for the allocation pointed above.Yep, see above,
Matt
Quoted 107 lines
Francois
Quoted 105 lines
tests/intel/xe_exec_system_allocator.c new file +105
-0+ } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + }
Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1478
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL );
Just a thought, the SVM page fault count is provided in GT stats:
cat /sys/kernel/debug/dri/*/gt*/stats
svm_pagefault_count: 134785Wondering if it would make sense to read the value before and after test execution then compare the delta with the expected number of page faults.
Francois
Quoted 369 lines
Quoted 369 lines
Quoted 369 lines
tests/intel/xe_exec_system_allocator.c new file +354
-0+ +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1
Quoted 276 lines · Francois Dugast
Hi,
Quoted 70 lines · Matthew Brost
Quoted 9 lines · Francois Dugast
Hi Matt,
I am still going through your patch but sharing some comments already.
The sequence is neither complex nor too different from existing tests but as it is a lot of multi-thread / multi-process code, I am trying to come up with a suggestion to break it down. Might not be possible though.
Good to hear you don't find it too complex, open spliting if it makes sense.
Quoted 56 lines · Matthew Brost
Quoted 44 lines
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)Signed-off-by Matthew Brost <[email protected]>--- include/drm-uapi/xe_drm.h | 57 +- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1832 ++++++++++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1896 insertions(+), 7 deletions(-) create mode 100644 tests/intel/xe_exec_system_allocator.cinclude/drm-uapi/xe_drm.h +0
-0include/drm-uapi/xe_drm.h line 7
Quoted 7 lines
include/drm-uapi/xe_drm.h +2
-2@@ -3,8 +3,8 @@* Copyright © 2023 Intel Corporation */ -#ifndef _XE_DRM_H_ -#define _XE_DRM_H_ +#ifndef _UAPI_XE_DRM_H_ +#define _UAPI_XE_DRM_H_Nit: The header seems to have been copied directly from the kernel tree, instead it should be generated with:
Quoted 7 lines
Quoted 4 lines
make
headers_installLet me split out the uAPI update into its own patch + use the proper flow.
Sounds good.
Quoted 182 lines
Quoted 4 lines
Quoted 4 lines
include/drm-uapi/xe_drm.h +0
-0#include "drm.h" Quoted 9 lines
include/drm-uapi/xe_drm.h extern "C" { · line 137
Quoted 6 lines
include/drm-uapi/xe_drm.h +1
-1@@ -134,7 +134,7 @@ extern "C" {* redefine the interface more easily than an ever growing struct of * increasing complexity, and for large parts of that interface to be * entirely optional. The downside is more pointer chasing; chasing across -* the boundary with pointers encapsulated inside u64. +* the __user boundary with pointers encapsulated inside u64. See above comment on make
headers_install.+1
Quoted 166 lines
Quoted 166 lines
include/drm-uapi/xe_drm.h +47
-4* * Example chaining: * @@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these @@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */ @@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/** @@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */ @@ -1926,4 +1969,4 @@ struct drm_xe_query_eu_stall { }#endif -#endif /* _XE_DRM_H_ */ +#endif /* _UAPI_XE_DRM_H_ */ lib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +4
-0@@ -0,0 +1,1832 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright © 2024 Intel Corporation +*/ Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 12
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Hardware building block +* Mega feature: Shared virtual memory +* Sub-category: execbuf +* Functionality: fault mode, system allocator +* GPU requirements: GPU needs support for DRM_XE_VM_CREATE_FLAG_FAULT_MODE Using this below will help test automation and reporting:
Quoted 7 lines
* TEST: Basic tests for execbuf functionality using system allocator
* Category: Core
* Mega feature: USM
* Sub-category: System allocator
* Functionality: fault mode, system allocator
* GPU: LNL, BMG, PVC
+1
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
tests/intel/xe_exec_system_allocator.c new file +26
-0+*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data; +}; +
Quoted 9 lines
Quoted 6 lines
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c line 44
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__, i__) ((data__)->expected_data)Argument
i__is not used.
Indeed, will remove.
Quoted 120 lines
Quoted 120 lines
Quoted 12 lines
Quoted 12 lines
tests/intel/xe_exec_system_allocator.c new file +12
-0+ +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr; + batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx)Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 60
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +}Slightly out of scope for this review but the 2 functions above might be helpful under
lib/to prevent adding more duplications of the dword write batch sequence.Yea we could split out generic batch writing functions into a library at some point but agree is probably out of scope for this series.
Quoted 13 lines
Quoted 13 lines
tests/intel/xe_exec_system_allocator.c new file +13
-0+ +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) { +struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data, i)); + +if (barrier) +pthread_barrier_wait (barrier);Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 77
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+ } +} + +#define SYNC_FILE "/tmp/xe_exec_system_allocator_sync"Might be worth creating and propagating a unique file name at runtime, for example with
mkstemp(), in order to avoid potential concurrent accesses from multiple instances of the test.Let me look into that.
Quoted 70 lines
Quoted 70 lines
tests/intel/xe_exec_system_allocator.c new file +70
-0+ +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL ); + +close (map_fd); +munmap (pdata,sizeof (*pdata));
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines
tests/intel/xe_exec_system_allocator.c line 152
Quoted 5 lines
tests/intel/xe_exec_system_allocator.c new file +5
-0+} + +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags)It would be helpful to add here a comment similar to the one below for
check_all_pages_threads().
Sure.
Quoted 165 lines
Quoted 154 lines
Quoted 154 lines
Quoted 154 lines
tests/intel/xe_exec_system_allocator.c new file +154
-0+{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec);Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 314
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +printf ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +printf ("FAIL EXEC_UFENCE: EXPEXCTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]);s/EXPEXCTED/EXPECTED/
Quoted 3 lines
Also, we should probably use IGT's log functions such as
igt_info()instead ofprintf().
Will fix.
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c new file +9
-0+ + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +
Quoted 9 lines
Quoted 6 lines
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c line 329
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+printf ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +printf ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +printf ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +printf ("FAIL DARA: EXPEXCTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data);s/DARA/DATA/ and s/EXPEXCTED/EXPECTED/ and same as above for
printf().
Will fix.
Quoted 185 lines
Quoted 24 lines
Quoted 14 lines
Quoted 14 lines
tests/intel/xe_exec_system_allocator.c new file +14
-0+ } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \ + __xe_vm_bind_assert(fd, vm, 0, \ + 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_MAP, \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR, \ + (__sync), (__num_sync), 0, 0)Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 348
Quoted 5 lines
tests/intel/xe_exec_system_allocator.c new file +5
-0+ +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0)Is there a reason here to favor static variable + macros over helper function with parameters?
Quoted 4 lines
va_bitsis static as it looked up exactly once when the test loads.I could change these helpers to static functions rather than macros if that is preferred.
It forces variable names in the caller but that is just a minor issue.
Quoted 154 lines
Quoted 154 lines
Quoted 154 lines
tests/intel/xe_exec_system_allocator.c new file +153
-0+ +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else if (flags &CPU_FAULT_THREADS ) + check_all_pages_threads(allocs[i].ptr, alloc_size, stride,
Quoted 9 lines
Quoted 6 lines
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c line 507
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+ NUM_CHECK_THREADS, flags); + else + check_all_pages(allocs[i].ptr, alloc_size, stride, NULL); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); + printf("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n",Also here, prefer IGT's log functions.
Will fix.
Quoted 776 lines
Quoted 776 lines
Quoted 535 lines
Quoted 535 lines
tests/intel/xe_exec_system_allocator.c new file +535
-0+1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), + 1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i], i)); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11) +#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21) + +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; + +if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) {Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 1048
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+char name[1024 ]; + +igt_assert (!(flags &NEW )); + +sprintf (name,"/tmp/xe_exec_system_allocator_dat%d\n" , +getpid ());Might be another candidate to use
mkstemp()or similar.Will look into that.
Quoted 229 lines
Quoted 229 lines
tests/intel/xe_exec_system_allocator.c new file +229
-0+ file_fd =open (name,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx], idx)); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx], idx)); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx], idx)); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE ) +munmap (data, bo_size); +else
Quoted 9 lines
Quoted 6 lines
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c line 1283
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+ pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP && (i %2 )) {The
odd()macro defined above can be used here.
Yep.
Quoted 224 lines
Quoted 213 lines
Quoted 25 lines
Quoted 25 lines
tests/intel/xe_exec_system_allocator.c new file +25
-0+if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + +xe_vm_bind_async (fd, vm,0 , bo,0 , +to_user_pointer (data), + bo_size,0 ,0 ); + }else { +if (!bo) { +if (flags &FREE )Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 1314
Quoted 6 lines
tests/intel/xe_exec_system_allocator.c new file +6
-0+free (data); + else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size);Large memory leaks come from this ^
aligned_alloc(), see below.Yea this will leak, thanks for pointing this out. Thomas also raised memory leaks as a concern.
Inline below with the changes to fix this...
Quoted 9 lines
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c new file +9
-0+igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + }Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1269
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size);data = NULL;Quoted 22 lines
Quoted 22 lines
tests/intel/xe_exec_system_allocator.c new file +22
-0+gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ;Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1361
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +if (flags &MMAP ) +munmap (pending_free[i], bo_size); +else +free (pending_free[i]); + } +free (pending_free); + }else {s/ else {/\nif (data) {
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1365
Quoted 4 lines
tests/intel/xe_exec_system_allocator.c new file +4
-0+if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data);Something seems wrong with the flags logic when skipping this ^
free()for the allocation pointed above.Yep, see above,
Matt
Quoted 107 lines
Francois
Quoted 105 lines
tests/intel/xe_exec_system_allocator.c new file +105
-0+ } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + }Quoted 8 lines
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c line 1478
Quoted 8 lines
tests/intel/xe_exec_system_allocator.c new file +8
-0+ +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL );Just a thought, the SVM page fault count is provided in GT stats:
Quoted 6 lines
cat /sys/kernel/debug/dri/*/gt*/stats svm_pagefault_count: 134785Wondering if it would make sense to read the value before and after test execution then compare the delta with the expected number of page faults.
I think that is a bit out of scope here and number of faults isn't 100% predictable because of intentional races, but I was thinking for prefetch tests we could hook into page fault stats to ensure no page faults occur if prefetches are done.
Matt
Quoted 371 lines
Francois
Quoted 369 lines
Quoted 369 lines
Quoted 369 lines
tests/intel/xe_exec_system_allocator.c new file +354
-0+ +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (SYNC_FILE ,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (SYNC_FILE ,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1