Radeon memory management improvements v3
Subsystems (3)
cover Radeon memory management improvements v3
While updating "[PATCH 5/6] drm/radeon: validate relocations in the order determined by userspace" based on feedback, which should have been a harmless change, I discovered that the performance dropped. The problem was that list_add/move from one list to another reversed the list of relocations and the only thing which restored performance was to sort the relocation list with a stable sort. (preserve ordering of buffers whose priorities are equal). The only explanation I've come up with is that the buffers which appear sooner in the list are likely to be used more often by an IB than buffers which appear later in the list. For example, a texture used in all draw commands will most probably be somewhere at the beginning. However, a texture which appears at the end is probably only used in a
few last draw commands.
Also, the stats are changed to atomic64_t as discusssed before.
Marek
PATCH 1/6 drm/radeon: add a way to get and set initial buffer domains
From: Marek Olšák <[email protected]>
When passing buffers between processes, the receiving process needs to know the original buffer domain, so that it doesn't accidentally move the buffer.
v2: reserve the buffer
--- drivers/gpu/drm/radeon/radeon.h | 3 +++ drivers/gpu/drm/radeon/radeon_drv.c | 3 ++- drivers/gpu/drm/radeon/radeon_gem.c | 36 ++++++++++++++++++++++++++++++++++ drivers/gpu/drm/radeon/radeon_kms.c | 1 + drivers/gpu/drm/radeon/radeon_object.c | 3 +++ include/uapi/drm/radeon_drm.h | 12 ++++++++++++ 6 files changed, 57 insertions(+), 1 deletion(-)
drivers/gpu/drm/radeon/radeon.h +3 -0
@@ -454,6 +454,7 @@ struct radeon_bo {/* Protected by gem.mutex */ struct list_head list;/* Protected by tbo.reserved */ +u32 initial_domain;u32 placements[3 ];struct ttm_placement placement;struct ttm_buffer_object tbo;
@@ -2114,6 +2115,8 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);int radeon_gem_va_ioctl (struct drm_device *dev,void *data,struct drm_file *filp); +int radeon_gem_op_ioctl (struct drm_device *dev,void *data, +struct drm_file *filp);int radeon_cs_ioctl (struct drm_device *dev,void *data,struct drm_file *filp);int radeon_gem_set_tiling_ioctl (struct drm_device *dev,void *data,struct drm_file *filp);
drivers/gpu/drm/radeon/radeon_drv.c +2 -1
@@ -79,9 +79,10 @@* 2.35.0 - Add CIK macrotile mode array query * 2.36.0 - Fix CIK DCE tiling setup * 2.37.0 - allow GS ring setup on r6xx/r7xx +* 2.38.0 - RADEON_GEM_OP (GET_INITIAL_DOMAIN, SET_INITIAL_DOMAIN) */ #define KMS_DRIVER_MAJOR 2 -#define KMS_DRIVER_MINOR 37 +#define KMS_DRIVER_MINOR 38#define KMS_DRIVER_PATCHLEVEL 0int radeon_driver_load_kms (struct drm_device *dev,unsigned long flags);int radeon_driver_unload_kms (struct drm_device *dev);
drivers/gpu/drm/radeon/radeon_gem.c +36 -0
@@ -533,6 +533,42 @@ out:return r; } +int radeon_gem_op_ioctl (struct drm_device *dev,void *data, +struct drm_file *filp) +{ +struct drm_radeon_gem_op *args = data; +struct drm_gem_object *gobj; +struct radeon_bo *robj; +int r; + + gobj =drm_gem_object_lookup (dev, filp, args->handle); +if (gobj ==NULL ) { +return -ENOENT ; + } + robj =gem_to_radeon_bo (gobj); + r =radeon_bo_reserve (robj, false); +if (unlikely (r)) + goto out; + +switch (args->op) { +case RADEON_GEM_OP_GET_INITIAL_DOMAIN : + args->value = robj->initial_domain; +break ; +case RADEON_GEM_OP_SET_INITIAL_DOMAIN : + robj->initial_domain = args->value & (RADEON_GEM_DOMAIN_VRAM | +RADEON_GEM_DOMAIN_GTT | +RADEON_GEM_DOMAIN_CPU ); +break ; +default : + r = -EINVAL ; + } + +radeon_bo_unreserve (robj); +out: +drm_gem_object_unreference_unlocked (gobj); +return r; +} +int radeon_mode_dumb_create (struct drm_file *file_priv,struct drm_device *dev,struct drm_mode_create_dumb *args)
drivers/gpu/drm/radeon/radeon_kms.c +1 -0
@@ -814,5 +814,6 @@ const struct drm_ioctl_desc radeon_ioctls_kms[] = {DRM_IOCTL_DEF_DRV (RADEON_GEM_GET_TILING , radeon_gem_get_tiling_ioctl,DRM_AUTH |DRM_UNLOCKED |DRM_RENDER_ALLOW ),DRM_IOCTL_DEF_DRV (RADEON_GEM_BUSY , radeon_gem_busy_ioctl,DRM_AUTH |DRM_UNLOCKED |DRM_RENDER_ALLOW ),DRM_IOCTL_DEF_DRV (RADEON_GEM_VA , radeon_gem_va_ioctl,DRM_AUTH |DRM_UNLOCKED |DRM_RENDER_ALLOW ), + DRM_IOCTL_DEF_DRV(RADEON_GEM_OP, radeon_gem_op_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW), };int radeon_max_kms_ioctl =DRM_ARRAY_SIZE (radeon_ioctls_kms);
drivers/gpu/drm/radeon/radeon_object.c +3 -0
@@ -145,6 +145,9 @@ int radeon_bo_create(struct radeon_device *rdev, bo->surface_reg =-1 ;INIT_LIST_HEAD (&bo->list);INIT_LIST_HEAD (&bo->va); + bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM | +RADEON_GEM_DOMAIN_GTT | +RADEON_GEM_DOMAIN_CPU );radeon_ttm_placement_from_domain (bo, domain);/* Kernel allocation are uninterruptible */ down_read (&rdev->pm.mclk_lock);
include/uapi/drm/radeon_drm.h +12 -0
@@ -510,6 +510,7 @@ typedef struct {#define DRM_RADEON_GEM_GET_TILING 0x29#define DRM_RADEON_GEM_BUSY 0x2a#define DRM_RADEON_GEM_VA 0x2b +#define DRM_RADEON_GEM_OP 0x2c#define DRM_IOCTL_RADEON_CP_INIT DRM_IOW( DRM_COMMAND_BASE + DRM_RADEON_CP_INIT, drm_radeon_init_t)#define DRM_IOCTL_RADEON_CP_START DRM_IO( DRM_COMMAND_BASE + DRM_RADEON_CP_START)
@@ -552,6 +553,7 @@ typedef struct {#define DRM_IOCTL_RADEON_GEM_GET_TILING DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_GET_TILING, struct drm_radeon_gem_get_tiling)#define DRM_IOCTL_RADEON_GEM_BUSY DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_BUSY, struct drm_radeon_gem_busy)#define DRM_IOCTL_RADEON_GEM_VA DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_VA, struct drm_radeon_gem_va) +#define DRM_IOCTL_RADEON_GEM_OP DRM_IOWR(DRM_COMMAND_BASE + DRM_RADEON_GEM_OP, struct drm_radeon_gem_op) typedef struct drm_radeon_init { enum {
@@ -884,6 +886,16 @@ struct drm_radeon_gem_pwrite {uint64_t data_ptr; }; +/* Sets or returns a value associated with a buffer. */ +struct drm_radeon_gem_op { +uint32_t handle;/* buffer */ +uint32_t op;/* RADEON_GEM_OP_* */ +uint64_t value;/* input or return value */ +}; + +#define RADEON_GEM_OP_GET_INITIAL_DOMAIN 0 +#define RADEON_GEM_OP_SET_INITIAL_DOMAIN 1 +#define RADEON_VA_MAP 1#define RADEON_VA_UNMAP 2
Signature · 6 lines
1.8.3.2 _______________________________________________ dri-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH 2/6 drm/radeon: track memory statistics about VRAM and GTT usage and buffer moves
In reply to Marek OlšákFrom: Marek Olšák <[email protected]>
The statistics are:
- VRAM usage in bytes
- GTT usage in bytes
- number of bytes moved by TTM
The last one is actually a counter, so you need to sample it before and after command submission and take the difference.
This is useful for finding performance bottlenecks. Userspace queries are also added.
v2: use atomic64_t
--- drivers/gpu/drm/radeon/radeon.h | 4 ++++ drivers/gpu/drm/radeon/radeon_kms.c | 15 ++++++++++++++ drivers/gpu/drm/radeon/radeon_object.c | 36 +++++++++++++++++++++++++++++++++- drivers/gpu/drm/radeon/radeon_object.h | 2 +- drivers/gpu/drm/radeon/radeon_ttm.c | 8 +++++++- include/uapi/drm/radeon_drm.h | 3 +++ 6 files changed, 65 insertions(+), 3 deletions(-)
drivers/gpu/drm/radeon/radeon.h +4 -0
@@ -2307,6 +2307,10 @@ struct radeon_device {/* virtual memory */ struct radeon_vm_manager vm_manager;struct mutex gpu_clock_mutex; +/* memory stats */ +atomic64_t vram_usage; +atomic64_t gtt_usage; +atomic64_t num_bytes_moved;/* ACPI interface */ struct radeon_atif atif;struct radeon_atcs atcs;
drivers/gpu/drm/radeon/radeon_kms.c +15 -0
@@ -486,6 +486,21 @@ static int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_filecase RADEON_INFO_VCE_FB_VERSION : *value = rdev->vce.fb_version;break ; +case RADEON_INFO_NUM_BYTES_MOVED : + value = (uint32_t *)&value64; + value_size =sizeof (uint64_t ); + value64 =atomic64_read (&rdev->num_bytes_moved); +break ; +case RADEON_INFO_VRAM_USAGE : + value = (uint32_t *)&value64; + value_size =sizeof (uint64_t ); + value64 =atomic64_read (&rdev->vram_usage); +break ; +case RADEON_INFO_GTT_USAGE : + value = (uint32_t *)&value64; + value_size =sizeof (uint64_t ); + value64 =atomic64_read (&rdev->gtt_usage); +break ;default :DRM_DEBUG_KMS ("Invalid request %d\n" , info->request);return -EINVAL ;
drivers/gpu/drm/radeon/radeon_object.c +35 -1
@@ -56,11 +56,36 @@ static void radeon_bo_clear_va(struct radeon_bo *bo) } } +static void radeon_update_memory_usage (struct radeon_bo *bo, +unsigned mem_type,int sign) +{ +struct radeon_device *rdev = bo->rdev; +u64 size = (u64 )bo->tbo.num_pages <<PAGE_SHIFT ; + +switch (mem_type) { +case TTM_PL_TT : +if (sign >0 ) +atomic64_add (size, &rdev->gtt_usage); +else +atomic64_sub (size, &rdev->gtt_usage); +break ; +case TTM_PL_VRAM : +if (sign >0 ) +atomic64_add (size, &rdev->vram_usage); +else +atomic64_sub (size, &rdev->vram_usage); +break ; + } +} +static void radeon_ttm_bo_destroy (struct ttm_buffer_object *tbo) {struct radeon_bo *bo; bo = container_of(tbo, struct radeon_bo, tbo); + +radeon_update_memory_usage (bo, bo->tbo.mem.mem_type,-1 ); +mutex_lock (&bo->rdev->gem.mutex);list_del_init (&bo->list);mutex_unlock (&bo->rdev->gem.mutex);
@@ -567,14 +592,23 @@ int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved, }void radeon_bo_move_notify (struct ttm_buffer_object *bo, -struct ttm_mem_reg *mem) +struct ttm_mem_reg *new_mem) {struct radeon_bo *rbo; +if (!radeon_ttm_bo_is_radeon_bo (bo))return ; + rbo = container_of(bo, struct radeon_bo, tbo);radeon_bo_check_tiling (rbo,0 ,1 );radeon_vm_bo_invalidate (rbo->rdev, rbo); + +/* update statistics */ +if (!new_mem) +return ; + +radeon_update_memory_usage (rbo, bo->mem.mem_type,-1 ); +radeon_update_memory_usage (rbo, new_mem->mem_type,1 ); }int radeon_bo_fault_reserve_notify (struct ttm_buffer_object *bo)
drivers/gpu/drm/radeon/radeon_object.h +1 -1
@@ -151,7 +151,7 @@ extern void radeon_bo_get_tiling_flags(struct radeon_bo *bo,extern int radeon_bo_check_tiling (struct radeon_bo *bo,bool has_moved,bool force_drop);extern void radeon_bo_move_notify (struct ttm_buffer_object *bo, -struct ttm_mem_reg *mem); +struct ttm_mem_reg *new_mem);extern int radeon_bo_fault_reserve_notify (struct ttm_buffer_object *bo);extern int radeon_bo_get_surface_reg (struct radeon_bo *bo);
drivers/gpu/drm/radeon/radeon_ttm.c +7 -1
@@ -406,8 +406,14 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,if (r) { memcpy: r =ttm_bo_move_memcpy (bo, evict, no_wait_gpu, new_mem); +if (r) { +return r; + } } -return r; + +/* update statistics */ +atomic64_add ((u64 )bo->num_pages <<PAGE_SHIFT , &rdev->num_bytes_moved); +return 0 ; }static int radeon_ttm_io_mem_reserve (struct ttm_bo_device *bdev,struct ttm_mem_reg *mem)
include/uapi/drm/radeon_drm.h +3 -0
@@ -1004,6 +1004,9 @@ struct drm_radeon_cs {#define RADEON_INFO_VCE_FW_VERSION 0x1b/* version of VCE feedback */ #define RADEON_INFO_VCE_FB_VERSION 0x1c +#define RADEON_INFO_NUM_BYTES_MOVED 0x1d +#define RADEON_INFO_VRAM_USAGE 0x1e +#define RADEON_INFO_GTT_USAGE 0x1fstruct drm_radeon_info {
Signature · 6 lines
1.8.3.2 _______________________________________________ dri-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH 3/6 drm/radeon: deduplicate code in radeon_gem_busy_ioctl
In reply to Marek OlšákFrom: Marek Olšák <[email protected]>
--- drivers/gpu/drm/radeon/radeon_gem.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-)
drivers/gpu/drm/radeon/radeon_gem.c +1 -12
@@ -344,18 +344,7 @@ int radeon_gem_busy_ioctl(struct drm_device *dev, void *data, } robj =gem_to_radeon_bo (gobj); r =radeon_bo_wait (robj, &cur_placement, true); -switch (cur_placement) { -case TTM_PL_VRAM : - args->domain =RADEON_GEM_DOMAIN_VRAM ; -break ; -case TTM_PL_TT : - args->domain =RADEON_GEM_DOMAIN_GTT ; -break ; -case TTM_PL_SYSTEM : - args->domain =RADEON_GEM_DOMAIN_CPU ; -default : -break ; - } + args->domain =radeon_mem_type_to_domain (cur_placement);drm_gem_object_unreference_unlocked (gobj); r =radeon_gem_handle_lockup (rdev, r);return r;
Signature · 6 lines
1.8.3.2 _______________________________________________ dri-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH 4/6 drm/radeon: add buffers to the LRU list from smallest to largest
In reply to Marek OlšákFrom: Marek Olšák <[email protected]>
--- drivers/gpu/drm/radeon/radeon_cs.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
drivers/gpu/drm/radeon/radeon_cs.c +23 -0
@@ -24,6 +24,7 @@* Authors: * Jerome Glisse <[email protected]> */ +#include <linux/list_sort.h> #include <drm/drmP.h> #include <drm/radeon_drm.h> #include "radeon_reg.h"
@@ -290,6 +291,16 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)return 0 ; } +static int cmp_size_smaller_first (void *priv,struct list_head *a, +struct list_head *b) +{ + struct radeon_bo_list *la = list_entry(a, struct radeon_bo_list, tv.head); + struct radeon_bo_list *lb = list_entry(b, struct radeon_bo_list, tv.head); + +/* Sort A before B if A is smaller. */ +return (int )la->bo->tbo.num_pages - (int )lb->bo->tbo.num_pages; +} +/** * cs_parser_fini() - clean parser states * @parser: parser structure holding parsing context.
@@ -303,6 +314,18 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bounsigned i;if (!error) { +/* Sort the buffer list from the smallest to largest buffer, +* which affects the order of buffers in the LRU list. +* This assures that the smallest buffers are added first +* to the LRU list, so they are likely to be later evicted +* first, instead of large buffers whose eviction is more +* expensive. +* +* This slightly lowers the number of bytes moved by TTM +* per frame under memory pressure. +*/ +list_sort (NULL , &parser->validated, cmp_size_smaller_first); +ttm_eu_fence_buffer_objects (&parser->ticket, &parser->validated, parser->ib.fence);
Signature · 6 lines
1.8.3.2 _______________________________________________ dri-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH 5/6 drm/radeon: validate relocations in the order determined by userspace
In reply to Marek OlšákFrom: Marek Olšák <[email protected]>
Userspace should set the first 4 bits of drm_radeon_cs_reloc::flags to a number from 0 to 15. The higher the number, the higher the priority, which means a buffer with a higher number will be validated sooner.
The old behavior is preserved: Buffers used for write are prioritized over read-only buffers if the userspace doesn't set the number.
v2: add buffers to buckets directly, then concatenate them
v3: use a stable sort
--- drivers/gpu/drm/radeon/radeon.h | 1 - drivers/gpu/drm/radeon/radeon_cs.c | 64 ++++++++++++++++++++++++++++++++-- drivers/gpu/drm/radeon/radeon_object.c | 10 ------ drivers/gpu/drm/radeon/radeon_object.h | 2 -- 4 files changed, 61 insertions(+), 16 deletions(-)
drivers/gpu/drm/radeon/radeon.h +0 -1
@@ -481,7 +481,6 @@ struct radeon_bo_list {struct ttm_validate_buffer tv;struct radeon_bo *bo;uint64_t gpu_offset; -bool written;unsigned domain;unsigned alt_domain;u32 tiling_flags;
drivers/gpu/drm/radeon/radeon_cs.c +61 -3
101 lines not on this page · raw message
drivers/gpu/drm/radeon/radeon_object.c +0 -10
@@ -366,16 +366,6 @@ void radeon_bo_fini(struct radeon_device *rdev)arch_phys_wc_del (rdev->mc.vram_mtrr); } -void radeon_bo_list_add_object (struct radeon_bo_list *lobj, -struct list_head *head) -{ -if (lobj->written) { -list_add (&lobj->tv.head, head); - }else { -list_add_tail (&lobj->tv.head, head); - } -} -int radeon_bo_list_validate (struct ww_acquire_ctx *ticket,struct list_head *head,int ring) {
drivers/gpu/drm/radeon/radeon_object.h +0 -2
@@ -138,8 +138,6 @@ extern int radeon_bo_evict_vram(struct radeon_device *rdev);extern void radeon_bo_force_delete (struct radeon_device *rdev);extern int radeon_bo_init (struct radeon_device *rdev);extern void radeon_bo_fini (struct radeon_device *rdev); -extern void radeon_bo_list_add_object (struct radeon_bo_list *lobj, -struct list_head *head);extern int radeon_bo_list_validate (struct ww_acquire_ctx *ticket,struct list_head *head,int ring); extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
Signature · 6 lines
1.8.3.2 _______________________________________________ dri-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH 6/6 drm/radeon: limit how much memory TTM can move per IB according to VRAM usage
In reply to Marek OlšákFrom: Marek Olšák <[email protected]>
--- drivers/gpu/drm/radeon/radeon_cs.c | 2 +- drivers/gpu/drm/radeon/radeon_object.c | 88 +++++++++++++++++++++++++++++++--- drivers/gpu/drm/radeon/radeon_object.h | 3 +- 3 files changed, 85 insertions(+), 8 deletions(-)
drivers/gpu/drm/radeon/radeon_cs.c +1 -1
@@ -168,7 +168,7 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)radeon_cs_buckets_get_list (&buckets, &p->validated); -return radeon_bo_list_validate (&p->ticket, &p->validated, p->ring); +return radeon_bo_list_validate (p->rdev, &p->ticket, &p->validated, p->ring); }static int radeon_cs_get_ring (struct radeon_cs_parser *p,u32 ring,s32 priority)
drivers/gpu/drm/radeon/radeon_object.c +82 -6
111 lines not on this page · raw message
drivers/gpu/drm/radeon/radeon_object.h +2 -1
@@ -138,7 +138,8 @@ extern int radeon_bo_evict_vram(struct radeon_device *rdev);extern void radeon_bo_force_delete (struct radeon_device *rdev);extern int radeon_bo_init (struct radeon_device *rdev);extern void radeon_bo_fini (struct radeon_device *rdev); -extern int radeon_bo_list_validate (struct ww_acquire_ctx *ticket, +extern int radeon_bo_list_validate (struct radeon_device *rdev, +struct ww_acquire_ctx *ticket,struct list_head *head,int ring);extern int radeon_bo_fbdev_mmap (struct radeon_bo *bo,struct vm_area_struct *vma);
Signature · 6 lines
1.8.3.2 _______________________________________________ dri-devel mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/dri-devel