Virtio-GPU suspend and resume
Subsystems (3)
v2 cover Virtio-GPU suspend and resume
From: Dongwon Kim <[email protected]>
This patch series introduces a freeze and restore mechanism for the virtio-gpu driver:
First patch adds virtgpu_freeze and virtgpu_restore functions. These functions handle the deletion of virtio queues before suspension and their recreation during the restoration process.
Second patch implements a mechanism for restoring virtio_gpu_object instances. This is necessary because the host (QEMU) deletes all associated resources during the virtio-gpu reset, which occurs as part of the restoration process.
These changes ensure that the virtio-gpu driver can properly handle suspend and resume scenarios without resource loss.
Dongwon Kim (2):
drm/virtio: Freeze and restore hooks to support suspend and resume drm/virtio: Implement save and restore for virtio_gpu_objects
drivers/gpu/drm/virtio/virtgpu_drv.c | 65 +++++++++++++++++++++- drivers/gpu/drm/virtio/virtgpu_drv.h | 11 ++++ drivers/gpu/drm/virtio/virtgpu_kms.c | 24 ++++++--- drivers/gpu/drm/virtio/virtgpu_object.c | 72 +++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 7 deletions(-)
Signature · 1 line
2.34.1
RFC PATCH v2 1/2 drm/virtio: Freeze and restore hooks to support suspend and resume
From: Dongwon Kim <[email protected]>
virtio device needs to delete before VM suspend happens then reinitialize all virtqueues again upon resume
v2: 10ms sleep was added in virtgpu_freeze to avoid the situation
the driver is locked up during resumption.
--- drivers/gpu/drm/virtio/virtgpu_drv.c | 59 +++++++++++++++++++++++++++- drivers/gpu/drm/virtio/virtgpu_drv.h | 1 + drivers/gpu/drm/virtio/virtgpu_kms.c | 23 ++++++++--- 3 files changed, 76 insertions(+), 7 deletions(-)
drivers/gpu/drm/virtio/virtgpu_drv.c +58 -1
@@ -32,6 +32,7 @@#include <linux/poll.h> #include <linux/vgaarb.h> #include <linux/wait.h> +#include <linux/delay.h> #include <drm/clients/drm_client_setup.h> #include <drm/drm.h>
@@ -163,6 +164,58 @@ static unsigned int features[] = {VIRTIO_GPU_F_RESOURCE_BLOB , VIRTIO_GPU_F_CONTEXT_INIT, }; + +#ifdef CONFIG_PM_SLEEP +static int virtgpu_freeze (struct virtio_device *vdev) +{ +struct drm_device *dev = vdev->priv; +struct virtio_gpu_device *vgdev = dev->dev_private; +int error; + + error =drm_mode_config_helper_suspend (dev); +if (error) { +DRM_ERROR ("suspend error %d\n" , error); +return error; + } + +/* TODO: Without this short sleep, virtio-gpu driver often hangs +* during restore while executing drm_mode_config_helper_resume +*/ +usleep_range (10000 ,10100 ); + +flush_work (&vgdev->obj_free_work); +flush_work (&vgdev->ctrlq.dequeue_work); +flush_work (&vgdev->cursorq.dequeue_work); +flush_work (&vgdev->config_changed_work); + vdev->config->del_vqs (vdev); + +return 0 ; +} + +static int virtgpu_restore (struct virtio_device *vdev) +{ +struct drm_device *dev = vdev->priv; +struct virtio_gpu_device *vgdev = dev->dev_private; +int error; + + error =virtio_gpu_find_vqs (vgdev); +if (error) { +DRM_ERROR ("failed to find virt queues\n" ); +return error; + } + +virtio_device_ready (vdev); + + error =drm_mode_config_helper_resume (dev); +if (error) { +DRM_ERROR ("resume error %d\n" , error); +return error; + } + +return 0 ; +} +#endif +static struct virtio_driver virtio_gpu_driver = { .feature_table = features, .feature_table_size = ARRAY_SIZE(features),
@@ -171,7 +224,11 @@ static struct virtio_driver virtio_gpu_driver = { .probe = virtio_gpu_probe, .remove = virtio_gpu_remove, .shutdown = virtio_gpu_shutdown, - .config_changed = virtio_gpu_config_changed + .config_changed = virtio_gpu_config_changed, +#ifdef CONFIG_PM_SLEEP + .freeze = virtgpu_freeze, + .restore = virtgpu_restore, +#endif };static int __init virtio_gpu_driver_init (void )
drivers/gpu/drm/virtio/virtgpu_drv.h +1 -0
@@ -300,6 +300,7 @@ void virtio_gpu_deinit(struct drm_device *dev);void virtio_gpu_release (struct drm_device *dev);int virtio_gpu_driver_open (struct drm_device *dev,struct drm_file *file);void virtio_gpu_driver_postclose (struct drm_device *dev,struct drm_file *file); +int virtio_gpu_find_vqs (struct virtio_gpu_device *vgdev);/* virtgpu_gem.c */ int virtio_gpu_gem_object_open(struct drm_gem_object *obj,
drivers/gpu/drm/virtio/virtgpu_kms.c +17 -6
@@ -114,15 +114,28 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev, vgdev->num_capsets = num_capsets; } -int virtio_gpu_init (struct virtio_device *vdev,struct drm_device *dev) +int virtio_gpu_find_vqs (struct virtio_gpu_device *vgdev) {struct virtqueue_info vqs_info[] = { {"control" , virtio_gpu_ctrl_ack }, {"cursor" , virtio_gpu_cursor_ack }, }; -struct virtio_gpu_device *vgdev; -/* this will expand later */ struct virtqueue *vqs[2 ]; +int ret; + + ret =virtio_find_vqs (vgdev->vdev,2 , vqs, vqs_info,NULL ); +if (ret) +return ret; + + vgdev->ctrlq.vq = vqs[0 ]; + vgdev->cursorq.vq = vqs[1 ]; + +return 0 ; +} + +int virtio_gpu_init (struct virtio_device *vdev,struct drm_device *dev) +{ +struct virtio_gpu_device *vgdev;u32 num_scanouts, num_capsets;int ret =0 ;
@@ -206,13 +219,11 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)DRM_INFO ("features: %ccontext_init\n" , vgdev->has_context_init ?'+' :'-' ); - ret =virtio_find_vqs (vgdev->vdev,2 , vqs, vqs_info,NULL ); + ret =virtio_gpu_find_vqs (vgdev);if (ret) {DRM_ERROR ("failed to find virt queues\n" ); goto err_vqs; } - vgdev->ctrlq.vq = vqs[0 ]; - vgdev->cursorq.vq = vqs[1 ]; ret =virtio_gpu_alloc_vbufs (vgdev);if (ret) {DRM_ERROR ("failed to alloc vbufs\n" );
Signature · 1 line
2.34.1
RFC PATCH v2 2/2 drm/virtio: Implement save and restore for virtio_gpu_objects
In reply to [email protected]From: Dongwon Kim <[email protected]>
Host KVM/QEMU loses all graphics resources submitted by the guest OS upon resumption from sleep or hibernation. This results in invalid resource errors when the guest OS attempts to interact with the host regarding those resources.
To address this issue, the virtio-gpu driver now resubmits all existing resources upon resumption. A linked list has been introduced to maintain references to all created virtio_gpu_object instances and their parameters.
Whenever a new object is created and sent to the host, it is added to this list. During the .resume function, all backed-up objects are re-sent to the host using the 'create resource' virtio GPU command, ensuring the resources are restored seamlessly.
v2: - reset bo->attached if bo->attached was set before so that attach
backing can be redone upon restore
--- drivers/gpu/drm/virtio/virtgpu_drv.c | 6 +++ drivers/gpu/drm/virtio/virtgpu_drv.h | 10 ++++ drivers/gpu/drm/virtio/virtgpu_kms.c | 1 + drivers/gpu/drm/virtio/virtgpu_object.c | 72 +++++++++++++++++++++++++ 4 files changed, 89 insertions(+)
drivers/gpu/drm/virtio/virtgpu_drv.c +6 -0
@@ -206,6 +206,12 @@ static int virtgpu_restore(struct virtio_device *vdev)virtio_device_ready (vdev); + error =virtio_gpu_object_restore_all (vgdev); +if (error) { +DRM_ERROR ("Failed to recover objects\n" ); +return error; + } + error =drm_mode_config_helper_resume (dev);if (error) {DRM_ERROR ("resume error %d\n" , error);
drivers/gpu/drm/virtio/virtgpu_drv.h +10 -0
@@ -126,6 +126,12 @@ struct virtio_gpu_object_array { struct drm_gem_object *objs[] __counted_by(total); }; +struct virtio_gpu_object_restore { +struct virtio_gpu_object *bo; +struct virtio_gpu_object_params params; +struct list_head node; +}; +struct virtio_gpu_vbuffer ;struct virtio_gpu_device ;
@@ -265,6 +271,7 @@ struct virtio_gpu_device {struct work_struct obj_free_work;spinlock_t obj_free_lock;struct list_head obj_free_list; +struct list_head obj_restore;struct virtio_gpu_drv_capset *capsets;uint32_t num_capsets;
@@ -479,6 +486,9 @@ bool virtio_gpu_is_shmem(struct virtio_gpu_object *bo);int virtio_gpu_resource_id_get (struct virtio_gpu_device *vgdev,uint32_t *resid); + +int virtio_gpu_object_restore_all (struct virtio_gpu_device *vgdev); +/* virtgpu_prime.c */ int virtio_gpu_resource_assign_uuid (struct virtio_gpu_device *vgdev,struct virtio_gpu_object *bo);
drivers/gpu/drm/virtio/virtgpu_kms.c +1 -0
@@ -162,6 +162,7 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev) vgdev->fence_drv.context =dma_fence_context_alloc (1 );spin_lock_init (&vgdev->fence_drv.lock);INIT_LIST_HEAD (&vgdev->fence_drv.fences); +INIT_LIST_HEAD (&vgdev->obj_restore);INIT_LIST_HEAD (&vgdev->cap_cache);INIT_WORK (&vgdev->config_changed_work, virtio_gpu_config_changed_work_func);
drivers/gpu/drm/virtio/virtgpu_object.c +72 -0
95 lines not on this page · raw message
Signature · 1 line
2.34.1
Dmitry Osipenko
maintainerHi,
Quoted 20 lines · On 5/24/25 01:00, [email protected] wrote:
From: Dongwon Kim <[email protected]>
This patch series introduces a freeze and restore mechanism for the virtio-gpu driver:
First patch adds
virtgpu_freezeandvirtgpu_restorefunctions. These functions handle the deletion of virtio queues before suspension and their recreation during the restoration process.Second patch implements a mechanism for restoring
virtio_gpu_objectinstances. This is necessary because the host (QEMU) deletes all associated resources during the virtio-gpu reset, which occurs as part of the restoration process.These changes ensure that the virtio-gpu driver can properly handle suspend and resume scenarios without resource loss.
Dongwon Kim (2):
drm/virtio: Freeze and restore hooks to support suspend and resume drm/virtio: Implement save and restore forvirtio_gpu_objects
Quoted 5 lines
drivers/gpu/drm/virtio/virtgpu_drv.c | 65 +++++++++++++++++++++- drivers/gpu/drm/virtio/virtgpu_drv.h | 11 ++++ drivers/gpu/drm/virtio/virtgpu_kms.c | 24 ++++++--- drivers/gpu/drm/virtio/virtgpu_object.c | 72 +++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 7 deletions(-)
Tested the patches, applied to v6.15.2. Suspend-resume works with v2, display works on resume.
Have you figured out why 10ms workaround is needed?
Signature · 2 lines
Best regards, Dmitry
Hi Dmitry,
Quoted 34 lines
Subject: Re: [RFC PATCH v2 0/2] Virtio-GPU suspend and resume
Hi,
Quoted 10 lines · On 5/24/25 01:00, [email protected] wrote:
From: Dongwon Kim <[email protected]>
This patch series introduces a freeze and restore mechanism for the virtio-gpu driver:
First patch adds
virtgpu_freezeandvirtgpu_restorefunctions. These functions handle the deletion of virtio queues before suspension and their recreation during the restoration process.Second patch implements a mechanism for restoring
virtio_gpu_objectinstances.
Quoted 2 lines
This is necessary because the host (QEMU) deletes all associated resources during the virtio-gpu reset, which occurs as part of the
restoration process.
Quoted 8 lines
These changes ensure that the virtio-gpu driver can properly handle suspend and resume scenarios without resource loss.
Dongwon Kim (2):
drm/virtio: Freeze and restore hooks to support suspend and resume drm/virtio: Implement save and restore forvirtio_gpu_objectsQuoted 6 lines
drivers/gpu/drm/virtio/virtgpu_drv.c | 65 +++++++++++++++++++++- drivers/gpu/drm/virtio/virtgpu_drv.h | 11 ++++ drivers/gpu/drm/virtio/virtgpu_kms.c | 24 ++++++--- drivers/gpu/drm/virtio/virtgpu_object.c | 72 +++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 7 deletions(-)
Quoted 4 lines
Tested the patches, applied to v6.15.2. Suspend-resume works with v2, display works on resume.
Have you figured out why 10ms workaround is needed?
[Kim, Dongwon] Unfortunately, I don't know why it fails without the delay. I wanted to narrow down further so enabled printk during suspend and resume but hang didn't occur with the timing changes caused by printks. I've also tried more deterministic methods that make it wait based on some kinds of "status" but none of them have worked so far. If you have any suggestions on possible condition we can check instead of just sleeping, please let me know.
10ms seems to be close to minimum to make it work 100% for several days (rtcwake sleep and wake up every 5 sec).
Quoted 4 lines
--
Best regards,
Dmitry
Dmitry Osipenko
maintainerOn 6/18/25 01:41, Kim, Dongwon wrote:
...
Quoted 9 lines
Quoted 1 line
Have you figured out why 10ms workaround is needed?
[Kim, Dongwon] Unfortunately, I don't know why it fails without the delay. I wanted to narrow down further so enabled printk during suspend and resume but hang didn't occur with the timing changes caused by printks. I've also tried more deterministic methods that make it wait based on some kinds of "status" but none of them have worked so far. If you have any suggestions on possible condition we can check instead of just sleeping, please let me know.
10ms seems to be close to minimum to make it work 100% for several days (rtcwake sleep and wake up every 5 sec).
Was able to reproduce the hang and got a crash backtrace with no_console_suspend:
[ 63.824827] PM: suspend entry (deep) [ 63.825041] Filesystems sync: 0.000 seconds [ 63.990951] Freezing user space processes [ 63.992488] Freezing user space processes completed (elapsed 0.001 seconds) [ 63.992775] OOM killer disabled. [ 63.992902] Freezing remaining freezable tasks [ 63.994099] Freezing remaining freezable tasks completed (elapsed 0.001 seconds) [ 64.002183] Oops: general protection fault, probably for non-canonical address 0x2abe0ea26847fb08: 0000 [#1] SMP NOPTI [ 64.003172] CPU: 9 UID: 0 PID: 178 Comm: kworker/9:2 Not tainted 6.15.4-00002-g01117b4373b2-dirty #123 PREEMPT(voluntary) [ 64.003614] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 64.004036] Workqueue: events virtio_gpu_dequeue_ctrl_func [ 64.004280] RIP: 0010:virtqueue_get_buf_ctx_split+0x86/0x130 [ 64.004515] Code: 01 66 23 43 50 0f b7 c0 8b 74 c1 04 8b 44 c1 08 41 89 45 00 3b 73 58 0f 83 96 d7 20 ff 89 f0 48 c1 e0 04 48 03 83 80 00 00 00 <4c> 8b 20 4d 85 e4 0f 84 5a d7 20 ff 48 89 df e8 46 fc ff ff 0f b7 [ 64.005227] RSP: 0018:ffffc90000b53d90 EFLAGS: 00010202 [ 64.005430] RAX: 2abe0ea26847fb08 RBX: ffff888102d58a00 RCX: ffff8881255314c0 [ 64.005698] RDX: 0000000000000000 RSI: 0000000000000008 RDI: ffff888102d58a00 [ 64.005975] RBP: ffffc90000b53db0 R08: 8080808080808080 R09: ffff88885b470b40 [ 64.006273] R10: ffff8881000508c8 R11: fefefefefefefeff R12: 0000000000000001 [ 64.006907] R13: ffffc90000b53dfc R14: ffffc90000b53dfc R15: ffff8881032d0568 [ 64.007205] FS: 0000000000000000(0000) GS:ffff8888d6650000(0000) knlGS:0000000000000000 [ 64.007511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 64.007732] CR2: 00007efedc4d3000 CR3: 00000001056e9000 CR4: 0000000000750ef0 [ 64.008014] PKRU: 55555554 [ 64.008123] Call Trace: [ 64.008223] <TASK> [ 64.008314] virtqueue_get_buf+0x46/0x60 [ 64.008465] virtio_gpu_dequeue_ctrl_func+0x86/0x2a0 [ 64.008655] process_one_work+0x18a/0x370 [ 64.008823] worker_thread+0x31a/0x460 [ 64.008971] ? _raw_spin_unlock_irqrestore+0x27/0x50 [ 64.009176] ? srso_alias_return_thunk+0x5/0xfbef5 [ 64.009369] ? __pfx_worker_thread+0x10/0x10 [ 64.009532] kthread+0x126/0x230 [ 64.009662] ? _raw_spin_unlock_irq+0x1f/0x40 [ 64.009836] ? __pfx_kthread+0x10/0x10 [ 64.009986] ret_from_fork+0x3a/0x60 [ 64.010156] ? __pfx_kthread+0x10/0x10 [ 64.010318] ret_from_fork_asm+0x1a/0x30 [ 64.010507] </TASK> [ 64.010616] Modules linked in: [ 64.010785] ---[ end trace 0000000000000000 ]---
==
The trace tells that virtio queue is active after it has been removed. This change fixes the crash, please test:
drivers/gpu/drm/virtio/virtgpu_drv.c +4 -0
@@ -187,6 +187,10 @@ static int virtgpu_freeze(struct virtio_device *vdev)flush_work (&vgdev->ctrlq.dequeue_work);flush_work (&vgdev->cursorq.dequeue_work);flush_work (&vgdev->config_changed_work); +wait_event (vgdev->ctrlq.ack_queue, + vgdev->ctrlq.vq->num_free == vgdev->ctrlq.vq->num_max); +wait_event (vgdev->cursorq.ack_queue, + vgdev->cursorq.vq->num_free == vgdev->cursorq.vq->num_max); vdev->config->del_vqs (vdev);return 0 ;
Signature · 2 lines
Best regards, Dmitry
Hi Dmitry,
This is great. I will test this and include the fix in the patch.
Quoted 107 lines
-----Original Message-----
From: Dmitry Osipenko <[email protected]>
Sent: Monday, June 30, 2025 7:00 PM
To: Kim, Dongwon <[email protected]>; dri- [email protected]
Cc: Kasireddy, Vivek <[email protected]>
Subject: Re: [RFC PATCH v2 0/2] Virtio-GPU suspend and resumeOn 6/18/25 01:41, Kim, Dongwon wrote:
...Quoted 7 lines
Quoted 1 line
Have you figured out why 10ms workaround is needed?
[Kim, Dongwon] Unfortunately, I don't know why it fails without the delay. I wanted to narrow down further so enabled printk during suspend and resume but hang didn't occur with the timing changes caused by printks. I've also tried more deterministic methods that make it wait based on some kinds of "status" but none of them have
worked so far. If you have any suggestions on possible condition we can check instead of just sleeping, please let me know.
Quoted 2 lines
10ms seems to be close to minimum to make it work 100% for several days (rtcwake sleep and wake up every 5 sec).
Was able to reproduce the hang and got a crash backtrace with
no_console_suspend:[ 63.824827] PM: suspend entry (deep) [ 63.825041] Filesystems sync: 0.000 seconds [ 63.990951] Freezing user space processes [ 63.992488] Freezing user space processes completed (elapsed 0.001 seconds) [ 63.992775] OOM killer disabled. [ 63.992902] Freezing remaining freezable tasks [ 63.994099] Freezing remaining freezable tasks completed (elapsed 0.001 seconds) [ 64.002183] Oops: general protection fault, probably for non-canonical address 0x2abe0ea26847fb08: 0000 [#1] SMP NOPTI [ 64.003172] CPU: 9 UID: 0 PID: 178 Comm: kworker/9:2 Not tainted 6.15.4- 00002-g01117b4373b2-dirty #123 PREEMPT(voluntary) [ 64.003614] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.16.3-0-ga6ed6b701f0a-prebuilt.qemu.org 04/01/2014 [ 64.004036] Workqueue: events virtio_gpu_dequeue_ctrl_func [ 64.004280] RIP: 0010:virtqueue_get_buf_ctx_split+0x86/0x130 [ 64.004515] Code: 01 66 23 43 50 0f b7 c0 8b 74 c1 04 8b 44 c1 08 41 89 45 00 3b 73 58 0f 83 96 d7 20 ff 89 f0 48 c1 e0 04 48 03 83 80 00 00 00 <4c> 8b 20 4d 85 e4 0f 84 5a d7 20 ff 48 89 df e8 46 fc ff ff 0f b7 [ 64.005227] RSP: 0018:ffffc90000b53d90 EFLAGS: 00010202 [ 64.005430] RAX: 2abe0ea26847fb08 RBX: ffff888102d58a00 RCX: ffff8881255314c0 [ 64.005698] RDX: 0000000000000000 RSI: 0000000000000008 RDI: ffff888102d58a00 [ 64.005975] RBP: ffffc90000b53db0 R08: 8080808080808080 R09: ffff88885b470b40 [ 64.006273] R10: ffff8881000508c8 R11: fefefefefefefeff R12: 0000000000000001 [ 64.006907] R13: ffffc90000b53dfc R14: ffffc90000b53dfc R15: ffff8881032d0568 [ 64.007205] FS: 0000000000000000(0000) GS:ffff8888d6650000(0000) knlGS:0000000000000000 [ 64.007511] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 64.007732] CR2: 00007efedc4d3000 CR3: 00000001056e9000 CR4: 0000000000750ef0 [ 64.008014] PKRU: 55555554 [ 64.008123] Call Trace: [ 64.008223] <TASK> [ 64.008314] virtqueue_get_buf+0x46/0x60 [ 64.008465] virtio_gpu_dequeue_ctrl_func+0x86/0x2a0 [ 64.008655] process_one_work+0x18a/0x370 [ 64.008823] worker_thread+0x31a/0x460 [ 64.008971] ? _raw_spin_unlock_irqrestore+0x27/0x50 [ 64.009176] ? srso_alias_return_thunk+0x5/0xfbef5 [ 64.009369] ? __pfx_worker_thread+0x10/0x10 [ 64.009532] kthread+0x126/0x230 [ 64.009662] ? _raw_spin_unlock_irq+0x1f/0x40 [ 64.009836] ? __pfx_kthread+0x10/0x10 [ 64.009986] ret_from_fork+0x3a/0x60 [ 64.010156] ? __pfx_kthread+0x10/0x10 [ 64.010318] ret_from_fork_asm+0x1a/0x30 [ 64.010507] </TASK> [ 64.010616] Modules linked in: [ 64.010785] ---[ end trace 0000000000000000 ]---==
The trace tells that virtio queue is active after it has been removed. This change fixes the crash, please test:
drivers/gpu/drm/virtio/virtgpu_drv.c +5
-1@@ -187,6 +187,10 @@ static int virtgpu_freeze(struct virtio_device *vdev)flush_work (&vgdev->ctrlq.dequeue_work);flush_work (&vgdev->cursorq.dequeue_work);flush_work (&vgdev->config_changed_work); +wait_event (vgdev->ctrlq.ack_queue, + vgdev->ctrlq.vq->num_free == vgdev->ctrlq.vq->num_max); +wait_event (vgdev->cursorq.ack_queue, + vgdev->cursorq.vq->num_free == + vgdev->cursorq.vq->num_max); vdev->config->del_vqs (vdev);return 0 ; --Best regards,
Dmitry