Skip to contentSkip to conversation

virtio-gpu: OUT_OF_MEMORY if failing to create udmabuf

PATCH Superseded Dongwon Kim 2 participants 3 messages Jun 21, 2023 – Jun 27, 2023

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 virtio-gpu: OUT_OF_MEMORY if failing to create udmabuf

Respond with VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY if it fails to create an udmabuf for the blob resource.

Cc Gerd Hoffmann <[email protected]>
Cc Marc-André Lureau <[email protected]>
Cc Vivek Kasireddy <[email protected]>
---
 hw/display/virtio-gpu.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
hw/display/virtio-gpu.c +3 -1
index 66cddd94d9..efe66ca7a3 100644
@@ -635,9 +635,11 @@ static void virtio_gpu_do_set_scanout(VirtIOGPU *g,
             if (!virtio_gpu_update_dmabuf(g, scanout_id, res, fb, r)) {
                 virtio_gpu_update_scanout(g, scanout_id, res, r);
                 return;
+            } else {
+                *error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+                return;
             }
         }
-
         data = res->blob;
     } else {
         data = (uint8_t *)pixman_image_get_data(res->image);
Signature · 1 line
2.34.1
Reviewed-by on v1 1/1
#1 lore raw

Hi

Quoted 16 lines · Dongwon Kim

Respond with VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY if it fails to create an udmabuf for the blob resource.

Cc Gerd Hoffmann <[email protected]>
Cc Marc-André Lureau <[email protected]>
Cc Vivek Kasireddy <[email protected]>
---
 hw/display/virtio-gpu.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
hw/display/virtio-gpu.c +0 -0
index 66cddd94d9..efe66ca7a3 100644

hw/display/virtio-gpu.c static void virtio_gpu_do_set_scanout(VirtIOGPU *g, · line 643

Quoted 10 lines
hw/display/virtio-gpu.c +3 -1
@@ -635,9 +635,11 @@ static void virtio_gpu_do_set_scanout(VirtIOGPU *g,
             if (!virtio_gpu_update_dmabuf(g, scanout_id, res, fb, r)) {
                 virtio_gpu_update_scanout(g, scanout_id, res, r);
                 return;
+            } else {
+                *error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+                return;
             }
         }
-
 

unrelated style change

Quoted 8 lines
hw/display/virtio-gpu.c +0 -0
         data = res->blob;
     } else {
         data = (uint8_t *)pixman_image_get_data(res->image);

--
2.34.1

Signature · 1 line
Marc-André Lureau
#2 lore raw

PATCH v2 1/1 virtio-gpu: OUT_OF_MEMORY if failing to create udmabuf

Respond with VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY if it fails to create an udmabuf for the blob resource.

v2: consolidated return statments and removed an unnecessary style change

Cc Gerd Hoffmann <[email protected]>
Cc Marc-André Lureau <[email protected]>
Cc Vivek Kasireddy <[email protected]>
---
 hw/display/virtio-gpu.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
hw/display/virtio-gpu.c +3 -1
index 66cddd94d9..ce019941b8 100644
@@ -634,8 +634,10 @@ static void virtio_gpu_do_set_scanout(VirtIOGPU *g,
         if (console_has_gl(scanout->con)) {
             if (!virtio_gpu_update_dmabuf(g, scanout_id, res, fb, r)) {
                 virtio_gpu_update_scanout(g, scanout_id, res, r);
-                return;
+            } else {
+                *error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
             }
+            return;
         }
 
         data = res->blob;
Signature · 1 line
2.34.1