Skip to contentSkip to conversation

vnc linesize

PATCH In review Stefano Stabellini 3 participants 8 messages Mar 19, 2008 – Mar 20, 2008

lore
Share & tools
lore

Stefano Stabellini

author
#0 lore raw

PATCH 1/1 vnc linesize

Hi all,
the patch I am attaching enables buffer sharing for vnc when the linesize != width * depth.

Best Regards,

Stefano Stabellini

Signed-off-by Stefano Stabellini <[email protected]>
tools/ioemu/hw/vga.c +2 -2
diff -r d1e91aba2758 tools/ioemu/hw/vga.c
@@ -1148,7 +1148,7 @@ static void vga_draw_text(VGAState *s, i
         cw != s->last_cw || cheight != s->last_ch) {
         s->last_scr_width = width * cw;
         s->last_scr_height = height * cheight;
-        dpy_resize(s->ds, s->last_scr_width, s->last_scr_height, s->last_scr_width * (depth / 8));
+        dpy_resize(s->ds, s->last_scr_width, s->last_scr_height, s->last_scr_width * (s->ds->depth / 8));
         s->last_width = width;
         s->last_height = height;
         s->last_ch = cheight;

@@ -1571,7 +1571,7 @@ static void vga_draw_graphic(VGAState *s
     vga_draw_line = vga_draw_line_table[v * NB_DEPTHS + get_depth_index(s->ds)];
     if (disp_width != s->last_width ||
         height != s->last_height) {
-        dpy_resize(s->ds, disp_width, height, disp_width * (depth / 8));
+        dpy_resize(s->ds, disp_width, height, s->line_offset);
         s->last_scr_width = disp_width;
         s->last_scr_height = height;
         s->last_width = disp_width;
tools/ioemu/vnc.c +5 -7
diff -r d1e91aba2758 tools/ioemu/vnc.c
@@ -369,20 +369,17 @@ static void vnc_dpy_resize(DisplayState 
     VncState *vs = ds->opaque;
     int o;
 
-    if (linesize != w * vs->depth)
-        ds->shared_buf = 0;
-
     if (!ds->shared_buf) {
         if (allocated)
-            ds->data = realloc(ds->data, w * h * vs->depth);
+            ds->data = realloc(ds->data,  h * linesize);
         else
-            ds->data = malloc(w * h * vs->depth);
+            ds->data = malloc(h * linesize);
         allocated = 1;
     } else if (allocated) {
         free(ds->data);
         allocated = 0;
     }
-    vs->old_data = realloc(vs->old_data, w * h * vs->depth);
+    vs->old_data = realloc(vs->old_data, h * linesize);
     vs->dirty_row = realloc(vs->dirty_row, h * sizeof(vs->dirty_row[0]));
     vs->update_row = realloc(vs->update_row, h * sizeof(vs->dirty_row[0]));
 

@@ -399,7 +396,7 @@ static void vnc_dpy_resize(DisplayState 
     size_changed = ds->width != w || ds->height != h;
     ds->width = w;
     ds->height = h;
-    ds->linesize = w * vs->depth;
+    ds->linesize = linesize;
     if (vs->csock != -1 && vs->has_resize && size_changed) {
         vs->width = ds->width;
         vs->height = ds->height;

@@ -2494,6 +2491,7 @@ void vnc_display_init(DisplayState *ds)
 
     vs->ds->width = 640;
     vs->ds->height = 400;
+    vs->ds->linesize = 640 * 4;
     vnc_dpy_colourdepth(vs->ds, 24);
 }

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#1 lore raw
Quoted 3 lines · Stefano Stabellini

Hi all,
the patch I am attaching enables buffer sharing for vnc when the linesize != width * depth.

Hi Stefano,

I'm seeing some odd issues with this on ia64, see the attached
screenshot. I'm using VNC, not SDL, and have specified opengl=0 in my domain config file. The left side of the screen is wrapped around overtop of the right side of the screen. The copy on the left side is not updated unless I change the resolution the guest is running or force the VNC client to manually refresh. Updates are normal on the copy on the right side of the screen. Any ideas? Thanks,

Alex

Signature · 1 line
Alex Williamson                             HP Open Source & Linux Org.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#2 lore raw
Quoted 7 lines · Alex Williamson
Quoted 3 lines · Stefano Stabellini

Hi all,
the patch I am attaching enables buffer sharing for vnc when the linesize != width * depth.

Hi Stefano,

Quoted 7 lines

I'm seeing some odd issues with this on ia64, see the attached
screenshot. I'm using VNC, not SDL, and have specified opengl=0 in my domain config file. The left side of the screen is wrapped around overtop of the right side of the screen. The copy on the left side is not updated unless I change the resolution the guest is running or force the VNC client to manually refresh. Updates are normal on the copy on the right side of the screen. Any ideas? Thanks,

A quick self-followup, the same HVM domain configured for SDL instead
of VNC works fine. Thanks,

Alex

Signature · 1 line
Alex Williamson                             HP Open Source & Linux Org.
author
#4 lore raw
In reply to Alex Williamson

Hi Alex,
I think I have found the bug that causes the strange behaviour you are describing, but I have no way to be sure because I cannot test it on a ia64 machine. Could you please try the patch I am attaching (to be applied on top of the other one)?
Best Regards,

Stefano Stabellini

Quoted 6 lines · Alex Williamson
Quoted 2 lines · Stefano Stabellini

Hi all,
the patch I am attaching enables buffer sharing for vnc when the linesize != width * depth.

Hi Stefano,

Quoted 8 lines

I'm seeing some odd issues with this on ia64, see the attached
screenshot. I'm using VNC, not SDL, and have specified opengl=0 in my domain config file. The left side of the screen is wrapped around overtop of the right side of the screen. The copy on the left side is not updated unless I change the resolution the guest is running or force the VNC client to manually refresh. Updates are normal on the copy on the right side of the screen. Any ideas? Thanks,

tools/ioemu/vnc.c +11 -8
diff -r 491a3b62ae5b tools/ioemu/vnc.c
@@ -370,16 +370,20 @@ static void vnc_dpy_resize(DisplayState 
     int o;
 
     if (!ds->shared_buf) {
+        ds->linesize = w * vs->depth;
         if (allocated)
-            ds->data = realloc(ds->data,  h * linesize);
+            ds->data = realloc(ds->data,  h * ds->linesize);
         else
-            ds->data = malloc(h * linesize);
+            ds->data = malloc(h * ds->linesize);
         allocated = 1;
-    } else if (allocated) {
-        free(ds->data);
-        allocated = 0;
-    }
-    vs->old_data = realloc(vs->old_data, h * linesize);
+    } else {
+        ds->linesize = linesize;
+        if (allocated) {
+            free(ds->data);
+            allocated = 0;
+        }
+    }
+    vs->old_data = realloc(vs->old_data, h * ds->linesize);
     vs->dirty_row = realloc(vs->dirty_row, h * sizeof(vs->dirty_row[0]));
     vs->update_row = realloc(vs->update_row, h * sizeof(vs->dirty_row[0]));
 

@@ -396,7 +400,6 @@ static void vnc_dpy_resize(DisplayState 
     size_changed = ds->width != w || ds->height != h;
     ds->width = w;
     ds->height = h;
-    ds->linesize = linesize;
     if (vs->csock != -1 && vs->has_resize && size_changed) {
         vs->width = ds->width;
         vs->height = ds->height;

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#3 lore raw
In reply to Alex Williamson
Quoted 10 lines
Quoted 7 lines

I'm seeing some odd issues with this on ia64, see the attached
screenshot. I'm using VNC, not SDL, and have specified opengl=0 in my domain config file. The left side of the screen is wrapped around overtop of the right side of the screen. The copy on the left side is not updated unless I change the resolution the guest is running or force the VNC client to manually refresh. Updates are normal on the copy on the right side of the screen. Any ideas? Thanks,

A quick self-followup, the same HVM domain configured for SDL instead
of VNC works fine. Thanks,

It also breaks x86 WinXP guest installation. RealVNC client exits with "connection reset by peer" at the start of the graphical stage of installation (i.e., soon after the first reboot during installation). The guest then hangs, consuming no CPU, and dom0 also is idle.

This changeset is out on its ear unless a quick fix is forthcoming. :-)

-- Keir

#5 lore raw
In reply to Stefano Stabellini

Hi Stefano,

Quoted 6 lines · Stefano Stabellini

Hi Alex,
I think I have found the bug that causes the strange behaviour you are describing, but I have no way to be sure because I cannot test it on a ia64 machine.
Could you please try the patch I am attaching (to be applied on top of the other one)?

Yes, this fixes it. Thanks!

Alex

Signature · 1 line
Alex Williamson                             HP Open Source & Linux Org.
#7 lore raw
In reply to Stefano Stabellini
Quoted 7 lines · Stefano Stabellini

Hi Alex,
I think I have found the bug that causes the strange behaviour you are describing, but I have no way to be sure because I cannot test it on a ia64 machine.
Could you please try the patch I am attaching (to be applied on top of the other one)?
Best Regards,

Fixes my problem!

Thanks,
Keir

author
#6 lore raw
In reply to Alex Williamson
Quoted 8 lines · Alex Williamson

Hi Stefano,

Quoted 3 lines · Stefano Stabellini

Hi Alex,
I think I have found the bug that causes the strange behaviour you are describing, but I have no way to be sure because I cannot test it on a ia64 machine. Could you please try the patch I am attaching (to be applied on top of the other one)?

Yes, this fixes it. Thanks!

great!
Sending the patch again more "officially" this time.

Signed-off-by Stefano Stabellini <[email protected]>
tools/ioemu/vnc.c +11 -8
diff -r 491a3b62ae5b tools/ioemu/vnc.c
@@ -370,16 +370,20 @@ static void vnc_dpy_resize(DisplayState 
     int o;
 
     if (!ds->shared_buf) {
+        ds->linesize = w * vs->depth;
         if (allocated)
-            ds->data = realloc(ds->data,  h * linesize);
+            ds->data = realloc(ds->data,  h * ds->linesize);
         else
-            ds->data = malloc(h * linesize);
+            ds->data = malloc(h * ds->linesize);
         allocated = 1;
-    } else if (allocated) {
-        free(ds->data);
-        allocated = 0;
-    }
-    vs->old_data = realloc(vs->old_data, h * linesize);
+    } else {
+        ds->linesize = linesize;
+        if (allocated) {
+            free(ds->data);
+            allocated = 0;
+        }
+    }
+    vs->old_data = realloc(vs->old_data, h * ds->linesize);
     vs->dirty_row = realloc(vs->dirty_row, h * sizeof(vs->dirty_row[0]));
     vs->update_row = realloc(vs->update_row, h * sizeof(vs->dirty_row[0]));
 

@@ -396,7 +400,6 @@ static void vnc_dpy_resize(DisplayState 
     size_changed = ds->width != w || ds->height != h;
     ds->width = w;
     ds->height = h;
-    ds->linesize = linesize;
     if (vs->csock != -1 && vs->has_resize && size_changed) {
         vs->width = ds->width;
         vs->height = ds->height;

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel