Skip to contentSkip to conversation

vnc dynamic resolution

PATCH In review Stefano Stabellini 2 participants 4 messages Sep 8, 2008 – Sep 11, 2008

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 vnc dynamic resolution

This patch implements dynamic colour depth changes in vnc.c:
this way the vnc server can change its own internal colour depth at run time to follow any guest resolution change.

---

vnc.c +170 -47
index c0a05a8..239fd56 100644
@@ -71,8 +71,8 @@ typedef void VncWritePixels(VncState *vs, void *data, int size);
 
 typedef void VncSendHextileTile(VncState *vs,
                                 int x, int y, int w, int h,
-                                uint32_t *last_bg,
-                                uint32_t *last_fg,
+                                void *last_bg,
+                                void *last_fg,
                                 int *has_bg, int *has_fg);
 
 #define VNC_MAX_WIDTH 2048
@@ -166,9 +166,9 @@ struct VncState
     VncWritePixels *write_pixels;
     VncSendHextileTile *send_hextile_tile;
     int pix_bpp, pix_big_endian;
-    int red_shift, red_max, red_shift1;
-    int green_shift, green_max, green_shift1;
-    int blue_shift, blue_max, blue_shift1;
+    int client_red_shift, client_red_max, server_red_shift, server_red_max;
+    int client_green_shift, client_green_max, server_green_shift, server_green_max;
+    int client_blue_shift, client_blue_max, server_blue_shift, server_blue_max;
 
     VncReadEvent *read_handler;
     size_t read_handler_expect;
@@ -210,6 +210,8 @@ static void vnc_flush(VncState *vs);
 static void vnc_update_client(void *opaque);
 static void vnc_client_read(void *opaque);
 
+static void vnc_colordepth(DisplayState *ds, int depth);
+
 static inline void vnc_set_bit(uint32_t *d, int k)
 {
     d[k >> 5] |= 1 << (k & 0x1f);
@@ -332,14 +334,17 @@ static void vnc_write_pixels_copy(VncState *vs, void *pixels, int size)
 /* slowest but generic code. */
 static void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
 {
-    unsigned int r, g, b;
-
-    r = (v >> vs->red_shift1) & vs->red_max;
-    g = (v >> vs->green_shift1) & vs->green_max;
-    b = (v >> vs->blue_shift1) & vs->blue_max;
-    v = (r << vs->red_shift) |
-        (g << vs->green_shift) |
-        (b << vs->blue_shift);
+    uint8_t r, g, b;
+
+    r = ((v >> vs->server_red_shift) & vs->server_red_max) * (vs->client_red_max + 1) /
+        (vs->server_red_max + 1);
+    g = ((v >> vs->server_green_shift) & vs->server_green_max) * (vs->client_green_max + 1) /
+        (vs->server_green_max + 1);
+    b = ((v >> vs->server_blue_shift) & vs->server_blue_max) * (vs->client_blue_max + 1) /
+        (vs->server_blue_max + 1);
+    v = (r << vs->client_red_shift) |
+        (g << vs->client_green_shift) |
+        (b << vs->client_blue_shift);
     switch(vs->pix_bpp) {
     case 1:
         buf[0] = v;
@@ -372,14 +377,34 @@ static void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
 
 static void vnc_write_pixels_generic(VncState *vs, void *pixels1, int size)
 {
-    uint32_t *pixels = pixels1;
     uint8_t buf[4];
-    int n, i;
 
-    n = size >> 2;
-    for(i = 0; i < n; i++) {
-        vnc_convert_pixel(vs, buf, pixels[i]);
-        vnc_write(vs, buf, vs->pix_bpp);
+    if (vs->depth == 4) {
+        uint32_t *pixels = pixels1;
+        int n, i;
+        n = size >> 2;
+        for(i = 0; i < n; i++) {
+            vnc_convert_pixel(vs, buf, pixels[i]);
+            vnc_write(vs, buf, vs->pix_bpp);
+        }
+    } else if (vs->depth == 2) {
+        uint16_t *pixels = pixels1;
+        int n, i;
+        n = size >> 1;
+        for(i = 0; i < n; i++) {
+            vnc_convert_pixel(vs, buf, pixels[i]);
+            vnc_write(vs, buf, vs->pix_bpp);
+        }
+    } else if (vs->depth == 1) {
+        uint8_t *pixels = pixels1;
+        int n, i;
+        n = size;
+        for(i = 0; i < n; i++) {
+            vnc_convert_pixel(vs, buf, pixels[i]);
+            vnc_write(vs, buf, vs->pix_bpp);
+        }
+    } else {
+        fprintf(stderr, "vnc_write_pixels_generic: VncState color depth not supported\n");
     }
 }
 
@@ -416,6 +441,18 @@ static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
 #undef BPP
 
 #define GENERIC
+#define BPP 8
+#include "vnchextile.h"
+#undef BPP
+#undef GENERIC
+
+#define GENERIC
+#define BPP 16
+#include "vnchextile.h"
+#undef BPP
+#undef GENERIC
+
+#define GENERIC
 #define BPP 32
 #include "vnchextile.h"
 #undef BPP
@@ -425,18 +462,23 @@ static void send_framebuffer_update_hextile(VncState *vs, int x, int y, int w, i
 {
     int i, j;
     int has_fg, has_bg;
-    uint32_t last_fg32, last_bg32;
+    uint8_t *last_fg, *last_bg;
 
     vnc_framebuffer_update(vs, x, y, w, h, 5);
 
+    last_fg = (uint8_t *) malloc(vs->depth);
+    last_bg = (uint8_t *) malloc(vs->depth);
     has_fg = has_bg = 0;
     for (j = y; j < (y + h); j += 16) {
 	for (i = x; i < (x + w); i += 16) {
             vs->send_hextile_tile(vs, i, j,
                                   MIN(16, x + w - i), MIN(16, y + h - j),
-                                  &last_bg32, &last_fg32, &has_bg, &has_fg);
+                                  last_bg, last_fg, &has_bg, &has_fg);
 	}
     }
+    free(last_fg);
+    free(last_bg);
+
 }
 
 static void send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
@@ -1135,17 +1177,6 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
     check_pointer_type_change(vs, kbd_mouse_is_absolute());
 }
 
-static int compute_nbits(unsigned int val)
-{
-    int n;
-    n = 0;
-    while (val != 0) {
-        n++;
-        val >>= 1;
-    }
-    return n;
-}
-
 static void set_pixel_format(VncState *vs,
 			     int bits_per_pixel, int depth,
 			     int big_endian_flag, int true_color_flag,
@@ -1165,6 +1196,7 @@ static void set_pixel_format(VncState *vs,
         return;
     }
     if (bits_per_pixel == 32 &&
+        bits_per_pixel == vs->depth * 8 &&
         host_big_endian_flag == big_endian_flag &&
         red_max == 0xff && green_max == 0xff && blue_max == 0xff &&
         red_shift == 16 && green_shift == 8 && blue_shift == 0) {
@@ -1173,6 +1205,7 @@ static void set_pixel_format(VncState *vs,
         vs->send_hextile_tile = send_hextile_tile_32;
     } else
     if (bits_per_pixel == 16 &&
+        bits_per_pixel == vs->depth * 8 && 
         host_big_endian_flag == big_endian_flag &&
         red_max == 31 && green_max == 63 && blue_max == 31 &&
         red_shift == 11 && green_shift == 5 && blue_shift == 0) {
@@ -1181,6 +1214,7 @@ static void set_pixel_format(VncState *vs,
         vs->send_hextile_tile = send_hextile_tile_16;
     } else
     if (bits_per_pixel == 8 &&
+        bits_per_pixel == vs->depth * 8 &&
         red_max == 7 && green_max == 7 && blue_max == 3 &&
         red_shift == 5 && green_shift == 2 && blue_shift == 0) {
         vs->depth = 1;
@@ -1193,28 +1227,116 @@ static void set_pixel_format(VncState *vs,
             bits_per_pixel != 16 &&
             bits_per_pixel != 32)
             goto fail;
-        vs->depth = 4;
-        vs->red_shift = red_shift;
-        vs->red_max = red_max;
-        vs->red_shift1 = 24 - compute_nbits(red_max);
-        vs->green_shift = green_shift;
-        vs->green_max = green_max;
-        vs->green_shift1 = 16 - compute_nbits(green_max);
-        vs->blue_shift = blue_shift;
-        vs->blue_max = blue_max;
-        vs->blue_shift1 = 8 - compute_nbits(blue_max);
-        vs->pix_bpp = bits_per_pixel / 8;
+        if (vs->depth == 4) {
+            vs->send_hextile_tile = send_hextile_tile_generic_32;
+        } else if (vs->depth == 2) {
+           vs->send_hextile_tile = send_hextile_tile_generic_16;
+        } else {
+            vs->send_hextile_tile = send_hextile_tile_generic_8;
+        }
+
         vs->pix_big_endian = big_endian_flag;
         vs->write_pixels = vnc_write_pixels_generic;
-        vs->send_hextile_tile = send_hextile_tile_generic;
     }
 
-    vnc_dpy_resize(vs->ds, vs->ds->width, vs->ds->height);
+    vs->client_red_shift = red_shift;
+    vs->client_red_max = red_max;
+    vs->client_green_shift = green_shift;
+    vs->client_green_max = green_max;
+    vs->client_blue_shift = blue_shift;
+    vs->client_blue_max = blue_max;
+    vs->pix_bpp = bits_per_pixel / 8;
 
     vga_hw_invalidate();
     vga_hw_update();
 }
 
+static void vnc_colordepth(DisplayState *ds, int depth)
+{
+    int host_big_endian_flag;
+    struct VncState *vs = ds->opaque;
+
+    switch (depth) {
+        case 24:
+            if (ds->depth == 32) return;
+            depth = 32;
+            break;
+        case 15:
+        case 8:
+        case 0:
+            return;
+        default:
+            break;
+    }
+
+#ifdef WORDS_BIGENDIAN
+    host_big_endian_flag = 1;
+#else
+    host_big_endian_flag = 0;
+#endif   
+    
+    switch (depth) {
+        case 8:
+            vs->depth = depth / 8;
+            vs->server_red_max = 7;
+            vs->server_green_max = 7;
+            vs->server_blue_max = 3;
+            vs->server_red_shift = 5;
+            vs->server_green_shift = 2;
+            vs->server_blue_shift = 0;
+            break;
+        case 16:
+            vs->depth = depth / 8;
+            vs->server_red_max = 31;
+            vs->server_green_max = 63;
+            vs->server_blue_max = 31;
+            vs->server_red_shift = 11;
+            vs->server_green_shift = 5;
+            vs->server_blue_shift = 0;
+            break;
+        case 32:
+            vs->depth = 4;
+            vs->server_red_max = 255;
+            vs->server_green_max = 255;
+            vs->server_blue_max = 255;
+            vs->server_red_shift = 16;
+            vs->server_green_shift = 8;
+            vs->server_blue_shift = 0;
+            break;
+        default:
+            return;
+    }
+
+    if (vs->pix_bpp == 4 && vs->depth == 4 &&
+            host_big_endian_flag == vs->pix_big_endian &&
+            vs->client_red_max == 0xff && vs->client_green_max == 0xff && vs->client_blue_max == 0xff &&
+            vs->client_red_shift == 16 && vs->client_green_shift == 8 && vs->client_blue_shift == 0) {
+        vs->write_pixels = vnc_write_pixels_copy;
+        vs->send_hextile_tile = send_hextile_tile_32;
+    } else if (vs->pix_bpp == 2 && vs->depth == 2 &&
+            host_big_endian_flag == vs->pix_big_endian &&
+            vs->client_red_max == 31 && vs->client_green_max == 63 && vs->client_blue_max == 31 &&
+            vs->client_red_shift == 11 && vs->client_green_shift == 5 && vs->client_blue_shift == 0) {
+        vs->write_pixels = vnc_write_pixels_copy;
+        vs->send_hextile_tile = send_hextile_tile_16;
+    } else if (vs->pix_bpp == 1 && vs->depth == 1 &&
+            host_big_endian_flag == vs->pix_big_endian &&
+            vs->client_red_max == 7 && vs->client_green_max == 7 && vs->client_blue_max == 3 &&
+            vs->client_red_shift == 5 && vs->client_green_shift == 2 && vs->client_blue_shift == 0) {
+        vs->write_pixels = vnc_write_pixels_copy;
+        vs->send_hextile_tile = send_hextile_tile_8;
+    } else {
+        if (vs->depth == 4) {
+            vs->send_hextile_tile = send_hextile_tile_generic_32;
+        } else if (vs->depth == 2) {
+            vs->send_hextile_tile = send_hextile_tile_generic_16;
+        } else {
+            vs->send_hextile_tile = send_hextile_tile_generic_8;
+        }
+        vs->write_pixels = vnc_write_pixels_generic;
+    }
+}
+
 static int protocol_client_msg(VncState *vs, uint8_t *data, size_t len)
 {
     int i;
@@ -1318,7 +1440,9 @@ static int protocol_client_init(VncState *vs, uint8_t *data, size_t len)
     vnc_write_u16(vs, vs->ds->height);
 
     vnc_write_u8(vs, vs->depth * 8); /* bits-per-pixel */
-    vnc_write_u8(vs, vs->depth * 8); /* depth */
+    if (vs->depth == 4) vnc_write_u8(vs, 24); /* depth */
+    else vnc_write_u8(vs, vs->depth * 8); /* depth */
+
 #ifdef WORDS_BIGENDIAN
     vnc_write_u8(vs, 1);             /* big-endian-flag */
 #else
@@ -2008,7 +2132,6 @@ void vnc_display_init(DisplayState *ds)
 
     vs->lsock = -1;
     vs->csock = -1;
-    vs->depth = 4;
     vs->last_x = -1;
     vs->last_y = -1;
 
vnchextile.h +6 -6
index 09c1b27..eb05feb 100644
@@ -2,29 +2,29 @@
 #define CONCAT(a, b) CONCAT_I(a, b)
 #define pixel_t CONCAT(uint, CONCAT(BPP, _t))
 #ifdef GENERIC
-#define NAME generic
+#define NAME CONCAT(generic_, BPP)
 #else
 #define NAME BPP
 #endif
 
 static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
                                              int x, int y, int w, int h,
-                                             uint32_t *last_bg32,
-                                             uint32_t *last_fg32,
+                                             void *last_bg_,
+                                             void *last_fg_,
                                              int *has_bg, int *has_fg)
 {
     uint8_t *row = (vs->ds->data + y * vs->ds->linesize + x * vs->depth);
     pixel_t *irow = (pixel_t *)row;
     int j, i;
-    pixel_t *last_bg = (pixel_t *)last_bg32;
-    pixel_t *last_fg = (pixel_t *)last_fg32;
+    pixel_t *last_bg = (pixel_t *)last_bg_;
+    pixel_t *last_fg = (pixel_t *)last_fg_;
     pixel_t bg = 0;
     pixel_t fg = 0;
     int n_colors = 0;
     int bg_count = 0;
     int fg_count = 0;
     int flags = 0;
-    uint8_t data[(sizeof(pixel_t) + 2) * 16 * 16];
+    uint8_t data[(vs->pix_bpp + 2) * 16 * 16];
     int n_data = 0;
     int n_subtiles = 0;
 
#1 lore raw
Quoted 4 lines · Stefano Stabellini

This patch implements dynamic colour depth changes in vnc.c:
this way the vnc server can change its own internal colour depth at run time to follow any guest resolution change.

When testing this patch, gvncviewer hangs when attempting to connect. It's waiting to receive the ServerPixelFormat and it doesn't appear to receive it.

vnc.c static int protocol_client_init(VncState *vs, uint8_t *data, size_t len) · line 1445

Quoted 8 lines
vnc.c +2 -1
@@ -1318,7 +1440,9 @@ static int protocol_client_init(VncState *vs, uint8_t *data, size_t len)
     vnc_write_u16(vs, vs->ds->height);
 
     vnc_write_u8(vs, vs->depth * 8); /* bits-per-pixel */
-    vnc_write_u8(vs, vs->depth * 8); /* depth */
+    if (vs->depth == 4) vnc_write_u8(vs, 24); /* depth */
+    else vnc_write_u8(vs, vs->depth * 8); /* depth */
 

Why unconditionally change to a depth of 24 instead of 32? There's no savings from a protocol perspective.

Regards,

Anthony Liguori

Quoted 54 lines
vnc.c +0 -1
 #ifdef WORDS_BIGENDIAN
     vnc_write_u8(vs, 1);             /* big-endian-flag */
 #else
@@ -2008,7 +2132,6 @@ void vnc_display_init(DisplayState *ds)
 
     vs->lsock = -1;
     vs->csock = -1;
-    vs->depth = 4;
     vs->last_x = -1;
     vs->last_y = -1;
 
vnchextile.h +6 -6
index 09c1b27..eb05feb 100644
@@ -2,29 +2,29 @@
 #define CONCAT(a, b) CONCAT_I(a, b)
 #define pixel_t CONCAT(uint, CONCAT(BPP, _t))
 #ifdef GENERIC
-#define NAME generic
+#define NAME CONCAT(generic_, BPP)
 #else
 #define NAME BPP
 #endif
 
 static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
                                              int x, int y, int w, int h,
-                                             uint32_t *last_bg32,
-                                             uint32_t *last_fg32,
+                                             void *last_bg_,
+                                             void *last_fg_,
                                              int *has_bg, int *has_fg)
 {
     uint8_t *row = (vs->ds->data + y * vs->ds->linesize + x * vs->depth);
     pixel_t *irow = (pixel_t *)row;
     int j, i;
-    pixel_t *last_bg = (pixel_t *)last_bg32;
-    pixel_t *last_fg = (pixel_t *)last_fg32;
+    pixel_t *last_bg = (pixel_t *)last_bg_;
+    pixel_t *last_fg = (pixel_t *)last_fg_;
     pixel_t bg = 0;
     pixel_t fg = 0;
     int n_colors = 0;
     int bg_count = 0;
     int fg_count = 0;
     int flags = 0;
-    uint8_t data[(sizeof(pixel_t) + 2) * 16 * 16];
+    uint8_t data[(vs->pix_bpp + 2) * 16 * 16];
     int n_data = 0;
     int n_subtiles = 0;
 
#2 lore raw
Quoted 4 lines · Anthony Liguori

When testing this patch, gvncviewer hangs when attempting to connect. It's waiting to receive the ServerPixelFormat and it doesn't appear to receive it.

I admit I didn't test the patch individually but only the full series at once.
The reality is that the first patch doesn't make any sense without the fourth.
Actually now that I split the shared buffer patch into two patches: the shared buffer in vga.c and the vnc implementation, it would even make sense to merge the first patch and the fourth.
I'll repost the full series that way, if you agree:

1) shared buffer (vga only and interfaces)
2) shared buffer vnc implementation
3) WMVi implementation in vnc
4) shared buffer sdl implementation
5) opengl rendering implementation

I could even merge 2) and 3) if you prefer.

Quoted 4 lines
Quoted 4 lines
vnc.c +0 -0
@@ -1318,7 +1440,9 @@ static int protocol_client_init(VncState *vs,
 uint8_t *data, size_t len)
     vnc_write_u16(vs, vs->ds->height);
 
Quoted 9 lines

vnc.c static int protocol_client_init(VncState *vs, uint8_t *data, size_t len)

Quoted 5 lines
vnc.c +2 -1
     vnc_write_u8(vs, vs->depth * 8); /* bits-per-pixel */
-    vnc_write_u8(vs, vs->depth * 8); /* depth */
+    if (vs->depth == 4) vnc_write_u8(vs, 24); /* depth */
+    else vnc_write_u8(vs, vs->depth * 8); /* depth */
 

Why unconditionally change to a depth of 24 instead of 32? There's no savings from a protocol perspective.

I guess this change is not really needed, it is only to better specify that the format is RGB, 1 byte per channel plus 1 byte for the unused alpha channel.

#3 lore raw
In reply to Anthony Liguori
Quoted 3 lines · Anthony Liguori

When testing this patch, gvncviewer hangs when attempting to connect. It's waiting to receive the ServerPixelFormat and it doesn't appear to receive it.

The patch "doesn't work" as it is because of a little initialization
problem: in vnc_display_init you need to call

vnc_colordepth(vs->ds, 32);

before

vnc_dpy_resize(vs->ds, 640, 400);

The vnc display initialization is changed anyway with the fourth patch (see my previous post).

If you want me to resend the series with this issue fixed, do you want the first and the fourth patch split or merged?