Attachment 298529 Details for Bug 431811 – Fix endian conversions

[patch] Fix endian conversions

gtk-vnc-0.3.2-big-endian-fix.patch (text/plain), 8.04 KB, created by Daniel Berrangé on 2008-03-19 15:56:22 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Daniel Berrangé

Created: 2008-03-19 15:56:22 UTC

Size: 8.04 KB

patch

obsolete

>diff -rup gtk-vnc-0.3.2.orig/AUTHORS gtk-vnc-0.3.2.new/AUTHORS
>--- gtk-vnc-0.3.2.orig/AUTHORS	2007-12-13 10:17:33.000000000 -0500
>+++ gtk-vnc-0.3.2.new/AUTHORS	2008-03-12 16:03:48.000000000 -0400
>@@ -10,6 +10,7 @@ GTK VNC is written & maintained by:
> Patches received from:
> 
>    Halton Huo <Halton-dot-Huo-at-Sun-dotCOM>
>+   Hiroyuki Kaguchi <fj7025cf-at-aa-dot-jp-dot-fujitsu-dot-com>
> 
>     ...send patches to get your name here...
> 
>diff -rup gtk-vnc-0.3.2.orig/src/blt.h gtk-vnc-0.3.2.new/src/blt.h
>--- gtk-vnc-0.3.2.orig/src/blt.h	2007-12-30 23:48:48.000000000 -0500
>+++ gtk-vnc-0.3.2.new/src/blt.h	2008-03-19 11:51:41.000000000 -0400
>@@ -12,6 +12,8 @@
> #define HEXTILE SPLICE(gvnc_hextile_, SUFFIX())
> #define RRE SPLICE(gvnc_rre_, SUFFIX())
> #define RICH_CURSOR_BLIT SPLICE(gvnc_rich_cursor_blt_, SUFFIX())
>+#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, SRC)(gvnc, pixel)
>+#define COMPONENT(color, pixel) ((SWAP(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
> 
> static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
> 		      int x, int y, int width, int height)
>@@ -35,18 +37,18 @@ static void FAST_FILL(struct gvnc *gvnc,
> 	}
> }
> 
>-static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t *sp)
>+static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t sp)
> {
>-	*dp = ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
>-		| ((*sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
>-		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls;
>+	*dp = ((sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
>+		| ((sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
>+		| ((sp >> gvnc->brs) & gvnc->bm) << gvnc->bls;
> }
> 
> static void SET_PIXEL_AT(struct gvnc *gvnc, int x, int y, src_pixel_t *sp)
> {
> 	dst_pixel_t *dp = (dst_pixel_t *)gvnc_get_local(gvnc, x, y);
> 
>-	SET_PIXEL(gvnc, dp, sp);
>+	SET_PIXEL(gvnc, dp, SWAP(gvnc, *sp));
> }
> 
> static void FILL(struct gvnc *gvnc, src_pixel_t *sp,
>@@ -60,7 +62,7 @@ static void FILL(struct gvnc *gvnc, src_
> 		int j;
> 
> 		for (j = 0; j < width; j++) {
>-			SET_PIXEL(gvnc, dp, sp);
>+			SET_PIXEL(gvnc, dp, SWAP(gvnc, *sp));
> 			dp++;
> 		}
> 		dst += gvnc->local.linesize;
>@@ -82,7 +84,7 @@ static void BLIT(struct gvnc *gvnc, uint
> 		int j;
> 
> 		for (j = 0; j < w; j++) {
>-			SET_PIXEL(gvnc, dp, sp);
>+			SET_PIXEL(gvnc, dp, SWAP(gvnc, *sp));
> 			dp++;
> 			sp++;
> 		}
>@@ -165,22 +167,47 @@ static void RICH_CURSOR_BLIT(struct gvnc
> 	uint32_t *dst = (uint32_t *)pixbuf;
> 	uint8_t *src = image;
> 	uint8_t *alpha = mask;
>-	int rs, gs, bs;
>+	int as, rs, gs, bs, n;
> 
>-	rs = 24 - ((sizeof(src_pixel_t) * 8) - gvnc->fmt.red_shift);
>-	gs = 16 - (gvnc->fmt.red_shift - gvnc->fmt.green_shift);
>-	bs = 8 - (gvnc->fmt.green_shift - gvnc->fmt.blue_shift);
>+	/*
>+	 * GdkPixbuf is always 32-bit RGB, so we can't use the precomputed
>+	 * left / right shift data from gvnc->{r,g,b}{r,l}s. The latter
>+	 * is set for the local display depth, which may be different
>+	 * to GdkPixbuf's fixed 32-bit RGBA
>+	 *
>+	 * This function isn't called often, so just re-compute them now
>+	 */
>+
>+#if G_BYTE_ORDER == G_BIG_ENDIAN
>+	as = 0;
>+	rs = 8;
>+	gs = 16;
>+	bs = 24;
>+#else
>+	as = 24;
>+	rs = 16;
>+	gs = 8;
>+	bs = 0;
>+#endif
>+
>+	/* Then this adjusts for remote having less bpp than 32 */
>+	for (n = 255 ; n > gvnc->fmt.red_max ; n>>= 1)
>+		rs++;
>+	for (n = 255 ; n > gvnc->fmt.green_max ; n>>= 1)
>+		gs++;
>+	for (n = 255 ; n > gvnc->fmt.blue_max ; n>>= 1)
>+		bs++;
> 
> 	for (y1 = 0; y1 < height; y1++) {
> 		src_pixel_t *sp = (src_pixel_t *)src;
> 		uint8_t *mp = alpha;
> 		for (x1 = 0; x1 < width; x1++) {
>-			*dst = (((*sp >> gvnc->fmt.red_shift) & gvnc->fmt.red_max) << rs)
>-				| (((*sp >> gvnc->fmt.green_shift) & gvnc->fmt.green_max) << gs)
>-				| (((*sp >> gvnc->fmt.blue_shift) & gvnc->fmt.blue_max) << bs);
>+			*dst = (COMPONENT(red, *sp) << rs)
>+				| (COMPONENT(green, *sp) << gs)
>+				| (COMPONENT(blue, *sp) << bs);
> 
> 			if ((mp[x1 / 8] >> (7 - (x1 % 8))) & 1)
>-				*dst |= 0xFF000000;
>+				*dst |= (0xFF << as);
> 
> 			dst++;
> 			sp++;
>@@ -191,6 +218,7 @@ static void RICH_CURSOR_BLIT(struct gvnc
> }
> #endif
> 
>+#undef COMPONENT
> #undef HEXTILE
> #undef FILL
> #undef FAST_FILL
>diff -rup gtk-vnc-0.3.2.orig/src/gvnc.c gtk-vnc-0.3.2.new/src/gvnc.c
>--- gtk-vnc-0.3.2.orig/src/gvnc.c	2007-12-30 23:48:48.000000000 -0500
>+++ gtk-vnc-0.3.2.new/src/gvnc.c	2008-03-19 11:40:04.000000000 -0400
>@@ -909,6 +909,31 @@ static inline uint8_t *gvnc_get_local(st
> 		(x * gvnc->local.bpp);
> }
> 
>+static uint8_t gvnc_swap_8(struct gvnc *gvnc, uint8_t pixel)
>+{
>+	return pixel;
>+}
>+
>+static uint16_t gvnc_swap_16(struct gvnc *gvnc, uint16_t pixel)
>+{
>+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
>+		return  (((pixel >> 8) & 0xFF) << 0) |
>+			(((pixel >> 0) & 0xFF) << 8);
>+	else
>+		return pixel;
>+}
>+
>+static uint32_t gvnc_swap_32(struct gvnc *gvnc, uint32_t pixel)
>+{
>+	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
>+		return  (((pixel >> 24) & 0xFF) <<  0) |
>+			(((pixel >> 16) & 0xFF) <<  8) |
>+			(((pixel >>  8) & 0xFF) << 16) |
>+			(((pixel >>  0) & 0xFF) << 24);
>+	else
>+		return pixel;
>+}
>+
> #define SPLICE_I(a, b) a ## b
> #define SPLICE(a, b) SPLICE_I(a, b)
> 
>@@ -1122,9 +1147,8 @@ static void gvnc_read_cpixel(struct gvnc
> 
> 	if (bpp == 4 && gvnc->fmt.true_color_flag && gvnc->fmt.depth == 24) {
> 		bpp = 3;
>-#if __BYTE_ORDER == __BIG_ENDIAN
>-		pixel += 1;
>-#endif
>+		if (gvnc->fmt.byte_order == __BIG_ENDIAN)
>+			pixel += 1;
> 	}
> 
> 	gvnc_read(gvnc, pixel, bpp);
>@@ -2566,7 +2590,7 @@ gboolean gvnc_set_local(struct gvnc *gvn
> 	    fb->red_shift == gvnc->fmt.red_shift &&
> 	    fb->green_shift == gvnc->fmt.green_shift &&
> 	    fb->blue_shift == gvnc->fmt.blue_shift &&
>-	    __BYTE_ORDER == gvnc->fmt.byte_order)
>+	    fb->byte_order == gvnc->fmt.byte_order)
> 		gvnc->perfect_match = TRUE;
> 	else
> 		gvnc->perfect_match = FALSE;
>@@ -2594,18 +2618,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
> 	gvnc->gls = gvnc->local.green_shift;
> 	gvnc->bls = gvnc->local.blue_shift;
> 
>-
>-	/* This adjusts for server/client endianness mismatch */
>-	if (__BYTE_ORDER != gvnc->fmt.byte_order) {
>-		gvnc->rrs = gvnc->fmt.bits_per_pixel - gvnc->rrs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
>-		gvnc->grs = gvnc->fmt.bits_per_pixel - gvnc->grs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
>-		gvnc->brs = gvnc->fmt.bits_per_pixel - gvnc->brs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
>-
>-		GVNC_DEBUG("Flipped shifts red: %3d, green: %3d, blue: %3d\n",
>-			   gvnc->rrs, gvnc->grs, gvnc->brs);
>-	}
>-
>-
> 	/* This adjusts for remote having more bpp than local */
> 	for (n = gvnc->fmt.red_max; n > gvnc->local.red_mask ; n>>= 1)
> 		gvnc->rrs++;
>diff -rup gtk-vnc-0.3.2.orig/src/gvnc.h gtk-vnc-0.3.2.new/src/gvnc.h
>--- gtk-vnc-0.3.2.orig/src/gvnc.h	2007-12-30 23:48:48.000000000 -0500
>+++ gtk-vnc-0.3.2.new/src/gvnc.h	2008-03-12 16:03:48.000000000 -0400
>@@ -45,6 +45,7 @@ struct gvnc_framebuffer
> 
> 	int linesize;
> 
>+	uint16_t byte_order;
> 	int depth;
> 	int bpp;
> 
>diff -rup gtk-vnc-0.3.2.orig/src/vncdisplay.c gtk-vnc-0.3.2.new/src/vncdisplay.c
>--- gtk-vnc-0.3.2.orig/src/vncdisplay.c	2008-03-12 15:39:17.000000000 -0400
>+++ gtk-vnc-0.3.2.new/src/vncdisplay.c	2008-03-19 11:39:41.000000000 -0400
>@@ -549,9 +549,17 @@ static gboolean on_resize(void *opaque, 
> 		priv->gc = gdk_gc_new(GTK_WIDGET(obj)->window);
> 	}
> 
>-	visual = gdk_drawable_get_visual(GTK_WIDGET(obj)->window);
>+	visual = gdk_screen_get_system_visual(gdk_screen_get_default());
> 	
> 	priv->image = gdk_image_new(GDK_IMAGE_FASTEST, visual, width, height);
>+	GVNC_DEBUG("Visual mask: %3d %3d %3d\n      shift: %3d %3d %3d\n",
>+		   visual->red_mask,
>+		   visual->green_mask,
>+		   visual->blue_mask,
>+		   visual->red_shift,
>+		   visual->green_shift,
>+		   visual->blue_shift);
>+
> 	priv->fb.red_mask = visual->red_mask >> visual->red_shift;
> 	priv->fb.green_mask = visual->green_mask >> visual->green_shift;
> 	priv->fb.blue_mask = visual->blue_mask >> visual->blue_shift;
>@@ -564,6 +572,7 @@ static gboolean on_resize(void *opaque, 
> 	priv->fb.height = priv->image->height;
> 	priv->fb.linesize = priv->image->bpl;
> 	priv->fb.data = (uint8_t *)priv->image->mem;
>+	priv->fb.byte_order = priv->image->byte_order == GDK_LSB_FIRST ? __LITTLE_ENDIAN : __BIG_ENDIAN;
> 
> 	gtk_widget_set_size_request(GTK_WIDGET(obj), width, height);
>

Actions: View | Diff

Attachments on bug 431811: 294182 | 298529