Attachment 162075 Details for Bug 253597 – Fix endianness handling, and BGR pixel formats

[patch] Fix endianness handling, and BGR pixel formats

endian2.patch (text/plain), 4.36 KB, created by Daniel Berrangé on 2007-08-22 16:19:56 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Daniel Berrangé

Created: 2007-08-22 16:19:56 UTC

Size: 4.36 KB

patch

obsolete

>diff -r 453f9ca2a737 src/blt.h
>--- a/src/blt.h	Wed Aug 22 10:40:33 2007 -0400
>+++ b/src/blt.h	Wed Aug 22 12:16:19 2007 -0400
>@@ -40,9 +40,9 @@ static void FILL(struct gvnc *gvnc, src_
> 		int j;
> 
> 		for (j = 0; j < width; j++) {
>-			*dp = ((*sp >> f->red_shift) & gvnc->rm) << gvnc->rp
>-			    | ((*sp >> f->green_shift) & gvnc->gm) << gvnc->gp
>-			    | ((*sp >> f->blue_shift) & gvnc->bm) << gvnc->bp;
>+			*dp = ((*sp >> f->red_shift) & gvnc->rm) << gvnc->local.red_shift
>+			    | ((*sp >> f->green_shift) & gvnc->gm) << gvnc->local.green_shift
>+			    | ((*sp >> f->blue_shift) & gvnc->bm) << gvnc->local.blue_shift;
> 			dp++;
> 		}
> 		dst += gvnc->local.linesize;
>@@ -65,9 +65,9 @@ static void BLIT(struct gvnc *gvnc, uint
> 		int j;
> 
> 		for (j = 0; j < w; j++) {
>-			*dp = ((*sp >> f->red_shift) & gvnc->rm) << gvnc->rp
>-			    | ((*sp >> f->green_shift) & gvnc->gm) << gvnc->gp
>-			    | ((*sp >> f->blue_shift) & gvnc->bm) << gvnc->bp;
>+			*dp = ((*sp >> f->red_shift) & gvnc->rm) << gvnc->local.red_shift
>+			    | ((*sp >> f->green_shift) & gvnc->gm) << gvnc->local.green_shift
>+			    | ((*sp >> f->blue_shift) & gvnc->bm) << gvnc->local.blue_shift;
> 			dp++;
> 			sp++;
> 		}
>diff -r 453f9ca2a737 src/gvnc.c
>--- a/src/gvnc.c	Wed Aug 22 10:40:33 2007 -0400
>+++ b/src/gvnc.c	Wed Aug 22 12:18:11 2007 -0400
>@@ -26,6 +26,7 @@
> #include <string.h>
> #include <sys/types.h>
> #include <sys/stat.h>
>+#include <endian.h>
> 
> #include "coroutine.h"
> #include "d3des.h"
>@@ -102,7 +103,6 @@ struct gvnc
> 	gboolean perfect_match;
> 	struct gvnc_framebuffer local;
> 
>-	int rp, gp, bp;
> 	int rm, gm, bm;
> 
> 	gvnc_blt_func *blt;
>@@ -676,9 +676,6 @@ static void gvnc_read_pixel_format(struc
> 	fmt->big_endian_flag = gvnc_read_u8(gvnc);
> 	fmt->true_color_flag = gvnc_read_u8(gvnc);
> 
>-	GVNC_DEBUG("Pixel format BPP: %d,  Depth: %d, Endian: %d, True color: %d\n",
>-		   fmt->bits_per_pixel, fmt->depth, fmt->big_endian_flag, fmt->true_color_flag);
>-
> 	fmt->red_max         = gvnc_read_u16(gvnc);
> 	fmt->green_max       = gvnc_read_u16(gvnc);
> 	fmt->blue_max        = gvnc_read_u16(gvnc);
>@@ -688,6 +685,24 @@ static void gvnc_read_pixel_format(struc
> 	fmt->blue_shift      = gvnc_read_u8(gvnc);
> 
> 	gvnc_read(gvnc, pad, 3);
>+
>+	GVNC_DEBUG("Pixel format BPP: %d,  Depth: %d, Endian: %d, True color: %d\n"
>+		   "             Mask  red: %3d, green: %3d, blue: %3d\n"
>+		   "             Shift red: %3d, green: %3d, blue: %3d\n",
>+		   fmt->bits_per_pixel, fmt->depth, fmt->big_endian_flag, fmt->true_color_flag,
>+		   fmt->red_max, fmt->green_max, fmt->blue_max,
>+		   fmt->red_shift, fmt->green_shift, fmt->blue_shift);
>+
>+
>+	if (((__BYTE_ORDER == __BIG_ENDIAN) && !fmt->big_endian_flag) ||
>+	    ((__BYTE_ORDER == __LITTLE_ENDIAN) && fmt->big_endian_flag)) {
>+		fmt->red_shift = fmt->bits_per_pixel - fmt->red_shift - (fmt->bits_per_pixel - fmt->depth);
>+		fmt->green_shift = fmt->bits_per_pixel - fmt->green_shift - (fmt->bits_per_pixel - fmt->depth);
>+		fmt->blue_shift = fmt->bits_per_pixel - fmt->blue_shift - (fmt->bits_per_pixel - fmt->depth);
>+
>+		GVNC_DEBUG("Flipped shifts Shift red: %3d, green: %3d, blue: %3d\n",
>+			   fmt->red_shift, fmt->green_shift, fmt->blue_shift);
>+	}
> }
> 
> /* initialize function */
>@@ -1981,7 +1996,9 @@ gboolean gvnc_set_local(struct gvnc *gvn
> 	    fb->blue_mask == gvnc->fmt.blue_max &&
> 	    fb->red_shift == gvnc->fmt.red_shift &&
> 	    fb->green_shift == gvnc->fmt.green_shift &&
>-	    fb->blue_shift == gvnc->fmt.blue_shift)
>+	    fb->blue_shift == gvnc->fmt.blue_shift &&
>+	    ((gvnc->fmt.big_endian_flag && (__BYTE_ORDER == __BIG_ENDIAN)) ||
>+	     (!gvnc->fmt.big_endian_flag && (__BYTE_ORDER == __LITTLE_ENDIAN))))
> 		gvnc->perfect_match = TRUE;
> 	else
> 		gvnc->perfect_match = FALSE;
>@@ -1989,17 +2006,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
> 	depth = gvnc->fmt.depth;
> 	if (depth == 32)
> 		depth = 24;
>-
>-	gvnc->rp =  (gvnc->local.depth - gvnc->local.red_shift);
>-	gvnc->rp -= (depth - gvnc->fmt.red_shift);
>-	gvnc->gp =  (gvnc->local.red_shift - gvnc->local.green_shift);
>-	gvnc->gp -= (gvnc->fmt.red_shift - gvnc->fmt.green_shift);
>-	gvnc->bp =  (gvnc->local.green_shift - gvnc->local.blue_shift);
>-	gvnc->bp -= (gvnc->fmt.green_shift - gvnc->fmt.blue_shift);
>-
>-	gvnc->rp = gvnc->local.red_shift + gvnc->rp;
>-	gvnc->gp = gvnc->local.green_shift + gvnc->gp;
>-	gvnc->bp = gvnc->local.blue_shift + gvnc->bp;
> 
> 	gvnc->rm = gvnc->local.red_mask & gvnc->fmt.red_max;
> 	gvnc->gm = gvnc->local.green_mask & gvnc->fmt.green_max;

Actions: View | Diff

Attachments on bug 253597: 161913 | 162073 | 162075