[Qemu-devel] [PATCH] fix endianness problem sharing the videoram buffer

[Stefano Stabellini]

This patch fixes vga rendering when the guest endianness differs from
the host endianness: in this case we can only share the buffer if the
bpp is 32 and we must change the pixelformat accordingly.

Signed-off-by: Stefano Stabellini <address@hidden>

diff --git a/console.c b/console.c
index dbb3b70..4bcdac1 100644
--- a/console.c
+++ b/console.c
@@ -1449,7 +1449,7 @@ void qemu_console_copy(DisplayState *ds, int src_x, int 
src_y,
     }
 }
 
-static PixelFormat qemu_default_pixelformat(int bpp)
+PixelFormat qemu_different_endianness_pixelformat(int bpp)
 {
     PixelFormat pf;
 
@@ -1460,17 +1460,48 @@ static PixelFormat qemu_default_pixelformat(int bpp)
     pf.depth = bpp == 32 ? 24 : bpp;
 
     switch (bpp) {
-        case 8:
-            pf.rmask = 0x000000E0;
-            pf.gmask = 0x0000001C;
-            pf.bmask = 0x00000003;
-            pf.rmax = 7;
-            pf.gmax = 7;
-            pf.bmax = 3;
-            pf.rshift = 5;
-            pf.gshift = 2;
-            pf.bshift = 0;
+        case 24:
+            pf.rmask = 0x000000FF;
+            pf.gmask = 0x0000FF00;
+            pf.bmask = 0x00FF0000;
+            pf.rmax = 255;
+            pf.gmax = 255;
+            pf.bmax = 255;
+            pf.rshift = 0;
+            pf.gshift = 8;
+            pf.bshift = 16;
             break;
+        case 32:
+            pf.rmask = 0x0000FF00;
+            pf.gmask = 0x00FF0000;
+            pf.bmask = 0xFF000000;
+            pf.amask = 0x00000000;
+            pf.amax = 255;
+            pf.rmax = 255;
+            pf.gmax = 255;
+            pf.bmax = 255;
+            pf.ashift = 0;
+            pf.rshift = 8;
+            pf.gshift = 16;
+            pf.bshift = 24;
+            break;
+        default:
+            break;
+    }
+    return pf;
+}
+
+PixelFormat qemu_default_pixelformat(int bpp)
+{
+    PixelFormat pf;
+
+    memset(&pf, 0x00, sizeof(PixelFormat));
+
+    pf.bits_per_pixel = bpp;
+    pf.bytes_per_pixel = bpp / 8;
+    pf.depth = bpp == 32 ? 24 : bpp;
+
+    switch (bpp) {
         case 16:
             pf.rmask = 0x0000F800;
             pf.gmask = 0x000007E0;
@@ -1483,13 +1514,24 @@ static PixelFormat qemu_default_pixelformat(int bpp)
             pf.bshift = 0;
             break;
         case 24:
+            pf.rmask = 0x00FF0000;
+            pf.gmask = 0x0000FF00;
+            pf.bmask = 0x000000FF;
+            pf.rmax = 255;
+            pf.gmax = 255;
+            pf.bmax = 255;
+            pf.rshift = 16;
+            pf.gshift = 8;
+            pf.bshift = 0;
         case 32:
             pf.rmask = 0x00FF0000;
             pf.gmask = 0x0000FF00;
             pf.bmask = 0x000000FF;
+            pf.amax = 255;
             pf.rmax = 255;
             pf.gmax = 255;
             pf.bmax = 255;
+            pf.ashift = 24;
             pf.rshift = 16;
             pf.gshift = 8;
             pf.bshift = 0;
diff --git a/console.h b/console.h
index 383ea1a..6e764b3 100644
--- a/console.h
+++ b/console.h
@@ -134,6 +134,8 @@ DisplaySurface* qemu_resize_displaysurface(DisplaySurface 
*surface,
 DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
                                                 int linesize, uint8_t *data);
 void qemu_free_displaysurface(DisplaySurface *surface);
+PixelFormat qemu_different_endianness_pixelformat(int bpp);
+PixelFormat qemu_default_pixelformat(int bpp);
 
 static inline int is_buffer_shared(DisplaySurface *surface)
 {
diff --git a/hw/vga.c b/hw/vga.c
index 776ead0..c9fef86 100644
--- a/hw/vga.c
+++ b/hw/vga.c
@@ -1623,12 +1623,19 @@ static void vga_draw_graphic(VGAState *s, int 
full_update)
         disp_width != s->last_width ||
         height != s->last_height ||
         s->last_depth != depth) {
+#if defined(WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
         if (depth == 16 || depth == 32) {
+#else
+        if (depth == 32) {
+#endif
             if (is_graphic_console()) {
                 qemu_free_displaysurface(s->ds->surface);
                 s->ds->surface = qemu_create_displaysurface_from(disp_width, 
height, depth,
                                                                s->line_offset,
                                                                s->vram_ptr + 
(s->start_addr * 4));
+#if defined(WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
+                s->ds->surface->pf = 
qemu_different_endianness_pixelformat(depth);
+#endif
                 dpy_resize(s->ds);
             } else {
                 qemu_console_resize(s->ds, disp_width, height);