shaders: support byte swapping of pixels

Some EGL implementations do not support GL_BGRA_EXT so we always use
a surface format of GL_RGBA and then swap the bytes with a fragment
shader.

include/ui/console.h

@@ -116,6 +116,7 @@ typedef struct DisplaySurface {
     GLenum glformat;
     GLenum gltype;
     GLuint texture;
+    bool   glswapped;
 #endif
 } DisplaySurface;
 

include/ui/egl-helpers.h

@@ -36,9 +36,9 @@ void egl_fb_setup_new_tex(egl_fb *fb, int width, int height);
 void egl_fb_blit(egl_fb *dst, egl_fb *src, bool flip);
 void egl_fb_read(DisplaySurface *dst, egl_fb *src);
 
-void egl_texture_blit(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip);
+void egl_texture_blit(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip, bool swap);
 void egl_texture_blend(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip,
-                       int x, int y, double scale_x, double scale_y);
+                       bool swap, int x, int y, double scale_x, double scale_y);
 
 #ifdef CONFIG_GBM
 

include/ui/shader.h

@@ -5,7 +5,7 @@
 
 typedef struct QemuGLShader QemuGLShader;
 
-void qemu_gl_run_texture_blit(QemuGLShader *gls, bool flip);
+void qemu_gl_run_texture_blit(QemuGLShader *gls, bool flip, bool swapped);
 
 QemuGLShader *qemu_gl_init_shader(void);
 void qemu_gl_fini_shader(QemuGLShader *gls);

ui/cocoa/main.m

@@ -574,7 +574,7 @@ static void cocoa_gl_render_cursor()
     glBindTexture(GL_TEXTURE_2D, cursor_texture);
     glEnable(GL_BLEND);
     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
-    qemu_gl_run_texture_blit(gls, false);
+    qemu_gl_run_texture_blit(gls, false, false);
     glDisable(GL_BLEND);
 }
 
@@ -671,7 +671,7 @@ static void cocoa_gl_scanout_flush(DisplayChangeListener *dcl,
         glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
         glViewport(0, 0, size.width, size.height);
         glBindTexture(GL_TEXTURE_2D, texture);
-        qemu_gl_run_texture_blit(gls, y0_top);
+        qemu_gl_run_texture_blit(gls, y0_top, false);
 
         cocoa_gl_render_cursor();
 

ui/console-gl.c

@@ -52,8 +52,9 @@ void surface_gl_create_texture(QemuGLShader *gls,
     switch (surface->format) {
     case PIXMAN_BE_b8g8r8x8:
     case PIXMAN_BE_b8g8r8a8:
-        surface->glformat = GL_BGRA_EXT;
+        surface->glformat = GL_RGBA;
         surface->gltype = GL_UNSIGNED_BYTE;
+        surface->glswapped = true;
         break;
     case PIXMAN_BE_x8r8g8b8:
     case PIXMAN_BE_a8r8g8b8:
@@ -121,7 +122,7 @@ void surface_gl_render_texture(QemuGLShader *gls,
     glClearColor(0.1f, 0.1f, 0.1f, 0.0f);
     glClear(GL_COLOR_BUFFER_BIT);
 
-    qemu_gl_run_texture_blit(gls, false);
+    qemu_gl_run_texture_blit(gls, false, false);
 }
 
 void surface_gl_destroy_texture(QemuGLShader *gls,

ui/egl-headless.c

@@ -165,9 +165,9 @@ static void egl_scanout_flush(DisplayChangeListener *dcl,
     if (edpy->cursor_fb.texture) {
         /* have cursor -> render using textures */
         egl_texture_blit(edpy->gls, &edpy->blit_fb, &edpy->guest_fb,
-                         !edpy->y_0_top);
+                         !edpy->y_0_top, false);
         egl_texture_blend(edpy->gls, &edpy->blit_fb, &edpy->cursor_fb,
-                          !edpy->y_0_top, edpy->pos_x, edpy->pos_y,
+                          false, !edpy->y_0_top, edpy->pos_x, edpy->pos_y,
                           1.0, 1.0);
     } else {
         /* no cursor -> use simple framebuffer blit */

ui/egl-helpers.c

@@ -122,17 +122,17 @@ void egl_fb_read(DisplaySurface *dst, egl_fb *src)
                  GL_BGRA, GL_UNSIGNED_BYTE, surface_data(dst));
 }
 
-void egl_texture_blit(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip)
+void egl_texture_blit(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip, bool swap)
 {
     glBindFramebuffer(GL_FRAMEBUFFER_EXT, dst->framebuffer);
     glViewport(0, 0, dst->width, dst->height);
     glEnable(GL_TEXTURE_2D);
     glBindTexture(src->texture_target, src->texture);
-    qemu_gl_run_texture_blit(gls, flip);
+    qemu_gl_run_texture_blit(gls, flip, swap);
 }
 
 void egl_texture_blend(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip,
-                       int x, int y, double scale_x, double scale_y)
+                       bool swap, int x, int y, double scale_x, double scale_y)
 {
     glBindFramebuffer(GL_FRAMEBUFFER_EXT, dst->framebuffer);
     int w = scale_x * src->width;
@@ -146,7 +146,7 @@ void egl_texture_blend(QemuGLShader *gls, egl_fb *dst, egl_fb *src, bool flip,
     glBindTexture(src->texture_target, src->texture);
     glEnable(GL_BLEND);
     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
-    qemu_gl_run_texture_blit(gls, flip);
+    qemu_gl_run_texture_blit(gls, flip, swap);
     glDisable(GL_BLEND);
 }
 

ui/gtk-egl.c

@@ -299,9 +299,9 @@ void gd_egl_scanout_flush(DisplayChangeListener *dcl,
     egl_fb_setup_default(&vc->gfx.win_fb, ww, wh);
     if (vc->gfx.cursor_fb.texture) {
         egl_texture_blit(vc->gfx.gls, &vc->gfx.win_fb, &vc->gfx.guest_fb,
-                         vc->gfx.y0_top);
+                         vc->gfx.y0_top, false);
         egl_texture_blend(vc->gfx.gls, &vc->gfx.win_fb, &vc->gfx.cursor_fb,
-                          vc->gfx.y0_top,
+                          vc->gfx.y0_top, false,
                           vc->gfx.cursor_x, vc->gfx.cursor_y,
                           vc->gfx.scale_x, vc->gfx.scale_y);
     } else {

ui/shader.c

@@ -30,10 +30,13 @@
 #include "ui/shader/texture-blit-vert.h"
 #include "ui/shader/texture-blit-flip-vert.h"
 #include "ui/shader/texture-blit-frag.h"
+#include "ui/shader/texture-blit-swap-frag.h"
 
 struct QemuGLShader {
     GLint texture_blit_prog;
     GLint texture_blit_flip_prog;
+    GLint texture_blit_swap_prog;
+    GLint texture_blit_flip_swap_prog;
     GLint texture_blit_vao;
 };
 
@@ -69,11 +72,17 @@ static GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog)
     return vao;
 }
 
-void qemu_gl_run_texture_blit(QemuGLShader *gls, bool flip)
+void qemu_gl_run_texture_blit(QemuGLShader *gls, bool flip, bool swapped)
 {
-    glUseProgram(flip
-                 ? gls->texture_blit_flip_prog
-                 : gls->texture_blit_prog);
+    if (flip && swapped) {
+        glUseProgram(gls->texture_blit_flip_swap_prog);
+    } else if (flip && !swapped) {
+        glUseProgram(gls->texture_blit_flip_prog);
+    } else if (!flip && swapped) {
+        glUseProgram(gls->texture_blit_swap_prog);
+    } else { // !flip && !swapped
+        glUseProgram(gls->texture_blit_prog);
+    }
     glBindVertexArray(gls->texture_blit_vao);
     glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
 }
@@ -163,7 +172,14 @@ QemuGLShader *qemu_gl_init_shader(void)
     strcpy(vert_src_body, texture_blit_flip_vert_src);
     gls->texture_blit_flip_prog = qemu_gl_create_compile_link_program
         (vert_src, frag_src);
-    if (!gls->texture_blit_prog || !gls->texture_blit_flip_prog) {
+    strcpy(frag_src_body, texture_blit_swap_frag_src);
+    gls->texture_blit_flip_swap_prog = qemu_gl_create_compile_link_program
+        (vert_src, frag_src);
+    strcpy(vert_src_body, texture_blit_vert_src);
+    gls->texture_blit_swap_prog = qemu_gl_create_compile_link_program
+        (vert_src, frag_src);
+    if (!gls->texture_blit_prog || !gls->texture_blit_flip_prog ||
+        !gls->texture_blit_swap_prog || !gls->texture_blit_flip_swap_prog) {
         exit(1);
     }
 

ui/shader/meson.build

@@ -1,5 +1,6 @@
 shaders = [
   ['texture-blit', 'frag'],
+  ['texture-blit-swap', 'frag'],
   ['texture-blit', 'vert'],
   ['texture-blit-flip', 'vert'],
 ]

ui/shader/texture-blit-swap.frag

@@ -0,0 +1,7 @@
+uniform sampler2D image;
+in  mediump vec2 ex_tex_coord;
+out mediump vec4 out_frag_color;
+
+void main(void) {
+     out_frag_color = texture(image, ex_tex_coord).zyxw;
+}

ui/spice-display.c

@@ -929,7 +929,7 @@ static int spice_iosurface_create_fd(SimpleSpiceDisplay *ssd, int *fourcc)
     return fds[0];
 }
 
-static void spice_iosurface_blit(SimpleSpiceDisplay *ssd, GLuint src_texture, bool flip)
+static void spice_iosurface_blit(SimpleSpiceDisplay *ssd, GLuint src_texture, bool flip, bool swap)
 {
     egl_fb tmp_fb = { .texture = src_texture, .texture_target = GL_TEXTURE_2D };
     if (!ssd->iosurface) {
@@ -940,7 +940,7 @@ static void spice_iosurface_blit(SimpleSpiceDisplay *ssd, GLuint src_texture, bo
     eglMakeCurrent(qemu_egl_display, ssd->esurface, ssd->esurface, spice_gl_ctx);
     glBindTexture(ssd->iosurface_fb.texture_target, ssd->iosurface_fb.texture);
     eglBindTexImage(qemu_egl_display, ssd->esurface, EGL_BACK_BUFFER);
-    egl_texture_blit(ssd->gls, &ssd->iosurface_fb, &tmp_fb, flip);
+    egl_texture_blit(ssd->gls, &ssd->iosurface_fb, &tmp_fb, flip, swap);
 #endif
 }
 
@@ -1041,7 +1041,7 @@ static void spice_gl_update(DisplayChangeListener *dcl,
     surface_gl_update_texture(ssd->gls, ssd->ds, x, y, w, h);
 #if defined(CONFIG_IOSURFACE)
     if (!qemu_console_is_gl_blocked(ssd->dcl.con)) {
-        spice_iosurface_blit(ssd, ssd->ds->texture, true);
+        spice_iosurface_blit(ssd, ssd->ds->texture, true, ssd->ds->glswapped);
     }
 #endif
     ssd->gl_updates++;
@@ -1315,15 +1315,15 @@ static void qemu_spice_gl_update(DisplayChangeListener *dcl,
         y = ssd->ptr_y;
         qemu_mutex_unlock(&ssd->lock);
         egl_texture_blit(ssd->gls, &ssd->blit_fb, &ssd->guest_fb,
-                         !y_0_top);
+                         !y_0_top, false);
         egl_texture_blend(ssd->gls, &ssd->blit_fb, &ssd->cursor_fb,
-                          !y_0_top, x, y, 1.0, 1.0);
+                          !y_0_top, false, x, y, 1.0, 1.0);
         glFlush();
     }
 #elif defined(CONFIG_ANGLE) && defined(CONFIG_IOSURFACE)
     GLuint tex_id = ssd->backing_borrow(ssd->backing_id, &y_0_top,
                                         NULL, NULL);
-    spice_iosurface_blit(ssd, tex_id, !y_0_top);
+    spice_iosurface_blit(ssd, tex_id, !y_0_top, false);
     spice_iosurface_flush(ssd);
     //TODO: cursor stuff
 #endif