Move Renderer from UTM to CocoaSpiceRenderer

Package.swift

@@ -13,9 +13,13 @@ let package = Package(
             targets: ["CocoaSpice"]),
     ],
     targets: [
+        .target(
+            name: "CocoaSpiceRenderer",
+            dependencies: []),
         .target(
             name: "CocoaSpice",
-            dependencies: [],
+            dependencies: ["CocoaSpiceRenderer"],
+            exclude: ["ExternalHeaders"],
             cSettings: [
                 .define("WITH_USB_SUPPORT"),
                 .headerSearchPath("ExternalHeaders"),
@@ -26,8 +30,9 @@ let package = Package(
                 .headerSearchPath("ExternalHeaders/spice-client-glib-2.0")]),
         .target(
             name: "CocoaSpiceNoUsb",
-            dependencies: [],
+            dependencies: ["CocoaSpiceRenderer"],
             exclude: [
+                "ExternalHeaders",
                 "CSUSBDevice.m",
                 "CSUSBManager.m"],
             cSettings: [

Sources/CocoaSpice/CSDisplayMetal.m

@@ -591,7 +591,7 @@ static void cs_channel_destroy(SpiceSession *s, SpiceChannel *channel, gpointer
 
 - (void)rebuildDisplayVertices {
     // We flip the y-coordinates because pixman renders flipped
-    CSDisplayVertex quadVertices[] =
+    CSRenderVertex quadVertices[] =
     {
         // Pixel positions, Texture coordinates
         { {  self.visibleArea.size.width/2,   self.visibleArea.size.height/2 },  { 1.f, 0.f } },
@@ -609,7 +609,7 @@ static void cs_channel_destroy(SpiceSession *s, SpiceChannel *channel, gpointer
                                                    options:MTLResourceStorageModeShared];
 
     // Calculate the number of vertices by dividing the byte length by the size of each vertex
-    self.displayNumVertices = sizeof(quadVertices) / sizeof(CSDisplayVertex);
+    self.displayNumVertices = sizeof(quadVertices) / sizeof(CSRenderVertex);
 }
 
 - (void)drawRegion:(CGRect)rect {
@@ -672,7 +672,7 @@ static void cs_channel_destroy(SpiceSession *s, SpiceChannel *channel, gpointer
     self.cursorTexture = [self.device newTextureWithDescriptor:textureDescriptor];
 
     // We flip the y-coordinates because pixman renders flipped
-    CSDisplayVertex quadVertices[] =
+    CSRenderVertex quadVertices[] =
     {
      // Pixel positions, Texture coordinates
      { { -hotspot.x + size.width, hotspot.y               },  { 1.f, 0.f } },
@@ -690,7 +690,7 @@ static void cs_channel_destroy(SpiceSession *s, SpiceChannel *channel, gpointer
                                                   options:MTLResourceStorageModeShared];
 
     // Calculate the number of vertices by dividing the byte length by the size of each vertex
-    self.cursorNumVertices = sizeof(quadVertices) / sizeof(CSDisplayVertex);
+    self.cursorNumVertices = sizeof(quadVertices) / sizeof(CSRenderVertex);
     self.cursorSize = size;
     self.cursorHotspot = hotspot;
     self.hasCursor = YES;

Sources/CocoaSpice/include/CSDisplayMetal.h

@@ -15,8 +15,8 @@
 //
 
 #import <Foundation/Foundation.h>
-#import "CSRenderSource.h"
 @import CoreGraphics;
+@import CocoaSpiceRenderer;
 
 @class CSScreenshot;
 

Sources/CocoaSpice/include/CocoaSpice.h

@@ -23,11 +23,9 @@
 #include "CSInput.h"
 #include "CSMain.h"
 #include "CSPasteboardDelegate.h"
-#include "CSRenderSource.h"
 #include "CSScreenshot.h"
 #include "CSSession.h"
 #include "CSSession+Sharing.h"
-#include "CSShaderTypes.h"
 #include "CSUSBDevice.h"
 #include "CSUSBManager.h"
 #include "CSUSBManagerDelegate.h"

Sources/CocoaSpiceRenderer/CSRenderer.m

@@ -0,0 +1,274 @@
+//
+// Copyright © 2022 osy. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+@import simd;
+@import MetalKit;
+
+#import "CSRenderer.h"
+#import "CSRenderSource.h"
+
+// Header shared between C code here, which executes Metal API commands, and .metal files, which
+//   uses these types as inputs to the shaders
+#import "CSShaderTypes.h"
+
+// Main class performing the rendering
+@implementation CSRenderer
+{
+    // The device (aka GPU) we're using to render
+    id<MTLDevice> _device;
+
+    // Our render pipeline composed of our vertex and fragment shaders in the .metal shader file
+    id<MTLRenderPipelineState> _pipelineState;
+
+    // The command Queue from which we'll obtain command buffers
+    id<MTLCommandQueue> _commandQueue;
+
+    // The current size of our view so we can use this in our render pipeline
+    vector_uint2 _viewportSize;
+    
+    // Sampler object
+    id<MTLSamplerState> _sampler;
+}
+
+- (void)setSource:(id<CSRenderSource>)source {
+    source.device = _device;
+    _source = source;
+}
+
+/// Initialize with the MetalKit view from which we'll obtain our Metal device
+- (nonnull instancetype)initWithMetalKitView:(nonnull MTKView *)mtkView
+{
+    self = [super init];
+    if(self)
+    {
+        _device = mtkView.device;
+
+        /// Create our render pipeline
+
+        // Load all the shader files with a .metal file extension in the project
+        id<MTLLibrary> defaultLibrary = [_device newDefaultLibrary];
+
+        // Load the vertex function from the library
+        id<MTLFunction> vertexFunction = [defaultLibrary newFunctionWithName:@"vertexShader"];
+
+        // Load the fragment function from the library
+        id<MTLFunction> fragmentFunction = [defaultLibrary newFunctionWithName:@"samplingShader"];
+
+        // Set up a descriptor for creating a pipeline state object
+        MTLRenderPipelineDescriptor *pipelineStateDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
+        pipelineStateDescriptor.label = @"Texturing Pipeline";
+        pipelineStateDescriptor.vertexFunction = vertexFunction;
+        pipelineStateDescriptor.fragmentFunction = fragmentFunction;
+        pipelineStateDescriptor.colorAttachments[0].pixelFormat = mtkView.colorPixelFormat;
+        pipelineStateDescriptor.colorAttachments[0].blendingEnabled = YES;
+        pipelineStateDescriptor.colorAttachments[0].rgbBlendOperation = MTLBlendOperationAdd;
+        pipelineStateDescriptor.colorAttachments[0].alphaBlendOperation = MTLBlendOperationAdd;
+        pipelineStateDescriptor.colorAttachments[0].sourceRGBBlendFactor = MTLBlendFactorSourceAlpha;
+        pipelineStateDescriptor.colorAttachments[0].destinationRGBBlendFactor = MTLBlendFactorOneMinusSourceAlpha;
+        pipelineStateDescriptor.colorAttachments[0].sourceAlphaBlendFactor = MTLBlendFactorOne;
+        pipelineStateDescriptor.colorAttachments[0].destinationAlphaBlendFactor = MTLBlendFactorOne;
+        pipelineStateDescriptor.depthAttachmentPixelFormat = mtkView.depthStencilPixelFormat;
+
+        NSError *error = NULL;
+        _pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineStateDescriptor
+                                                                 error:&error];
+        if (!_pipelineState)
+        {
+            // Pipeline State creation could fail if we haven't properly set up our pipeline descriptor.
+            //  If the Metal API validation is enabled, we can find out more information about what
+            //  went wrong.  (Metal API validation is enabled by default when a debug build is run
+            //  from Xcode)
+            NSLog(@"Failed to created pipeline state, error %@", error);
+        }
+
+        // Create the command queue
+        _commandQueue = [_device newCommandQueue];
+        
+        // Sampler
+        [self changeUpscaler:MTLSamplerMinMagFilterLinear downscaler:MTLSamplerMinMagFilterLinear];
+    }
+
+    return self;
+}
+
+/// Scalers from VM settings
+- (void)changeUpscaler:(MTLSamplerMinMagFilter)upscaler downscaler:(MTLSamplerMinMagFilter)downscaler {
+    MTLSamplerDescriptor *samplerDescriptor = [MTLSamplerDescriptor new];
+    samplerDescriptor.minFilter = downscaler;
+    samplerDescriptor.magFilter = upscaler;
+     
+    _sampler = [_device newSamplerStateWithDescriptor:samplerDescriptor];
+}
+
+/// Called whenever view changes orientation or is resized
+- (void)mtkView:(nonnull MTKView *)view drawableSizeWillChange:(CGSize)size
+{
+    // Save the size of the drawable as we'll pass these
+    //   values to our vertex shader when we draw
+    _viewportSize.x = size.width;
+    _viewportSize.y = size.height;
+}
+
+/// Create a translation+scale matrix
+static matrix_float4x4 matrix_scale_translate(CGFloat scale, CGPoint translate)
+{
+    matrix_float4x4 m = {
+        .columns[0] = {
+            scale,
+            0,
+            0,
+            0
+        },
+        .columns[1] = {
+            0,
+            scale,
+            0,
+            0
+        },
+        .columns[2] = {
+            0,
+            0,
+            1,
+            0
+        },
+        .columns[3] = {
+            translate.x,
+            -translate.y, // y flipped
+            0,
+            1
+        }
+        
+    };
+    return m;
+}
+
+/// Called whenever the view needs to render a frame
+- (void)drawInMTKView:(nonnull MTKView *)view
+{
+    id<CSRenderSource> source = self.source;
+    if (view.hidden || !source) {
+        return;
+    }
+
+    // Create a new command buffer for each render pass to the current drawable
+    id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
+    commandBuffer.label = @"MyCommand";
+
+    // Obtain a renderPassDescriptor generated from the view's drawable textures
+    MTLRenderPassDescriptor *renderPassDescriptor = view.currentRenderPassDescriptor;
+
+    if(renderPassDescriptor != nil)
+    {
+        // Create a render command encoder so we can render into something
+        id<MTLRenderCommandEncoder> renderEncoder =
+        [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
+        renderEncoder.label = @"MyRenderEncoder";
+        
+        // Render the screen first
+        
+        bool hasAlpha = NO;
+        bool isInverted = NO;
+        matrix_float4x4 transform = matrix_scale_translate(source.viewportScale,
+                                                           source.viewportOrigin);
+
+        [renderEncoder setRenderPipelineState:_pipelineState];
+
+        [renderEncoder setVertexBuffer:source.displayVertices
+                                offset:0
+                              atIndex:CSRenderVertexInputIndexVertices];
+
+        [renderEncoder setVertexBytes:&_viewportSize
+                               length:sizeof(_viewportSize)
+                              atIndex:CSRenderVertexInputIndexViewportSize];
+
+        [renderEncoder setVertexBytes:&transform
+                               length:sizeof(transform)
+                              atIndex:CSRenderVertexInputIndexTransform];
+
+        [renderEncoder setVertexBytes:&hasAlpha
+                               length:sizeof(hasAlpha)
+                              atIndex:CSRenderVertexInputIndexHasAlpha];
+
+        // Set the texture object.  The CSRenderTextureIndexBaseColor enum value corresponds
+        ///  to the 'colorMap' argument in our 'samplingShader' function because its
+        //   texture attribute qualifier also uses CSRenderTextureIndexBaseColor for its index
+        [renderEncoder setFragmentTexture:source.displayTexture
+                                  atIndex:CSRenderTextureIndexBaseColor];
+        
+        [renderEncoder setFragmentSamplerState:_sampler
+                                       atIndex:CSRenderSamplerIndexTexture];
+        
+        [renderEncoder setFragmentBytes:&isInverted
+                                 length:sizeof(isInverted)
+                                atIndex:CSRenderFragmentBufferIndexIsInverted];
+
+        // Draw the vertices of our triangles
+        [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangle
+                          vertexStart:0
+                          vertexCount:source.displayNumVertices];
+        
+        // Draw cursor
+        if (source.cursorVisible) {
+            // Next render the cursor
+            bool hasAlpha = YES;
+            bool isInverted = source.cursorInverted;
+            matrix_float4x4 transform = matrix_scale_translate(source.viewportScale,
+                                                               CGPointMake(source.viewportOrigin.x +
+                                                                           source.cursorOrigin.x,
+                                                                           source.viewportOrigin.y +
+                                                                           source.cursorOrigin.y));
+            [renderEncoder setVertexBuffer:source.cursorVertices
+                                    offset:0
+                                  atIndex:CSRenderVertexInputIndexVertices];
+            [renderEncoder setVertexBytes:&_viewportSize
+                                   length:sizeof(_viewportSize)
+                                  atIndex:CSRenderVertexInputIndexViewportSize];
+            [renderEncoder setVertexBytes:&transform
+                                 length:sizeof(transform)
+                                atIndex:CSRenderVertexInputIndexTransform];
+            [renderEncoder setVertexBytes:&hasAlpha
+                                 length:sizeof(hasAlpha)
+                                atIndex:CSRenderVertexInputIndexHasAlpha];
+            [renderEncoder setFragmentTexture:source.cursorTexture
+                                      atIndex:CSRenderTextureIndexBaseColor];
+            [renderEncoder setFragmentSamplerState:_sampler
+                                           atIndex:CSRenderSamplerIndexTexture];
+            [renderEncoder setFragmentBytes:&isInverted
+                                     length:sizeof(isInverted)
+                                    atIndex:CSRenderFragmentBufferIndexIsInverted];
+            [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangle
+                              vertexStart:0
+                              vertexCount:source.cursorNumVertices];
+        }
+
+        [renderEncoder endEncoding];
+        
+        // Schedule a present once the framebuffer is complete using the current drawable
+        [commandBuffer presentDrawable:view.currentDrawable];
+        
+        // Release lock after GPU is done
+        [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> commandBuffer) {
+            // GPU work is complete
+            [source rendererFrameHasRendered];
+        }];
+    }
+
+
+    // Finalize rendering here & push the command buffer to the GPU
+    [commandBuffer commit];
+}
+
+@end

Sources/CocoaSpiceRenderer/CSShaders.metal

@@ -0,0 +1,111 @@
+//
+// Copyright © 2022 osy. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <metal_stdlib>
+#include <simd/simd.h>
+
+using namespace metal;
+
+// Include header shared between this Metal shader code and C code executing Metal API commands
+#import "include/CSShaderTypes.h"
+
+// Vertex shader outputs and per-fragment inputs. Includes clip-space position and vertex outputs
+//  interpolated by rasterizer and fed to each fragment generated by clip-space primitives.
+typedef struct
+{
+    // The [[position]] attribute qualifier of this member indicates this value is the clip space
+    //   position of the vertex wen this structure is returned from the vertex shader
+    float4 clipSpacePosition [[position]];
+
+    // Since this member does not have a special attribute qualifier, the rasterizer will
+    //   interpolate its value with values of other vertices making up the triangle and
+    //   pass that interpolated value to the fragment shader for each fragment in that triangle;
+    float2 textureCoordinate;
+
+    // If no, then we fake an alpha value
+    bool hasAlpha;
+} RasterizerData;
+
+// Vertex Function
+vertex RasterizerData
+vertexShader(uint vertexID [[ vertex_id ]],
+             constant CSRenderVertex *vertexArray [[ buffer(CSRenderVertexInputIndexVertices) ]],
+             constant vector_uint2 *viewportSizePointer  [[ buffer(CSRenderVertexInputIndexViewportSize) ]],
+             constant matrix_float4x4 &transformation [[ buffer(CSRenderVertexInputIndexTransform) ]],
+             constant bool *hasAlpha [[ buffer(CSRenderVertexInputIndexHasAlpha) ]])
+
+{
+
+    RasterizerData out;
+    
+    // Transform the vertex
+    vector_float4 position = transformation * float4(vertexArray[vertexID].position,0,1);
+
+    // Index into our array of positions to get the current vertex
+    //   Our positions are specified in pixel dimensions (i.e. a value of 100 is 100 pixels from
+    //   the origin)
+    float2 pixelSpacePosition = position.xy;
+
+    // Get the size of the drawable so that we can convert to normalized device coordinates,
+    float2 viewportSize = float2(*viewportSizePointer);
+
+    // The output position of every vertex shader is in clip space (also known as normalized device
+    //   coordinate space, or NDC). A value of (-1.0, -1.0) in clip-space represents the
+    //   lower-left corner of the viewport whereas (1.0, 1.0) represents the upper-right corner of
+    //   the viewport.
+
+    // In order to convert from positions in pixel space to positions in clip space we divide the
+    //   pixel coordinates by half the size of the viewport.
+    out.clipSpacePosition.xy = pixelSpacePosition / (viewportSize / 2.0);
+
+    // Set the z component of our clip space position 0 (since we're only rendering in
+    //   2-Dimensions for this sample)
+    out.clipSpacePosition.z = 0.0;
+
+    // Set the w component to 1.0 since we don't need a perspective divide, which is also not
+    //   necessary when rendering in 2-Dimensions
+    out.clipSpacePosition.w = 1.0;
+
+    // Pass our input textureCoordinate straight to our output RasterizerData. This value will be
+    //   interpolated with the other textureCoordinate values in the vertices that make up the
+    //   triangle.
+    out.textureCoordinate = vertexArray[vertexID].textureCoordinate;
+    
+    // All fragments have alpha or not
+    out.hasAlpha = *hasAlpha;
+
+    return out;
+}
+
+// Fragment function
+fragment float4
+samplingShader(RasterizerData in [[stage_in]],
+               texture2d<half> colorTexture [[ texture(CSRenderTextureIndexBaseColor) ]],
+               sampler textureSampler [[ sampler(CSRenderSamplerIndexTexture) ]],
+               constant bool *isInverted [[ buffer(CSRenderFragmentBufferIndexIsInverted) ]])
+{
+    // Sample the texture to obtain a color
+    half4 colorSample = colorTexture.sample(textureSampler, in.textureCoordinate);
+
+    // fake alpha
+    if (!in.hasAlpha) {
+        colorSample.a = 0xff;
+    }
+
+    // We return the color of the texture inverted when requested
+    return float4(*isInverted ? colorSample.bgra : colorSample);
+}
+

Sources/CocoaSpiceRenderer/include/CSRenderer.h

@@ -0,0 +1,41 @@
+//
+// Copyright © 2022 osy. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+@import MetalKit;
+@import CoreGraphics;
+
+@protocol CSRenderSource;
+
+NS_ASSUME_NONNULL_BEGIN
+
+/// Simple platform independent renderer for CocoaSpice
+@interface CSRenderer : NSObject<MTKViewDelegate>
+
+/// Render source (comes from `CSDisplayMetal`)
+@property (nonatomic, weak, nullable) id<CSRenderSource> source;
+
+/// Create a new renderer for a MTKView
+/// @param mtkView The MetalKit View
+- (nonnull instancetype)initWithMetalKitView:(nonnull MTKView *)mtkView;
+
+/// Modify upscaler and downscaler settings
+/// @param upscaler Upscaler to use
+/// @param downscaler Downscaler to use
+- (void)changeUpscaler:(MTLSamplerMinMagFilter)upscaler downscaler:(MTLSamplerMinMagFilter)downscaler;
+
+@end
+
+NS_ASSUME_NONNULL_END

Sources/CocoaSpice/include/CSShaderTypes.h → Sources/CocoaSpiceRenderer/include/CSShaderTypes.h

@@ -19,6 +19,33 @@
 
 #include <simd/simd.h>
 
+// Buffer index values shared between shader and C code to ensure Metal shader buffer inputs match
+//   Metal API buffer set calls
+typedef enum CSRenderVertexInputIndex
+{
+    CSRenderVertexInputIndexVertices     = 0,
+    CSRenderVertexInputIndexViewportSize = 1,
+    CSRenderVertexInputIndexTransform    = 2,
+    CSRenderVertexInputIndexHasAlpha     = 3,
+} CSRenderVertexInputIndex;
+
+// Texture index values shared between shader and C code to ensure Metal shader buffer inputs match
+//   Metal API texture set calls
+typedef enum CSRenderTextureIndex
+{
+    CSRenderTextureIndexBaseColor = 0,
+} CSRenderTextureIndex;
+
+typedef enum CSRenderSamplerIndex
+{
+    CSRenderSamplerIndexTexture = 0,
+} CSRenderSamplerIndex;
+
+typedef enum CSRenderFragmentBufferIndex
+{
+    CSRenderFragmentBufferIndexIsInverted = 0,
+} CSRenderFragmentBufferIndex;
+
 //  This structure defines the layout of each vertex in the array of vertices set as an input to our
 //    Metal vertex shader.  Since this header is shared between our .metal shader and C code,
 //    we can be sure that the layout of the vertex array in the code matches the layout that
@@ -30,6 +57,6 @@ typedef struct
 
     // 2D texture coordinate
     vector_float2 textureCoordinate;
-} CSDisplayVertex;
+} CSRenderVertex;
 
 #endif /* CSShaderTypes_h */

Sources/CocoaSpiceRenderer/include/CocoaSpiceRenderer.h

@@ -0,0 +1,24 @@
+//
+// Copyright © 2022 osy. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef CocoaSpiceRenderer_h
+#define CocoaSpiceRenderer_h
+
+#include "CSRenderer.h"
+#include "CSRenderSource.h"
+#include "CSShaderTypes.h"
+
+#endif /* CocoaSpiceRenderer_h */