qemu: use dylib for macOS

This works around an App Sandbox bug where an inherited sandbox cannot
use the hypervisor. We use fork() instead.

QEMUHelper/Bootstrap.c

@@ -0,0 +1,58 @@
+//
+// Copyright © 2021 osy. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include "Bootstrap.h"
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static int launchQemu(const char *dylibPath, int argc, const char **argv) {
+    void *dlctx;
+    int (*qemu_init)(int, const char *[], const char *[]);
+    void (*qemu_main_loop)(void);
+    void (*qemu_cleanup)(void);
+    
+    if ((dlctx = dlopen(dylibPath, RTLD_LAZY | RTLD_FIRST)) == NULL) {
+        fprintf(stderr, "Error loading %s: %s\n", dylibPath, dlerror());
+        return -1;
+    }
+    qemu_init = dlsym(dlctx, "qemu_init");
+    qemu_main_loop = dlsym(dlctx, "qemu_main_loop");
+    qemu_cleanup = dlsym(dlctx, "qemu_cleanup");
+    if (qemu_init == NULL || qemu_main_loop == NULL || qemu_cleanup == NULL) {
+        fprintf(stderr, "Error resolving %s: %s\n", dylibPath, dlerror());
+        return -1;
+    }
+    const char *envp[] = { NULL };
+    qemu_init(argc, argv, envp);
+    qemu_main_loop();
+    qemu_cleanup();
+    return 0;
+}
+
+pid_t startQemu(const char *dylibPath, int argc, const char **argv, int newStdout, int newStderr) {
+    pid_t pid = fork();
+    if (pid != 0) { // parent or error
+        return pid;
+    }
+    // set up console output
+    dup2(newStdout, STDOUT_FILENO);
+    dup2(newStderr, STDERR_FILENO);
+    close(newStdout);
+    close(newStderr);
+    int res = launchQemu(dylibPath, argc, argv);
+    exit(res);
+}

QEMUHelper/Bootstrap.h

@@ -0,0 +1,24 @@
+//
+// Copyright © 2021 osy. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef Bootstrap_h
+#define Bootstrap_h
+
+#include <unistd.h>
+
+pid_t startQemu(const char *dylibPath, int argc, const char **argv, int newStdout, int newStderr);
+
+#endif /* Bootstrap_h */

QEMUHelper/QEMUHelper.m

@@ -16,10 +16,12 @@
 
 #import "QEMUHelper.h"
 #import <stdio.h>
+#import "Bootstrap.h"
 
 @interface QEMUHelper ()
 
 @property NSMutableArray<NSURL *> *urls;
+@property dispatch_queue_t childWaitQueue;
 
 @end
 
@@ -28,6 +30,7 @@
 - (instancetype)init {
     if (self = [super init]) {
         self.urls = [NSMutableArray array];
+        self.childWaitQueue = dispatch_queue_create("childWaitQueue", DISPATCH_QUEUE_CONCURRENT);
     }
     return self;
 }
@@ -104,6 +107,7 @@
         return;
     }
     
+#if 0 // old NSTask code does not work with inherited sandbox
     NSTask *task = [NSTask new];
     task.executableURL = qemuURL;
     task.arguments = argv;
@@ -119,6 +123,46 @@
         NSLog(@"Error starting QEMU: %@", err);
         onExit(NO, NSLocalizedString(@"Error starting QEMU.", @"QEMUHelper"));
     }
+#endif
+    // convert all the Objective-C strings to C strings as we should not use objects in this context after fork()
+    NSString *path = [libraryPath URLByAppendingPathComponent:binName].path;
+    char *cpath = strdup(path.UTF8String);
+    int argc = (int)argv.count + 1;
+    char **cargv = calloc(argc, sizeof(char *));
+    cargv[0] = cpath;
+    for (int i = 0; i < argc; i++) {
+        cargv[i+1] = strdup(argv[i].UTF8String);
+    }
+    int newStdOut = standardOutput.fileDescriptor;
+    int newStdErr = standardError.fileDescriptor;
+    pid_t pid = startQemu(cpath, argc, (const char **)cargv, newStdOut, newStdErr);
+    // free all resources regardless of error because on success, child has a copy
+    [standardOutput closeFile];
+    [standardError closeFile];
+    for (int i = 0; i < argc; i++) {
+        free(cargv[i]);
+    }
+    free(cargv);
+    if (pid < 0) {
+        NSLog(@"Error starting QEMU: %d", pid);
+        onExit(NO, NSLocalizedString(@"Error starting QEMU.", @"QEMUHelper"));
+    } else {
+        // a new thread to reap the child and wait on its status
+        dispatch_async(self.childWaitQueue, ^{
+            do {
+                int status;
+                if (waitpid(pid, &status, WNOHANG) <= 0) {
+                    NSLog(@"waitpid(%d) returned error.", pid);
+                    onExit(NO, NSLocalizedString(@"QEMU exited unexpectedly.", @"QEMUHelper"));
+                } else if (WIFEXITED(status)) {
+                    NSLog(@"child process %d terminated", pid);
+                    onExit(WEXITSTATUS(status) == 0, nil);
+                } else {
+                    continue; // another reason, we ignore
+                }
+            } while (0);
+        });
+    }
 }
 
 @end

UTM.xcodeproj/project.pbxproj

@@ -273,6 +273,7 @@
 		CE0B6F2F24AD67BE00FE012D /* libjson-glib-1.0.0.utm.dylib in Embed Libraries */ = {isa = PBXBuildFile; fileRef = CE2D63E222653C7400FC7E63 /* libjson-glib-1.0.0.utm.dylib */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; };
 		CE0B6F3124AD67C100FE012D /* libphodav-2.0.0.utm.dylib in Embed Libraries */ = {isa = PBXBuildFile; fileRef = CE059DC0243BD67100338317 /* libphodav-2.0.0.utm.dylib */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; };
 		CE0B6F5424AD67FA00FE012D /* libspice-client-glib-2.0.8.utm.dylib in Embed Libraries */ = {isa = PBXBuildFile; fileRef = CE2D63FE22653C7500FC7E63 /* libspice-client-glib-2.0.8.utm.dylib */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; };
+		CE0DF17225A80B6300A51894 /* Bootstrap.c in Sources */ = {isa = PBXBuildFile; fileRef = CE0DF17125A80B6300A51894 /* Bootstrap.c */; };
 		CE0E9B87252FD06B0026E02B /* SwiftUI.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = CE0E9B86252FD06B0026E02B /* SwiftUI.framework */; settings = {ATTRIBUTES = (Weak, ); }; };
 		CE0FE12824D3B08B0086CEF0 /* VMDisplayWindow.xib in Resources */ = {isa = PBXBuildFile; fileRef = CE0FE12724D3B08B0086CEF0 /* VMDisplayWindow.xib */; };
 		CE178ED725017EB100CE0E33 /* qemu-system-i386 in CopyFiles */ = {isa = PBXBuildFile; fileRef = CE178EB025017EAE00CE0E33 /* qemu-system-i386 */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; };
@@ -994,6 +995,8 @@
 		CE059DCA243FBA3C00338317 /* VMConfigCell.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = VMConfigCell.m; sourceTree = "<group>"; };
 		CE0B6D8324AD5ADE00FE012D /* UTMScreenshot.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = UTMScreenshot.h; sourceTree = "<group>"; };
 		CE0B6D8424AD5ADE00FE012D /* UTMScreenshot.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = UTMScreenshot.m; sourceTree = "<group>"; };
+		CE0DF17025A80B6300A51894 /* Bootstrap.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Bootstrap.h; sourceTree = "<group>"; };
+		CE0DF17125A80B6300A51894 /* Bootstrap.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = Bootstrap.c; sourceTree = "<group>"; };
 		CE0E828D24E4D4CE003EA9FE /* Base */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = Base; path = Base.lproj/Localizable.strings; sourceTree = "<group>"; };
 		CE0E829424E4D509003EA9FE /* Base */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = Base; path = Base.lproj/InfoPlist.strings; sourceTree = "<group>"; };
 		CE0E9B86252FD06B0026E02B /* SwiftUI.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftUI.framework; path = System/Library/Frameworks/SwiftUI.framework; sourceTree = SDKROOT; };
@@ -2444,6 +2447,8 @@
 			isa = PBXGroup;
 			children = (
 				CE03D0D024D9A62B00F76B84 /* QEMUHelper.entitlements */,
+				CE0DF17025A80B6300A51894 /* Bootstrap.h */,
+				CE0DF17125A80B6300A51894 /* Bootstrap.c */,
 				CEBDA1DC24D8BDDA0010B5EC /* QEMUHelperProtocol.h */,
 				CEBDA1DD24D8BDDB0010B5EC /* QEMUHelper.h */,
 				CEBDA1DE24D8BDDB0010B5EC /* QEMUHelper.m */,
@@ -3235,6 +3240,7 @@
 			files = (
 				CEBDA1E124D8BDDB0010B5EC /* main.m in Sources */,
 				CEBDA1DF24D8BDDB0010B5EC /* QEMUHelper.m in Sources */,
+				CE0DF17225A80B6300A51894 /* Bootstrap.c in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

scripts/build_dependencies.sh

@@ -428,7 +428,7 @@ macos )
     CFLAGS_MINVER="-mmacos-version-min=$SDKMINVER"
     CFLAGS_TARGET="-target $ARCH-apple-macos"
     PLATFORM_FAMILY_NAME="macOS"
-    QEMU_PLATFORM_BUILD_FLAGS="--disable-cocoa --disable-curl --cpu=$CPU"
+    QEMU_PLATFORM_BUILD_FLAGS="--enable-shared-lib --disable-cocoa --disable-curl --cpu=$CPU"
     ;;
 * )
     usage