macOS: rework XPC helper with NSTask

Turns out you can only have one instance of a helper. Therefore we use NSTask
to launch multiple instances of QEMU. As a result, for macOS, we now embed the
QEMU binaries with the helper and launch them instead of using dylib. With this
change we also add a suffix to all the built dylibs in order to avoid collision
with macOS system libraries.

Managers/UTMQemu.h

@@ -25,19 +25,19 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface UTMQemu : NSObject
 
+@property (nonatomic, readonly) NSURL *libraryURL;
 @property (nonatomic) NSArray<NSString *> *argv;
 @property (nonatomic) dispatch_semaphore_t done;
 @property (nonatomic) NSInteger status;
 @property (nonatomic) NSInteger fatal;
 @property (nonatomic) UTMQemuThreadEntry entry;
-@property (nonatomic) QEMUHelperType type;
 
 - (instancetype)init;
 - (instancetype)initWithArgv:(NSArray<NSString *> *)argv NS_DESIGNATED_INITIALIZER;
 - (BOOL)setupXpc;
 - (void)pushArgv:(nullable NSString *)arg;
 - (void)clearArgv;
-- (void)startDylib:(nonnull NSString *)dylib completion:(void(^)(BOOL,NSString *))completion;
+- (void)start:(nonnull NSString *)name completion:(void(^)(BOOL,NSString *))completion;
 - (void)ping:(void (^)(BOOL))onResponse;
 - (void)accessDataWithBookmark:(NSData *)bookmark;
 - (void)accessDataWithBookmark:(NSData *)bookmark securityScoped:(BOOL)securityScoped completion:(void(^)(BOOL, NSData * _Nullable, NSString * _Nullable))completion;

Managers/UTMQemu.m

@@ -34,8 +34,19 @@ static const NSURLBookmarkResolutionOptions kBookmarkResolutionOptions = NSURLBo
     NSXPCConnection *_connection;
 }
 
+#pragma mark - Properties
+
 @synthesize argv = _argv;
 
+- (NSURL *)libraryURL {
+    NSURL *bundleURL = [[NSBundle mainBundle] bundleURL];
+    NSURL *contentsURL = [bundleURL URLByAppendingPathComponent:@"Contents" isDirectory:YES];
+    NSURL *frameworksURL = [contentsURL URLByAppendingPathComponent:@"Frameworks" isDirectory:YES];
+    return frameworksURL;
+}
+
+#pragma mark - Construction
+
 - (instancetype)init {
     return [self initWithArgv:[NSArray<NSString *> array]];
 }
@@ -57,6 +68,8 @@ static const NSURLBookmarkResolutionOptions kBookmarkResolutionOptions = NSURLBo
     }
 }
 
+#pragma mark - Methods
+
 - (BOOL)setupXpc {
 #if !TARGET_OS_IPHONE // only supported on macOS
     _connection = [[NSXPCConnection alloc] initWithServiceName:@"com.osy86.QEMUHelper"];
@@ -147,12 +160,41 @@ static const NSURLBookmarkResolutionOptions kBookmarkResolutionOptions = NSURLBo
     });
 }
 
-- (void)startDylib:(nonnull NSString *)dylib completion:(void(^)(BOOL,NSString *))completion {
-    if (_connection) {
+- (void)startQemuRemote:(nonnull NSString *)name completion:(void(^)(BOOL,NSString *))completion {
+    dispatch_semaphore_t lock = dispatch_semaphore_create(0);
+    __block NSString *taskIdentifier;
+    NSError *error;
+    NSData *libBookmark = [self.libraryURL bookmarkDataWithOptions:0
+                                    includingResourceValuesForKeys:nil
+                                                     relativeToURL:nil
+                                                             error:&error];
+    if (!libBookmark) {
+        completion(NO, error.localizedDescription);
+        return;
+    }
+    [[_connection remoteObjectProxyWithErrorHandler:^(NSError * _Nonnull error) {
+        completion(NO, error.localizedDescription);
+        dispatch_semaphore_signal(lock);
+    }] startQemu:name libraryBookmark:libBookmark argv:self.argv onStarted:^(NSString *identifier) {
+        UTMLog(@"started %@!", identifier);
+        taskIdentifier = identifier;
+        dispatch_semaphore_signal(lock);
+    }];
+    dispatch_semaphore_wait(lock, DISPATCH_TIME_FOREVER);
+    if (!taskIdentifier) {
+        completion(NO, NSLocalizedString(@"Failed to start QEMU process.", @"UTMQemu"));
+    } else {
         [[_connection remoteObjectProxyWithErrorHandler:^(NSError * _Nonnull error) {
             completion(NO, error.localizedDescription);
-        }] startDylib:dylib type:self.type argv:self.argv completion:completion];
+        }] registerExitHandlerForIdentifier:taskIdentifier handler:completion];
+    }
+}
+
+- (void)start:(nonnull NSString *)name completion:(void(^)(BOOL,NSString *))completion {
+    if (_connection) {
+        [self startQemuRemote:name completion:completion];
     } else {
+        NSString *dylib = [NSString stringWithFormat:@"lib%@.utm.dylib", name];
         [self startDylibThread:dylib completion:completion];
     }
 }

Managers/UTMQemuImg.m

@@ -28,10 +28,11 @@ static void *start_qemu_img(void *args) {
     NSCAssert(self->_main != NULL, @"Started thread with invalid function.");
     NSCAssert(self.argv, @"Started thread with invalid argv.");
     
-    int argc = (int)self.argv.count;
+    int argc = (int)self.argv.count + 1;
     const char *argv[argc];
+    argv[0] = "qemu-img";
     for (int i = 0; i < self.argv.count; i++) {
-        argv[i] = [self.argv[i] UTF8String];
+        argv[i+1] = [self.argv[i] UTF8String];
     }
     self.status = self->_main(argc, argv);
     dispatch_semaphore_signal(self.done);
@@ -41,14 +42,12 @@ static void *start_qemu_img(void *args) {
 - (instancetype)initWithArgv:(NSArray<NSString *> *)argv {
     if (self = [super initWithArgv:argv]) {
         self.entry = start_qemu_img;
-        self.type = QEMUHelperTypeImg;
     }
     return self;
 }
 
 - (void)buildArgv {
     [self clearArgv];
-    [self pushArgv:@"qemu-img"];
     switch (self.op) {
         case kUTMQemuImgCreate: {
             [self pushArgv:@"create"];
@@ -100,7 +99,7 @@ static void *start_qemu_img(void *args) {
         return;
     }
     [self buildArgv];
-    [self startDylib:@"libqemu-img.dylib" completion:completion];
+    [self start:@"qemu-img" completion:completion];
     once = YES;
 }
 

Managers/UTMQemuSystem.m

@@ -33,10 +33,11 @@ static void *start_qemu(void *args) {
     NSCAssert(self->_qemu_cleanup != NULL, @"Started thread with invalid function.");
     NSCAssert(qemuArgv, @"Started thread with invalid argv.");
     
-    int argc = (int)qemuArgv.count;
+    int argc = (int)qemuArgv.count + 1;
     const char *argv[argc];
+    argv[0] = "qemu-system";
     for (int i = 0; i < qemuArgv.count; i++) {
-        argv[i] = [qemuArgv[i] UTF8String];
+        argv[i+1] = [qemuArgv[i] UTF8String];
     }
     const char *envp[] = { NULL };
     self->_qemu_init(argc, argv, envp);
@@ -50,7 +51,6 @@ static void *start_qemu(void *args) {
 - (instancetype)initWithArgv:(NSArray<NSString *> *)argv {
     if (self = [super initWithArgv:argv]) {
         self.entry = start_qemu;
-        self.type = QEMUHelperTypeSystem;
     }
     return self;
 }

Managers/UTMQemuSystemConfiguration.m

@@ -253,7 +253,6 @@
 - (void)argsFromConfiguration:(BOOL)withUserArgs {
     NSURL *resourceURL = [[NSBundle mainBundle] URLForResource:@"qemu" withExtension:nil];
     [self clearArgv];
-    [self pushArgv:@"qemu"];
     [self pushArgv:@"-L"];
     [self accessDataWithBookmark:[resourceURL bookmarkDataWithOptions:0
                                        includingResourceValuesForKeys:nil
@@ -364,9 +363,9 @@
 }
 
 - (void)startWithCompletion:(void (^)(BOOL, NSString * _Nonnull))completion {
-    NSString *dylib = [NSString stringWithFormat:@"libqemu-system-%@.dylib", self.configuration.systemArchitecture];
+    NSString *name = [NSString stringWithFormat:@"qemu-system-%@", self.configuration.systemArchitecture];
     [self argsFromConfiguration:YES];
-    [self startDylib:dylib completion:completion];
+    [self start:name completion:completion];
 }
 
 @end

Managers/UTMVirtualMachine.m

@@ -291,26 +291,19 @@ error:
         }
         dispatch_semaphore_signal(self->_qemu_exit_sema);
     }];
-    
-    [_qemu_system ping:^(BOOL pong) {
-        if (!pong) {
-            [self errorTriggered:NSLocalizedString(@"Timed out waiting for QEMU to launch.", @"UTMVirtualMachine")];
+    [self->_ioService connectWithCompletion:^(BOOL success, NSError * _Nullable error) {
+        if (!success) {
+            [self errorTriggered:NSLocalizedString(@"Failed to connect to display server.", @"UTMVirtualMachine")];
         } else {
-            [self->_ioService connectWithCompletion:^(BOOL success, NSError * _Nullable error) {
-                if (!success) {
-                    [self errorTriggered:NSLocalizedString(@"Failed to connect to display server.", @"UTMVirtualMachine")];
-                } else {
-                    [self changeState:kVMStarted];
-                    [self restoreViewState];
-                    if (self.viewState.suspended) {
-                        [self deleteSaveVM];
-                    }
-                }
-            }];
-            self->_qemu.retries = kQMPMaxConnectionTries;
-            [self->_qemu connect];
+            [self changeState:kVMStarted];
+            [self restoreViewState];
+            if (self.viewState.suspended) {
+                [self deleteSaveVM];
+            }
         }
     }];
+    self->_qemu.retries = kQMPMaxConnectionTries;
+    [self->_qemu connect];
     _is_busy = NO;
     return YES;
 }

QEMUHelper/QEMUHelper.m

@@ -15,33 +15,42 @@
 //
 
 #import "QEMUHelper.h"
-#import "UTMQemu.h"
-#import "UTMQemuImg.h"
-#import "UTMQemuSystem.h"
-#import "UTMLogging.h"
 #import <stdio.h>
 
-@implementation QEMUHelper {
-    UTMQemu *_qemu;
-    NSMutableArray<NSData *> *_bookmarks;
-}
+@interface QEMUHelper ()
+
+@property NSMutableArray<NSURL *> *urls;
+@property NSMutableDictionary<NSString *, NSTask *> *processes;
+@property NSMutableDictionary<NSString *, void(^)(BOOL, NSString *)> *exitHandlers;
+
+@end
+
+@implementation QEMUHelper
 
 - (instancetype)init {
     if (self = [super init]) {
-        _bookmarks = [NSMutableArray<NSData *> array];
+        self.urls = [NSMutableArray array];
+        self.processes = [NSMutableDictionary dictionary];
     }
     return self;
 }
 
-- (void)accessDataWithBookmark:(NSData *)bookmark securityScoped:(BOOL)securityScoped completion:(void(^)(BOOL, NSData * _Nullable, NSString * _Nullable))completion {
-    BOOL stale = false;
+- (void)dealloc {
+    for (NSURL *url in self.urls) {
+        [url stopAccessingSecurityScopedResource];
+    }
+}
+
+- (void)accessDataWithBookmark:(NSData *)bookmark securityScoped:(BOOL)securityScoped completion:(void(^)(BOOL, NSData * _Nullable, NSString * _Nullable))completion  {
+    BOOL stale = NO;
+    NSError *err;
     NSURL *url = [NSURL URLByResolvingBookmarkData:bookmark
                                            options:(securityScoped ? NSURLBookmarkResolutionWithSecurityScope : 0)
                                      relativeToURL:nil
                                bookmarkDataIsStale:&stale
-                                             error:nil];
+                                             error:&err];
     if (!url) {
-        UTMLog(@"Failed to resolve bookmark!");
+        NSLog(@"Failed to access bookmark data.");
         completion(NO, nil, nil);
         return;
     }
@@ -49,54 +58,106 @@
         bookmark = [url bookmarkDataWithOptions:NSURLBookmarkCreationWithSecurityScope
                  includingResourceValuesForKeys:nil
                                   relativeToURL:nil
-                                          error:nil];
+                                          error:&err];
         if (!bookmark) {
-            UTMLog(@"Failed to create new bookmark!");
+            NSLog(@"Failed to create new bookmark!");
             completion(NO, bookmark, url.path);
             return;
         }
     }
-    if (_qemu == nil) {
-        [_bookmarks addObject:bookmark];
-        completion(YES, bookmark, url.path);
+    if ([url startAccessingSecurityScopedResource]) {
+        [self.urls addObject:url];
+    } else {
+        NSLog(@"Failed to access security scoped resource for: %@", url);
+    }
+    completion(YES, bookmark, url.path);
+}
+
+- (void)startQemu:(NSString *)binName libraryBookmark:(NSData *)libBookmark  argv:(NSArray<NSString *> *)argv onStarted:(void(^)(NSString * _Nullable))onStarted {
+    NSURL *qemuURL = [[NSBundle mainBundle] URLForAuxiliaryExecutable:binName];
+    if (!qemuURL || ![[NSFileManager defaultManager] fileExistsAtPath:qemuURL.path]) {
+        NSLog(@"Cannot find executable for %@", binName);
+        onStarted(nil);
+        return;
+    }
+    
+    NSError *err;
+    NSURL *libraryPath = [NSURL URLByResolvingBookmarkData:libBookmark
+                                                   options:0
+                                             relativeToURL:nil
+                                       bookmarkDataIsStale:nil
+                                                     error:&err];
+    if (!libraryPath || ![[NSFileManager defaultManager] fileExistsAtPath:libraryPath.path]) {
+        NSLog(@"Cannot resolve library path: %@", err);
+        onStarted(nil);
+        return;
+    }
+    
+    NSTask *task = [NSTask new];
+    NSString *identifier = [self storeProcess:task];
+    task.executableURL = qemuURL;
+    task.arguments = argv;
+    task.environment = @{@"DYLD_LIBRARY_PATH": libraryPath.path};
+    task.qualityOfService = NSQualityOfServiceUserInitiated;
+    task.terminationHandler = ^(NSTask *task) {
+        [self exitProcessForIdentifier:identifier];
+    };
+    if (![task launchAndReturnError:&err]) {
+        NSLog(@"Error starting QEMU: %@", err);
+        [self exitProcessForIdentifier:identifier];
+        onStarted(nil);
     } else {
-        [_qemu accessDataWithBookmark:bookmark securityScoped:YES completion:completion];
+        onStarted(identifier);
     }
 }
 
-- (void)ping:(void (^)(BOOL))onResponse {
-    onResponse(_qemu != nil);
+- (void)registerExitHandlerForIdentifier:(NSString *)identifier handler:(void(^)(BOOL,NSString *))handler {
+    if (![self storeExitHandlerForIdentifier:identifier handler:handler]) {
+        handler(NO, NSLocalizedString(@"QEMU already exited.", @"QEMUHelper"));
+    }
 }
 
-- (void)startDylib:(NSString *)dylib type:(QEMUHelperType)type argv:(NSArray<NSString *> *)argv completion:(void(^)(BOOL,NSString *))completion {
-    switch (type) {
-        case QEMUHelperTypeImg:
-            _qemu = [[UTMQemuImg alloc] initWithArgv:argv];
-            break;
-        case QEMUHelperTypeSystem:
-            _qemu = [[UTMQemuSystem alloc] initWithArgv:argv];
-            break;
-        default:
-            NSAssert(0, @"Invalid helper type.");
-            break;
+#pragma mark - Helpers
+
+- (NSString *)storeProcess:(NSTask *)process {
+    @synchronized (self.processes) {
+        NSString *identifier = [NSUUID UUID].UUIDString;
+        self.processes[identifier] = process;
+        return identifier;
     }
-    
-    // pass in any bookmarks in queue
-    if (_bookmarks.count > 0) {
-        for (NSData *bookmark in _bookmarks) {
-            [_qemu accessDataWithBookmark:bookmark securityScoped:YES completion:^(BOOL success, NSData *bookmark, NSString *path) {
-                if (!success) {
-                    UTMLog(@"Access bookmark failed for: %@", path);
-                }
-            }];
+}
+
+- (BOOL)storeExitHandlerForIdentifier:(NSString *)identifier handler:(void(^)(BOOL,NSString *))handler {
+    @synchronized (self.processes) {
+        if (self.processes[identifier]) {
+            self.exitHandlers[identifier] = handler;
+            return YES;
+        } else {
+            return NO;
         }
-        [_bookmarks removeAllObjects];
     }
-    
-    [_qemu startDylib:dylib completion:^(BOOL success, NSString *msg) {
-        completion(success, msg);
-        self->_qemu = nil;
-    }];
+}
+
+- (void)exitProcessForIdentifier:(NSString *)identifier {
+    NSTask *task;
+    void (^exitHandler)(BOOL, NSString *);
+    BOOL normalExit = NO;
+    @synchronized (self.processes) {
+        task = self.processes[identifier];
+        [self.processes removeObjectForKey:identifier];
+        exitHandler = self.exitHandlers[identifier];
+        [self.exitHandlers removeObjectForKey:identifier];
+    }
+    if (task) {
+        if (task.running) {
+            [task terminate];
+        } else {
+            normalExit = task.terminationReason == NSTaskTerminationReasonExit && task.terminationStatus == 0;
+        }
+    }
+    if (exitHandler) {
+        exitHandler(normalExit, nil); // TODO: get last error line
+    }
 }
 
 @end

QEMUHelper/QEMUHelperProtocol.h

@@ -16,20 +16,14 @@
 
 #import <Foundation/Foundation.h>
 
-typedef NS_ENUM(NSInteger, QEMUHelperType) {
-    QEMUHelperTypeImg,
-    QEMUHelperTypeSystem,
-    QEMUHelperTypeMax
-};
-
 NS_ASSUME_NONNULL_BEGIN
 
 // The protocol that this service will vend as its API. This header file will also need to be visible to the process hosting the service.
 @protocol QEMUHelperProtocol
 
 - (void)accessDataWithBookmark:(NSData *)bookmark securityScoped:(BOOL)securityScoped completion:(void(^)(BOOL, NSData * _Nullable, NSString * _Nullable))completion;
-- (void)ping:(void (^)(BOOL))onResponse;
-- (void)startDylib:(NSString *)dylib type:(QEMUHelperType)type argv:(NSArray<NSString *> *)argv completion:(void(^)(BOOL,NSString *))completion;
+- (void)startQemu:(NSString *)binName libraryBookmark:(NSData *)libBookmark argv:(NSArray<NSString *> *)argv onStarted:(void(^)(NSString * _Nullable))onStarted;
+- (void)registerExitHandlerForIdentifier:(NSString *)identifier handler:(void(^)(BOOL,NSString *))handler;
     
 @end
 

scripts/build_dependencies.sh

@@ -212,7 +212,7 @@ build_qemu () {
     CFLAGS=
     CXXFLAGS=
     LDFLAGS=
-    build $QEMU_SRC --enable-shared-lib --with-coroutine=libucontext
+    build $QEMU_SRC $1 --with-coroutine=libucontext
     CFLAGS="$QEMU_CFLAGS"
     CXXFLAGS="$QEMU_CXXFLAGS"
     LDFLAGS="$QEMU_LDFLAGS"
@@ -239,34 +239,43 @@ build_spice_client () {
 
 fixup () {
     FILE=$1
+    BASE=$(basename "$FILE")
+    BASEFILENAME=${BASE%.*}
+    BASEFILEEXT=${BASE:${#BASEFILENAME}}
+    NEWFILENAME="$BASEFILENAME.utm$BASEFILEEXT"
+    if [ -z "$BASEFILEEXT" ]; then
+        NEWFILENAME="$BASE"
+    fi
     LIST=$(otool -L "$FILE" | tail -n +3 | cut -d ' ' -f 1 | awk '{$1=$1};1')
     OLDIFS=$IFS
     IFS=$'\n'
     echo "${GREEN}Fixing up $FILE...${NC}"
-    newname="@executable_path/Frameworks/$(basename "$FILE")"
-    if [ "x$PLATFORM" == "xmacos" ]; then
-        newname="@rpath/$(basename "$FILE")"
-    fi
+    newname="@rpath/$NEWFILENAME"
     install_name_tool -id "$newname" "$FILE"
     for f in $LIST
     do
         base=$(basename "$f")
+        basefilename=${base%.*}
+        basefileext=${base:${#basefilename}}
         dir=$(dirname "$f")
         if [ "$dir" == "$PREFIX/lib" ]; then
-            newname="@executable_path/Frameworks/$base"
-            if [ "x$PLATFORM" == "xmacos" ]; then
-                newname="@rpath/$base"
-            fi
+            newname="@rpath/$basefilename.utm$basefileext"
             install_name_tool -change "$f" "$newname" "$FILE"
         fi
     done
+    mv "$FILE" "$(dirname "$FILE")/$NEWFILENAME"
     IFS=$OLDIFS
 }
 
 fixup_all () {
-    FILES=$(find "$SYSROOT_DIR/lib" -type f -name "*.dylib")
     OLDIFS=$IFS
     IFS=$'\n'
+    FILES=$(find "$SYSROOT_DIR/lib" -type f -name "*.dylib")
+    for f in $FILES
+    do
+        fixup $f
+    done
+    FILES=$(find "$SYSROOT_DIR/bin" -type f -name "qemu-*")
     for f in $FILES
     do
         fixup $f
@@ -363,6 +372,7 @@ ios )
         ;;
     esac
     PLATFORM_FAMILY_NAME="iOS"
+    QEMU_PLATFORM_BUILD_FLAGS="--enable-shared-lib"
     ;;
 macos )
     if [ -z "$SDKMINVER" ]; then
@@ -371,6 +381,7 @@ macos )
     fi
     SDK=macosx
     PLATFORM_FAMILY_NAME="macOS"
+    QEMU_PLATFORM_BUILD_FLAGS=""
     ;;
 * )
     usage
@@ -446,7 +457,7 @@ if [ -z "$QEMU_ONLY" ]; then
     rm -f "$BUILD_DIR/BUILD_SUCCESS"
     build_qemu_dependencies
 fi
-build_qemu
+build_qemu $QEMU_PLATFORM_BUILD_FLAGS
 if [ -z "$QEMU_ONLY" ]; then
     steal_libucontext # should be a better way...
     build_spice_client