display(visionOS): integrate VisionKeyboardKit

Platform/iOS/VMDisplayHostedView.swift

@@ -168,7 +168,12 @@ struct VMDisplayHostedView: UIViewControllerRepresentable {
         if let vc = uiViewController as? VMDisplayMetalViewController {
             vc.vmInput = session.primaryInput
         }
-        if state.isKeyboardShown != state.isKeyboardRequested {
+        #if os(visionOS)
+        let useSystemOsk = !(uiViewController is VMDisplayMetalViewController)
+        #else
+        let useSystemOsk = true
+        #endif
+        if useSystemOsk && state.isKeyboardShown != state.isKeyboardRequested {
             DispatchQueue.main.async {
                 if state.isKeyboardRequested {
                     uiViewController.showKeyboard()

Platform/iOS/VMWindowView.swift

@@ -16,6 +16,9 @@
 
 import SwiftUI
 import SwiftUIVisualEffects
+#if os(visionOS)
+import VisionKeyboardKit
+#endif
 
 struct VMWindowView: View {
     let id: VMSessionState.WindowID
@@ -24,7 +27,10 @@ struct VMWindowView: View {
     @State private var state: VMWindowState
     @EnvironmentObject private var session: VMSessionState
     @Environment(\.scenePhase) private var scenePhase
-    
+    #if os(visionOS)
+    @Environment(\.dismissWindow) private var dismissWindow
+    #endif
+
     private let keyboardDidShowNotification = NotificationCenter.default.publisher(for: UIResponder.keyboardDidShowNotification)
     private let keyboardDidHideNotification = NotificationCenter.default.publisher(for: UIResponder.keyboardDidHideNotification)
     private let didReceiveMemoryWarningNotification = NotificationCenter.default.publisher(for: UIApplication.didReceiveMemoryWarningNotification)
@@ -228,6 +234,9 @@ struct VMWindowView: View {
             if !isInteractive {
                 session.externalWindowBinding = nil
             }
+            #if os(visionOS)
+            dismissWindow(keyboardFor: state.id)
+            #endif
         }
     }
     

Platform/visionOS/UTMApp.swift

@@ -15,6 +15,7 @@
 //
 
 import SwiftUI
+import VisionKeyboardKit
 
 @MainActor
 struct UTMApp: App {
@@ -73,5 +74,6 @@ struct UTMApp: App {
         }
         .windowStyle(.plain)
         .windowResizability(.contentMinSize)
+        KeyboardWindowGroup()
     }
 }

Platform/visionOS/VMToolbarOrnamentModifier.swift

@@ -15,11 +15,19 @@
 //
 
 import SwiftUI
+import VisionKeyboardKit
+#if !WITH_USB
+import CocoaSpiceNoUsb
+#else
+import CocoaSpice
+#endif
 
 struct VMToolbarOrnamentModifier: ViewModifier {
     @Binding var state: VMWindowState
     @EnvironmentObject private var session: VMSessionState
     @AppStorage("ToolbarIsCollapsed") private var isCollapsed: Bool = false
+    @Environment(\.openWindow) private var openWindow
+    @Environment(\.dismissWindow) private var dismissWindow
 
     func body(content: Content) -> some View {
         content.ornament(visibility: isCollapsed ? .hidden : .visible, attachmentAnchor: .scene(.top)) {
@@ -71,11 +79,39 @@ struct VMToolbarOrnamentModifier: ViewModifier {
                 VMToolbarDisplayMenuView(state: $state)
                     .disabled(state.isBusy)
                 Button {
-                    state.isKeyboardRequested = true
+                    if case .display(_, _) = state.device {
+                        state.isKeyboardRequested = !state.isKeyboardShown
+                    } else {
+                        state.isKeyboardRequested = true
+                    }
                 } label: {
                     Label("Keyboard", systemImage: "keyboard")
                 }
                 .disabled(state.isBusy)
+                .onChange(of: state.isKeyboardRequested) { _, newValue in
+                    guard case .display(_, _) = state.device else {
+                        return
+                    }
+                    if newValue {
+                        openWindow(keyboardFor: state.id)
+                    } else {
+                        dismissWindow(keyboardFor: state.id)
+                    }
+                }
+                .onReceive(KeyboardEvent.publisher(for: state.id)) { event in
+                    switch event {
+                    case .keyboardDidAppear:
+                        state.isKeyboardShown = true
+                        state.isKeyboardRequested = true
+                    case .keyboardDidDisappear:
+                        state.isKeyboardShown = false
+                        state.isKeyboardRequested = false
+                    case .keyUp(let keyCode, let modifier):
+                        handleKeyEvent(keyCode, modifier: modifier, isKeyDown: false)
+                    case .keyDown(let keyCode, let modifier):
+                        handleKeyEvent(keyCode, modifier: modifier, isKeyDown: true)
+                    }
+                }
                 Divider()
                 Button {
                     isCollapsed = true
@@ -94,6 +130,18 @@ struct VMToolbarOrnamentModifier: ViewModifier {
                 .modifier(ToolbarOrnamentViewModifier())
         }
     }
+
+    private func handleKeyEvent(_ keyCode: KeyboardKeyCode, modifier: KeyboardModifier, isKeyDown: Bool) {
+        guard let primaryInput = session.primaryInput else {
+            logger.debug("ignoring key event because input channel is not ready")
+            return
+        }
+        var scanCode = keyCode.ps2Set1ScanMake(modifier).reduce(Int32(0), { ($0 << 8) | Int32($1) })
+        if ((scanCode & 0xFF00) == 0xE000) {
+            scanCode = 0x100 | (scanCode & 0xFF);
+        }
+        primaryInput.send(isKeyDown ? .press : .release, code: scanCode)
+    }
 }
 
 // the following was suggested by Apple via Feedback to look close to .toolbar() with .bottomOrnament

UTM.xcodeproj/project.pbxproj

@@ -646,6 +646,9 @@
 		CE8813D324CD230300532628 /* ActivityView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE8813D224CD230300532628 /* ActivityView.swift */; };
 		CE8813D524CD265700532628 /* VMShareFileModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE8813D424CD265700532628 /* VMShareFileModifier.swift */; };
 		CE8813D624CD265700532628 /* VMShareFileModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE8813D424CD265700532628 /* VMShareFileModifier.swift */; };
+		CE89CB0E2B8B1B5A006B2CC2 /* VisionKeyboardKit in Frameworks */ = {isa = PBXBuildFile; platformFilters = (xros, ); productRef = CE89CB0D2B8B1B5A006B2CC2 /* VisionKeyboardKit */; };
+		CE89CB102B8B1B6A006B2CC2 /* VisionKeyboardKit in Frameworks */ = {isa = PBXBuildFile; platformFilters = (xros, ); productRef = CE89CB0F2B8B1B6A006B2CC2 /* VisionKeyboardKit */; };
+		CE89CB122B8B1B7A006B2CC2 /* VisionKeyboardKit in Frameworks */ = {isa = PBXBuildFile; platformFilters = (xros, ); productRef = CE89CB112B8B1B7A006B2CC2 /* VisionKeyboardKit */; };
 		CE928C2A26ABE6690099F293 /* UTMAppleVirtualMachine.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE928C2926ABE6690099F293 /* UTMAppleVirtualMachine.swift */; };
 		CE928C3126ACCDEA0099F293 /* VMAppleRemovableDrivesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE928C3026ACCDEA0099F293 /* VMAppleRemovableDrivesView.swift */; };
 		CE93758924B930270074066F /* BusyOverlay.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE7D972B24B2B17D0080CB69 /* BusyOverlay.swift */; };
@@ -2121,6 +2124,7 @@
 				84818C0C2898A07A009EDB67 /* AVFAudio.framework in Frameworks */,
 				CE2D934924AD46670059923A /* gstreamer-1.0.0.framework in Frameworks */,
 				CE2D934B24AD46670059923A /* json-glib-1.0.0.framework in Frameworks */,
+				CE89CB0E2B8B1B5A006B2CC2 /* VisionKeyboardKit in Frameworks */,
 				CE2D934C24AD46670059923A /* ffi.7.framework in Frameworks */,
 				CE2D934D24AD46670059923A /* gstnet-1.0.0.framework in Frameworks */,
 				CE2D934E24AD46670059923A /* gstbase-1.0.0.framework in Frameworks */,
@@ -2251,6 +2255,7 @@
 				CEA45F25263519B5002FA97D /* libgstaudiotestsrc.a in Frameworks */,
 				CEA45F26263519B5002FA97D /* libgstvideoconvert.a in Frameworks */,
 				CEA45F27263519B5002FA97D /* libgstaudioconvert.a in Frameworks */,
+				CE89CB102B8B1B6A006B2CC2 /* VisionKeyboardKit in Frameworks */,
 				8401865C2887AFDC0050AC51 /* SwiftTerm in Frameworks */,
 				CEA45F28263519B5002FA97D /* libgstvideoscale.a in Frameworks */,
 				CEA45F29263519B5002FA97D /* IQKeyboardManagerSwift in Frameworks */,
@@ -2386,6 +2391,7 @@
 				CEF7F6692AEEDCC400E34952 /* opus.0.framework in Frameworks */,
 				CEF7F66A2AEEDCC400E34952 /* glib-2.0.0.framework in Frameworks */,
 				CEF7F66B2AEEDCC400E34952 /* png16.16.framework in Frameworks */,
+				CE89CB122B8B1B7A006B2CC2 /* VisionKeyboardKit in Frameworks */,
 				CEF7F66C2AEEDCC400E34952 /* gstfft-1.0.0.framework in Frameworks */,
 				CEF7F66D2AEEDCC400E34952 /* crypto.1.1.framework in Frameworks */,
 				CEF7F66E2AEEDCC400E34952 /* gstpbutils-1.0.0.framework in Frameworks */,
@@ -3040,6 +3046,7 @@
 				84CE3DAB2904C14100FF068B /* InAppSettingsKit */,
 				84A0A8892A47D5D10038F329 /* QEMUKit */,
 				CE9B15372B11A4A7003A32DD /* SwiftConnect */,
+				CE89CB0D2B8B1B5A006B2CC2 /* VisionKeyboardKit */,
 			);
 			productName = UTM;
 			productReference = CE2D93BE24AD46670059923A /* UTM.app */;
@@ -3121,6 +3128,7 @@
 				846D878529050B6B0095F10B /* InAppSettingsKit */,
 				84A0A88B2A47D5D70038F329 /* QEMUKit */,
 				CE9B15392B11A4AE003A32DD /* SwiftConnect */,
+				CE89CB0F2B8B1B6A006B2CC2 /* VisionKeyboardKit */,
 			);
 			productName = UTM;
 			productReference = CEA45FB9263519B5002FA97D /* UTM SE.app */;
@@ -3169,6 +3177,7 @@
 				CEF7F5922AEEDCC400E34952 /* InAppSettingsKit */,
 				CEF7F6D52AEEEF7D00E34952 /* CocoaSpiceNoUsb */,
 				CE9B153B2B11A4B4003A32DD /* SwiftConnect */,
+				CE89CB112B8B1B7A006B2CC2 /* VisionKeyboardKit */,
 			);
 			productName = UTM;
 			productReference = CEF7F6D32AEEDCC400E34952 /* UTM Remote.app */;
@@ -3240,6 +3249,7 @@
 				84A0A8862A47D5C50038F329 /* XCRemoteSwiftPackageReference "QEMUKit" */,
 				CE9B15342B11A491003A32DD /* XCRemoteSwiftPackageReference "SwiftConnect" */,
 				CEDD11BF2B7C74D7004DDAC6 /* XCRemoteSwiftPackageReference "SwiftPortmap" */,
+				CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */,
 			);
 			productRefGroup = CE550BCA225947990063E575 /* Products */;
 			projectDirPath = "";
@@ -5090,6 +5100,14 @@
 				minimumVersion = 1.5.3;
 			};
 		};
+		CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */ = {
+			isa = XCRemoteSwiftPackageReference;
+			repositoryURL = "https://github.com/utmapp/VisionKeyboardKit.git";
+			requirement = {
+				branch = main;
+				kind = branch;
+			};
+		};
 		CE93759724BB821F0074066F /* XCRemoteSwiftPackageReference "IQKeyboardManager" */ = {
 			isa = XCRemoteSwiftPackageReference;
 			repositoryURL = "https://github.com/hackiftekhar/IQKeyboardManager.git";
@@ -5295,6 +5313,21 @@
 			package = CE020BA524AEDEF000B44AB6 /* XCRemoteSwiftPackageReference "swift-log" */;
 			productName = Logging;
 		};
+		CE89CB0D2B8B1B5A006B2CC2 /* VisionKeyboardKit */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */;
+			productName = VisionKeyboardKit;
+		};
+		CE89CB0F2B8B1B6A006B2CC2 /* VisionKeyboardKit */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */;
+			productName = VisionKeyboardKit;
+		};
+		CE89CB112B8B1B7A006B2CC2 /* VisionKeyboardKit */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = CE89CB0C2B8B1B49006B2CC2 /* XCRemoteSwiftPackageReference "VisionKeyboardKit" */;
+			productName = VisionKeyboardKit;
+		};
 		CE93759824BB821F0074066F /* IQKeyboardManagerSwift */ = {
 			isa = XCSwiftPackageProductDependency;
 			package = CE93759724BB821F0074066F /* XCRemoteSwiftPackageReference "IQKeyboardManager" */;

UTM.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -108,6 +108,15 @@
         "version" : "1.0.3"
       }
     },
+    {
+      "identity" : "visionkeyboardkit",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/utmapp/VisionKeyboardKit.git",
+      "state" : {
+        "branch" : "main",
+        "revision" : "0804e4d64267acc8d08fb23160f5b6ac6134414f"
+      }
+    },
     {
       "identity" : "zipfoundation",
       "kind" : "remoteSourceControl",