scripting: add input automation suite

Resolves #6462

Scripting/UTM.sdef

@@ -787,4 +787,68 @@
                </parameter>
            </command>
     </suite>
+
+    <suite name="UTM Input Automation Suite" code="UTMi" description="UTM virtual machine input automation suite. Only supported on QEMU backend.">
+        <access-group identifier="com.utmapp.UTM.vm-access" />
+
+        <class-extension extends="virtual machine" description="Input automation.">
+            <responds-to command="input scan code">
+              <cocoa method="sendScanCode:"/>
+            </responds-to>
+            <responds-to command="input keystroke">
+              <cocoa method="sendKeystroke:"/>
+            </responds-to>
+            <responds-to command="input mouse click">
+              <cocoa method="sendMouseClick:"/>
+            </responds-to>
+        </class-extension>
+
+        <command name="input scan code" code="UTMiInSc" description="Send raw PC AT scan codes. Only supported on QEMU backend.">
+          <direct-parameter description="Virtual machine to send scan code to." type="virtual machine"/>
+          <parameter name="codes" code="ScCd" description="List of PC AT scan codes.">
+            <type type="integer" list="yes"/>
+            <cocoa key="codes"/>
+          </parameter>
+        </command>
+
+        <enumeration name="modifier key" code="MoKy" description="Modifier key to send with keystroke.">
+            <enumerator name="caps lock" code="MoCl" description="Caps Lock (⇪)"/>
+            <enumerator name="shift" code="MoSh" description="Shift (⇧)"/>
+            <enumerator name="control" code="MoCt" description="Control (⌃)"/>
+            <enumerator name="option" code="MoOp" description="Option (⌥)"/>
+            <enumerator name="command" code="MoCm" description="Command (⌘)"/>
+            <enumerator name="escape" code="MoEs" description="Escape (⎋)"/>
+        </enumeration>
+
+        <command name="input keystroke" code="UTMiInKs" description="Send text as a sequence of keystrokes to the virtual machine. Only supported on QEMU backend.">
+          <direct-parameter description="Virtual machine to send keystrokes to." type="virtual machine"/>
+          <parameter name="text" code="InTx" description="ASCII characters to send as a sequence." type="text">
+            <cocoa key="keystrokes"/>
+          </parameter>
+          <parameter name="with modifiers" code="KsMd" description="List of modifier keys to hold down while sending key sequence." optional="yes">
+            <type type="modifier key" list="yes"/>
+            <cocoa key="modifiers"/>
+          </parameter>
+        </command>
+
+        <enumeration name="mouse button" code="MsBt" description="Mouse button.">
+            <enumerator name="left" code="MsLf" description="Left Click"/>
+            <enumerator name="right" code="MsRt" description="Right Click"/>
+            <enumerator name="middle" code="MsMd" description="Middle Click"/>
+        </enumeration>
+
+        <command name="input mouse click" code="UTMiInMc" description="Send a mouse position and click to the virtual machine. Only supported on QEMU backend.">
+          <direct-parameter description="Virtual machine to send scan code to." type="virtual machine"/>
+          <parameter name="at" code="McAt" description="X-Y coordinate of the absolute position on screen to perform the click. Must be a list of two numbers.">
+            <type type="integer" list="yes"/>
+            <cocoa key="coordinate"/>
+          </parameter>
+          <parameter name="to" code="McTo" description="Which monitor to target (starting at 1). If omitted, the first monitor will be used." type="integer" optional="yes">
+            <cocoa key="monitor"/>
+          </parameter>
+          <parameter name="with mouse button" code="McBt" description="Mouse button to click. If omitted, a left click will be used." type="mouse button" optional="yes">
+            <cocoa key="button"/>
+          </parameter>
+        </command>
+    </suite>
 </dictionary>

Scripting/UTMScripting.swift

@@ -144,6 +144,23 @@ import ScriptingBridge
     case bridged = 0x42724764 /* 'BrGd' */
 }
 
+// MARK: UTMScriptingModifierKey
+@objc public enum UTMScriptingModifierKey : AEKeyword {
+    case capsLock = 0x4d6f436c /* 'MoCl' */
+    case shift = 0x4d6f5368 /* 'MoSh' */
+    case control = 0x4d6f4374 /* 'MoCt' */
+    case option = 0x4d6f4f70 /* 'MoOp' */
+    case command = 0x4d6f436d /* 'MoCm' */
+    case escape = 0x4d6f4573 /* 'MoEs' */
+}
+
+// MARK: UTMScriptingMouseButton
+@objc public enum UTMScriptingMouseButton : AEKeyword {
+    case left = 0x4d734c66 /* 'MsLf' */
+    case right = 0x4d735274 /* 'MsRt' */
+    case middle = 0x4d734d64 /* 'MsMd' */
+}
+
 // MARK: UTMScriptingGenericMethods
 @objc public protocol UTMScriptingGenericMethods {
     @objc optional func closeSaving(_ saving: UTMScriptingSaveOptions, savingIn: URL!) // Close a document.
@@ -164,7 +181,6 @@ import ScriptingBridge
     @objc optional func `open`(_ x: Any!) -> Any // Open a document.
     @objc optional func print(_ x: Any!, withProperties: [AnyHashable : Any]!, printDialog: Bool) // Print a document.
     @objc optional func quitSaving(_ saving: UTMScriptingSaveOptions) // Quit the application.
-    @objc optional func exists(_ x: Any!) -> Bool // Verify that an object exists.
     @objc optional func importNew(_ new_: NSNumber!, from: URL!) -> SBObject // Import a new virtual machine from a file.
     @objc optional func virtualMachines() -> SBElementArray
     @objc optional var autoTerminate: Bool { get } // Auto terminate the application when all windows are closed?
@@ -220,10 +236,15 @@ extension SBObject: UTMScriptingWindow {}
     @objc optional func executeAt(_ at: String!, withArguments: [String]!, withEnvironment: [String]!, usingInput: String!, base64Encoding: Bool, outputCapturing: Bool) -> UTMScriptingGuestProcess // Execute a command or script on the guest.
     @objc optional func queryIp() -> [Any] // Query the guest for all IP addresses on its network interfaces (excluding loopback).
     @objc optional func updateConfigurationWith(_ with: Any!) // Update the configuration of the virtual machine. The VM must be in the stopped state.
+    @objc optional func updateRegistryWith(_ with: [URL]!) // Update the registry of the virtual machine.
+    @objc optional func inputScanCodeCodes(_ codes: [NSNumber]!) // Send raw PC AT scan codes. Only supported on QEMU backend.
+    @objc optional func inputKeystrokeText(_ text: String!, withModifiers: [NSAppleEventDescriptor]!) // Send text as a sequence of keystrokes to the virtual machine. Only supported on QEMU backend.
+    @objc optional func inputMouseClickAt(_ at: [NSNumber]!, to: Int, withMouseButton: UTMScriptingMouseButton) // Send a mouse position and click to the virtual machine. Only supported on QEMU backend.
     @objc optional func guestFiles() -> SBElementArray
     @objc optional func guestProcesses() -> SBElementArray
     @objc optional var configuration: Any { get } // The configuration of the virtual machine.
     @objc optional func usbDevices() -> SBElementArray
+    @objc optional var registry: [URL] { get } // The registry of the virtual machine.
 }
 extension SBObject: UTMScriptingVirtualMachine {}
 

Scripting/UTMScriptingInputImpl.swift

@@ -0,0 +1,138 @@
+//
+// Copyright © 2025 Turing Software, LLC. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import Foundation
+import CocoaSpice
+
+private let kDelayNs: UInt64 = 20000000
+
+@objc extension UTMScriptingVirtualMachineImpl {
+    @nonobjc private var primaryInput: CSInput {
+        get throws {
+            guard vm.state == .started else {
+                throw ScriptingError.notRunning
+            }
+            guard let ioService = (vm as? any UTMSpiceVirtualMachine)?.ioService else {
+                throw ScriptingError.operationNotSupported
+            }
+            guard let input = ioService.primaryInput else {
+                throw ScriptingError.operationNotAvailable
+            }
+            return input
+        }
+    }
+
+    @objc func sendScanCode(_ command: NSScriptCommand) {
+        let scanCodes = command.evaluatedArguments?["codes"] as? [Int]
+        withScriptCommand(command) { [self] in
+            guard let scanCodes = scanCodes else {
+                throw ScriptingError.invalidParameter
+            }
+            let input = try self.primaryInput
+            for scanCode in scanCodes {
+                var _scanCode = scanCode
+                if (_scanCode & 0xFF00) == 0xE000 {
+                    _scanCode = 0x100 | (_scanCode & 0xFF)
+                }
+                if (_scanCode & 0x80) == 0x80 {
+                    input.send(.release, code: Int32(_scanCode & 0x17F))
+                } else {
+                    input.send(.press, code: Int32(_scanCode))
+                }
+                try await Task.sleep(nanoseconds: kDelayNs)
+            }
+        }
+    }
+
+    @objc func sendKeystroke(_ command: NSScriptCommand) {
+        let keystrokes = command.evaluatedArguments?["keystrokes"] as? String
+        let _modifiers = command.evaluatedArguments?["modifiers"] as? [AEKeyword] ?? []
+        let modifiers = _modifiers.map({ UTMScriptingModifierKey(rawValue: $0)! })
+        withScriptCommand(command) { [self] in
+            func scanCodeToSpice(_ scanCode: Int) -> Int32 {
+                var keyCode = scanCode
+                if (keyCode & 0xFF00) == 0xE000 {
+                    keyCode = (keyCode & 0xFF) | 0x100
+                }
+                return Int32(keyCode)
+            }
+
+            guard let keystrokes = keystrokes else {
+                throw ScriptingError.invalidParameter
+            }
+            let input = try self.primaryInput
+            for modifier in modifiers {
+                input.send(.press, code: modifier.toSpiceKeyCode())
+                try await Task.sleep(nanoseconds: kDelayNs)
+            }
+            let keyboardMap = VMKeyboardMap()
+            await keyboardMap.mapText(keystrokes) { scanCode in
+                input.send(.release, code: scanCodeToSpice(scanCode))
+            } keyDown: { scanCode in
+                input.send(.press, code: scanCodeToSpice(scanCode))
+            }
+            try await Task.sleep(nanoseconds: kDelayNs)
+            for modifier in modifiers {
+                input.send(.release, code: modifier.toSpiceKeyCode())
+                try await Task.sleep(nanoseconds: kDelayNs)
+            }
+        }
+    }
+
+    @objc func sendMouseClick(_ command: NSScriptCommand) {
+        let coordinate = command.evaluatedArguments?["coordinate"] as? [Int]
+        let _mouseButton = command.evaluatedArguments?["button"] as? AEKeyword ?? UTMScriptingMouseButton.left.rawValue
+        let mouseButton = UTMScriptingMouseButton(rawValue: _mouseButton)!
+        let monitorNumber = command.evaluatedArguments?["monitor"] as? Int ?? 1
+        withScriptCommand(command) { [self] in
+            guard let coordinate = coordinate, coordinate.count == 2 else {
+                throw ScriptingError.invalidParameter
+            }
+            let xPosition = coordinate[0]
+            let yPosition = coordinate[1]
+            let input = try self.primaryInput
+            try await (vm as! UTMQemuVirtualMachine).changeInputTablet(true)
+            input.sendMousePosition(mouseButton.toSpiceButton(), absolutePoint: CGPoint(x: xPosition, y: yPosition), forMonitorID: monitorNumber-1)
+            try await Task.sleep(nanoseconds: kDelayNs)
+            input.sendMouseButton(mouseButton.toSpiceButton(), pressed: true)
+            try await Task.sleep(nanoseconds: kDelayNs)
+            input.sendMouseButton(mouseButton.toSpiceButton(), pressed: false)
+        }
+    }
+}
+
+private extension UTMScriptingModifierKey {
+    func toSpiceKeyCode() -> Int32 {
+        switch self {
+        case .capsLock: return 0x3a
+        case .shift: return 0x2a
+        case .control: return 0x1d
+        case .option: return 0x38
+        case .command: return 0x15b
+        case .escape: return 0x01
+        }
+    }
+}
+
+private extension UTMScriptingMouseButton {
+    func toSpiceButton() -> CSInputButton {
+        switch self {
+        case .left: return .left
+        case .right: return .right
+        case .middle: return .middle
+        }
+    }
+}

UTM.xcodeproj/project.pbxproj

@@ -651,6 +651,7 @@
 		CE65BABF26A4D8DD0001BD6B /* VMConfigDisplayConsoleView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8401FDA5269D44E400265F0D /* VMConfigDisplayConsoleView.swift */; };
 		CE65BAC026A4D8DE0001BD6B /* VMConfigDisplayConsoleView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8401FDA5269D44E400265F0D /* VMConfigDisplayConsoleView.swift */; };
 		CE6804802E493D71001671E9 /* UTMUSBManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE68047D2E493D71001671E9 /* UTMUSBManager.swift */; };
+		CE6804852E4E5D84001671E9 /* UTMScriptingInputImpl.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE6804842E4E5D84001671E9 /* UTMScriptingInputImpl.swift */; };
 		CE68E5442E3912E0006B3645 /* VMKeyboardShortcutsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE68E5422E3912E0006B3645 /* VMKeyboardShortcutsView.swift */; };
 		CE68E5452E3912E0006B3645 /* VMKeyboardShortcutsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE68E5422E3912E0006B3645 /* VMKeyboardShortcutsView.swift */; };
 		CE68E5482E3C3E0A006B3645 /* VMWizardOSClassicMacView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CE68E5472E3C3E0A006B3645 /* VMWizardOSClassicMacView.swift */; };
@@ -1984,6 +1985,7 @@
 		CE612AC524D3B50700FA6300 /* VMDisplayWindowController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMDisplayWindowController.swift; sourceTree = "<group>"; };
 		CE66450C2269313200B0849A /* MetalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = MetalKit.framework; path = System/Library/Frameworks/MetalKit.framework; sourceTree = SDKROOT; };
 		CE68047D2E493D71001671E9 /* UTMUSBManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UTMUSBManager.swift; sourceTree = "<group>"; };
+		CE6804842E4E5D84001671E9 /* UTMScriptingInputImpl.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UTMScriptingInputImpl.swift; sourceTree = "<group>"; };
 		CE68E5422E3912E0006B3645 /* VMKeyboardShortcutsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMKeyboardShortcutsView.swift; sourceTree = "<group>"; };
 		CE68E5472E3C3E0A006B3645 /* VMWizardOSClassicMacView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VMWizardOSClassicMacView.swift; sourceTree = "<group>"; };
 		CE6B240A25F1F3CE0020D43E /* main.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = main.c; sourceTree = "<group>"; };
@@ -3114,6 +3116,7 @@
 				CEC794B9294924E300121A9F /* UTMScriptingSerialPortImpl.swift */,
 				CE25124829BFDBA6000790AB /* UTMScriptingGuestFileImpl.swift */,
 				CE25124629BFDB87000790AB /* UTMScriptingGuestProcessImpl.swift */,
+				CE6804842E4E5D84001671E9 /* UTMScriptingInputImpl.swift */,
 				CD84C2082D3B446D00829850 /* UTMScriptingRegistryEntryImpl.swift */,
 				CE25124C29C55816000790AB /* UTMScriptingConfigImpl.swift */,
 				CE25125429C80CD4000790AB /* UTMScriptingCreateCommand.swift */,
@@ -3910,6 +3913,7 @@
 				848A98B8286A1589006F0550 /* UTMAppleConfigurationDisplay.swift in Sources */,
 				848A98BE286A1B62006F0550 /* UTMAppleConfigurationDrive.swift in Sources */,
 				CE93758924B930270074066F /* BusyOverlay.swift in Sources */,
+				CE6804852E4E5D84001671E9 /* UTMScriptingInputImpl.swift in Sources */,
 				848A98CA28720CFC006F0550 /* VMConfigAppleSerialView.swift in Sources */,
 				848D99C22866D9CE0055C215 /* QEMUArgumentBuilder.swift in Sources */,
 				8401FDA0269D266E00265F0D /* VMConfigAppleBootView.swift in Sources */,