Add ez-assistant and kerberos service folders

This commit is contained in:
kelin
2026-02-11 14:56:03 -05:00
parent e4e8ae1b87
commit 9ccfb36923
4471 changed files with 746463 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
parent_config: ../../.swiftlint.yml
included:
- Sources
- ../shared/ClawdisNodeKit/Sources

View File

@@ -0,0 +1,28 @@
# Clawdbot (iOS)
Internal-only SwiftUI app scaffold.
## Lint/format (required)
```bash
brew install swiftformat swiftlint
```
## Generate the Xcode project
```bash
cd apps/ios
xcodegen generate
open Clawdbot.xcodeproj
```
## Shared packages
- `../shared/MoltbotKit` — shared types/constants used by iOS (and later macOS bridge + gateway routing).
## fastlane
```bash
brew install fastlane
cd apps/ios
fastlane lanes
```
See `apps/ios/fastlane/SETUP.md` for App Store Connect auth + upload lanes.

View File

@@ -0,0 +1,31 @@
{
"images" : [
{ "filename" : "icon-20@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "20x20" },
{ "filename" : "icon-20@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "20x20" },
{ "filename" : "icon-20@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "20x20" },
{ "filename" : "icon-20@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "20x20" },
{ "filename" : "icon-29@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "29x29" },
{ "filename" : "icon-29@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "29x29" },
{ "filename" : "icon-29@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "29x29" },
{ "filename" : "icon-29@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "29x29" },
{ "filename" : "icon-40@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "40x40" },
{ "filename" : "icon-40@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "40x40" },
{ "filename" : "icon-40@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "40x40" },
{ "filename" : "icon-40@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "40x40" },
{ "filename" : "icon-60@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "60x60" },
{ "filename" : "icon-60@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "60x60" },
{ "filename" : "icon-76@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "76x76" },
{ "filename" : "icon-83.5@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "83.5x83.5" },
{ "filename" : "icon-1024.png", "idiom" : "ios-marketing", "scale" : "1x", "size" : "1024x1024" }
],
"info" : {
"author" : "xcode",
"version" : 1
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

View File

@@ -0,0 +1,6 @@
{
"info" : {
"author" : "xcode",
"version" : 1
}
}

View File

@@ -0,0 +1,406 @@
import AVFoundation
import MoltbotKit
import Foundation
actor CameraController {
struct CameraDeviceInfo: Codable, Sendable {
var id: String
var name: String
var position: String
var deviceType: String
}
enum CameraError: LocalizedError, Sendable {
case cameraUnavailable
case microphoneUnavailable
case permissionDenied(kind: String)
case invalidParams(String)
case captureFailed(String)
case exportFailed(String)
var errorDescription: String? {
switch self {
case .cameraUnavailable:
"Camera unavailable"
case .microphoneUnavailable:
"Microphone unavailable"
case let .permissionDenied(kind):
"\(kind) permission denied"
case let .invalidParams(msg):
msg
case let .captureFailed(msg):
msg
case let .exportFailed(msg):
msg
}
}
}
func snap(params: MoltbotCameraSnapParams) async throws -> (
format: String,
base64: String,
width: Int,
height: Int)
{
let facing = params.facing ?? .front
let format = params.format ?? .jpg
// Default to a reasonable max width to keep gateway payload sizes manageable.
// If you need the full-res photo, explicitly request a larger maxWidth.
let maxWidth = params.maxWidth.flatMap { $0 > 0 ? $0 : nil } ?? 1600
let quality = Self.clampQuality(params.quality)
let delayMs = max(0, params.delayMs ?? 0)
try await self.ensureAccess(for: .video)
let session = AVCaptureSession()
session.sessionPreset = .photo
guard let device = Self.pickCamera(facing: facing, deviceId: params.deviceId) else {
throw CameraError.cameraUnavailable
}
let input = try AVCaptureDeviceInput(device: device)
guard session.canAddInput(input) else {
throw CameraError.captureFailed("Failed to add camera input")
}
session.addInput(input)
let output = AVCapturePhotoOutput()
guard session.canAddOutput(output) else {
throw CameraError.captureFailed("Failed to add photo output")
}
session.addOutput(output)
output.maxPhotoQualityPrioritization = .quality
session.startRunning()
defer { session.stopRunning() }
await Self.warmUpCaptureSession()
await Self.sleepDelayMs(delayMs)
let settings: AVCapturePhotoSettings = {
if output.availablePhotoCodecTypes.contains(.jpeg) {
return AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg])
}
return AVCapturePhotoSettings()
}()
settings.photoQualityPrioritization = .quality
var delegate: PhotoCaptureDelegate?
let rawData: Data = try await withCheckedThrowingContinuation { cont in
let d = PhotoCaptureDelegate(cont)
delegate = d
output.capturePhoto(with: settings, delegate: d)
}
withExtendedLifetime(delegate) {}
let maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
let res = try JPEGTranscoder.transcodeToJPEG(
imageData: rawData,
maxWidthPx: maxWidth,
quality: quality,
maxBytes: maxEncodedBytes)
return (
format: format.rawValue,
base64: res.data.base64EncodedString(),
width: res.widthPx,
height: res.heightPx)
}
func clip(params: MoltbotCameraClipParams) async throws -> (
format: String,
base64: String,
durationMs: Int,
hasAudio: Bool)
{
let facing = params.facing ?? .front
let durationMs = Self.clampDurationMs(params.durationMs)
let includeAudio = params.includeAudio ?? true
let format = params.format ?? .mp4
try await self.ensureAccess(for: .video)
if includeAudio {
try await self.ensureAccess(for: .audio)
}
let session = AVCaptureSession()
session.sessionPreset = .high
guard let camera = Self.pickCamera(facing: facing, deviceId: params.deviceId) else {
throw CameraError.cameraUnavailable
}
let cameraInput = try AVCaptureDeviceInput(device: camera)
guard session.canAddInput(cameraInput) else {
throw CameraError.captureFailed("Failed to add camera input")
}
session.addInput(cameraInput)
if includeAudio {
guard let mic = AVCaptureDevice.default(for: .audio) else {
throw CameraError.microphoneUnavailable
}
let micInput = try AVCaptureDeviceInput(device: mic)
if session.canAddInput(micInput) {
session.addInput(micInput)
} else {
throw CameraError.captureFailed("Failed to add microphone input")
}
}
let output = AVCaptureMovieFileOutput()
guard session.canAddOutput(output) else {
throw CameraError.captureFailed("Failed to add movie output")
}
session.addOutput(output)
output.maxRecordedDuration = CMTime(value: Int64(durationMs), timescale: 1000)
session.startRunning()
defer { session.stopRunning() }
await Self.warmUpCaptureSession()
let movURL = FileManager().temporaryDirectory
.appendingPathComponent("moltbot-camera-\(UUID().uuidString).mov")
let mp4URL = FileManager().temporaryDirectory
.appendingPathComponent("moltbot-camera-\(UUID().uuidString).mp4")
defer {
try? FileManager().removeItem(at: movURL)
try? FileManager().removeItem(at: mp4URL)
}
var delegate: MovieFileDelegate?
let recordedURL: URL = try await withCheckedThrowingContinuation { cont in
let d = MovieFileDelegate(cont)
delegate = d
output.startRecording(to: movURL, recordingDelegate: d)
}
withExtendedLifetime(delegate) {}
// Transcode .mov -> .mp4 for easier downstream handling.
try await Self.exportToMP4(inputURL: recordedURL, outputURL: mp4URL)
let data = try Data(contentsOf: mp4URL)
return (
format: format.rawValue,
base64: data.base64EncodedString(),
durationMs: durationMs,
hasAudio: includeAudio)
}
func listDevices() -> [CameraDeviceInfo] {
return Self.discoverVideoDevices().map { device in
CameraDeviceInfo(
id: device.uniqueID,
name: device.localizedName,
position: Self.positionLabel(device.position),
deviceType: device.deviceType.rawValue)
}
}
private func ensureAccess(for mediaType: AVMediaType) async throws {
let status = AVCaptureDevice.authorizationStatus(for: mediaType)
switch status {
case .authorized:
return
case .notDetermined:
let ok = await withCheckedContinuation(isolation: nil) { cont in
AVCaptureDevice.requestAccess(for: mediaType) { granted in
cont.resume(returning: granted)
}
}
if !ok {
throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone")
}
case .denied, .restricted:
throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone")
@unknown default:
throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone")
}
}
private nonisolated static func pickCamera(
facing: MoltbotCameraFacing,
deviceId: String?) -> AVCaptureDevice?
{
if let deviceId, !deviceId.isEmpty {
if let match = Self.discoverVideoDevices().first(where: { $0.uniqueID == deviceId }) {
return match
}
}
let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back
if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) {
return device
}
// Fall back to any default camera (e.g. simulator / unusual device configurations).
return AVCaptureDevice.default(for: .video)
}
private nonisolated static func positionLabel(_ position: AVCaptureDevice.Position) -> String {
switch position {
case .front: "front"
case .back: "back"
default: "unspecified"
}
}
private nonisolated static func discoverVideoDevices() -> [AVCaptureDevice] {
let types: [AVCaptureDevice.DeviceType] = [
.builtInWideAngleCamera,
.builtInUltraWideCamera,
.builtInTelephotoCamera,
.builtInDualCamera,
.builtInDualWideCamera,
.builtInTripleCamera,
.builtInTrueDepthCamera,
.builtInLiDARDepthCamera,
]
let session = AVCaptureDevice.DiscoverySession(
deviceTypes: types,
mediaType: .video,
position: .unspecified)
return session.devices
}
nonisolated static func clampQuality(_ quality: Double?) -> Double {
let q = quality ?? 0.9
return min(1.0, max(0.05, q))
}
nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
let v = ms ?? 3000
// Keep clips short by default; avoid huge base64 payloads on the gateway.
return min(60000, max(250, v))
}
private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws {
let asset = AVURLAsset(url: inputURL)
guard let exporter = AVAssetExportSession(asset: asset, presetName: AVAssetExportPresetMediumQuality) else {
throw CameraError.exportFailed("Failed to create export session")
}
exporter.shouldOptimizeForNetworkUse = true
if #available(iOS 18.0, tvOS 18.0, visionOS 2.0, *) {
do {
try await exporter.export(to: outputURL, as: .mp4)
return
} catch {
throw CameraError.exportFailed(error.localizedDescription)
}
} else {
exporter.outputURL = outputURL
exporter.outputFileType = .mp4
try await withCheckedThrowingContinuation(isolation: nil) { (cont: CheckedContinuation<Void, Error>) in
exporter.exportAsynchronously {
cont.resume(returning: ())
}
}
switch exporter.status {
case .completed:
return
case .failed:
throw CameraError.exportFailed(exporter.error?.localizedDescription ?? "export failed")
case .cancelled:
throw CameraError.exportFailed("export cancelled")
default:
throw CameraError.exportFailed("export did not complete")
}
}
}
private nonisolated static func warmUpCaptureSession() async {
// A short delay after `startRunning()` significantly reduces "blank first frame" captures on some devices.
try? await Task.sleep(nanoseconds: 150_000_000) // 150ms
}
private nonisolated static func sleepDelayMs(_ delayMs: Int) async {
guard delayMs > 0 else { return }
let maxDelayMs = 10 * 1000
let ns = UInt64(min(delayMs, maxDelayMs)) * UInt64(NSEC_PER_MSEC)
try? await Task.sleep(nanoseconds: ns)
}
}
private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate {
private let continuation: CheckedContinuation<Data, Error>
private var didResume = false
init(_ continuation: CheckedContinuation<Data, Error>) {
self.continuation = continuation
}
func photoOutput(
_ output: AVCapturePhotoOutput,
didFinishProcessingPhoto photo: AVCapturePhoto,
error: Error?)
{
guard !self.didResume else { return }
self.didResume = true
if let error {
self.continuation.resume(throwing: error)
return
}
guard let data = photo.fileDataRepresentation() else {
self.continuation.resume(
throwing: NSError(domain: "Camera", code: 1, userInfo: [
NSLocalizedDescriptionKey: "photo data missing",
]))
return
}
if data.isEmpty {
self.continuation.resume(
throwing: NSError(domain: "Camera", code: 2, userInfo: [
NSLocalizedDescriptionKey: "photo data empty",
]))
return
}
self.continuation.resume(returning: data)
}
func photoOutput(
_ output: AVCapturePhotoOutput,
didFinishCaptureFor resolvedSettings: AVCaptureResolvedPhotoSettings,
error: Error?)
{
guard let error else { return }
guard !self.didResume else { return }
self.didResume = true
self.continuation.resume(throwing: error)
}
}
private final class MovieFileDelegate: NSObject, AVCaptureFileOutputRecordingDelegate {
private let continuation: CheckedContinuation<URL, Error>
private var didResume = false
init(_ continuation: CheckedContinuation<URL, Error>) {
self.continuation = continuation
}
func fileOutput(
_ output: AVCaptureFileOutput,
didFinishRecordingTo outputFileURL: URL,
from connections: [AVCaptureConnection],
error: Error?)
{
guard !self.didResume else { return }
self.didResume = true
if let error {
let ns = error as NSError
if ns.domain == AVFoundationErrorDomain,
ns.code == AVError.maximumDurationReached.rawValue
{
self.continuation.resume(returning: outputFileURL)
return
}
self.continuation.resume(throwing: error)
return
}
self.continuation.resume(returning: outputFileURL)
}
}

View File

@@ -0,0 +1,39 @@
import MoltbotChatUI
import MoltbotKit
import SwiftUI
struct ChatSheet: View {
@Environment(\.dismiss) private var dismiss
@State private var viewModel: MoltbotChatViewModel
private let userAccent: Color?
init(gateway: GatewayNodeSession, sessionKey: String, userAccent: Color? = nil) {
let transport = IOSGatewayChatTransport(gateway: gateway)
self._viewModel = State(
initialValue: MoltbotChatViewModel(
sessionKey: sessionKey,
transport: transport))
self.userAccent = userAccent
}
var body: some View {
NavigationStack {
MoltbotChatView(
viewModel: self.viewModel,
showsSessionSwitcher: true,
userAccent: self.userAccent)
.navigationTitle("Chat")
.navigationBarTitleDisplayMode(.inline)
.toolbar {
ToolbarItem(placement: .topBarTrailing) {
Button {
self.dismiss()
} label: {
Image(systemName: "xmark")
}
.accessibilityLabel("Close")
}
}
}
}
}

View File

@@ -0,0 +1,129 @@
import MoltbotChatUI
import MoltbotKit
import MoltbotProtocol
import Foundation
struct IOSGatewayChatTransport: MoltbotChatTransport, Sendable {
private let gateway: GatewayNodeSession
init(gateway: GatewayNodeSession) {
self.gateway = gateway
}
func abortRun(sessionKey: String, runId: String) async throws {
struct Params: Codable {
var sessionKey: String
var runId: String
}
let data = try JSONEncoder().encode(Params(sessionKey: sessionKey, runId: runId))
let json = String(data: data, encoding: .utf8)
_ = try await self.gateway.request(method: "chat.abort", paramsJSON: json, timeoutSeconds: 10)
}
func listSessions(limit: Int?) async throws -> MoltbotChatSessionsListResponse {
struct Params: Codable {
var includeGlobal: Bool
var includeUnknown: Bool
var limit: Int?
}
let data = try JSONEncoder().encode(Params(includeGlobal: true, includeUnknown: false, limit: limit))
let json = String(data: data, encoding: .utf8)
let res = try await self.gateway.request(method: "sessions.list", paramsJSON: json, timeoutSeconds: 15)
return try JSONDecoder().decode(MoltbotChatSessionsListResponse.self, from: res)
}
func setActiveSessionKey(_ sessionKey: String) async throws {
struct Subscribe: Codable { var sessionKey: String }
let data = try JSONEncoder().encode(Subscribe(sessionKey: sessionKey))
let json = String(data: data, encoding: .utf8)
await self.gateway.sendEvent(event: "chat.subscribe", payloadJSON: json)
}
func requestHistory(sessionKey: String) async throws -> MoltbotChatHistoryPayload {
struct Params: Codable { var sessionKey: String }
let data = try JSONEncoder().encode(Params(sessionKey: sessionKey))
let json = String(data: data, encoding: .utf8)
let res = try await self.gateway.request(method: "chat.history", paramsJSON: json, timeoutSeconds: 15)
return try JSONDecoder().decode(MoltbotChatHistoryPayload.self, from: res)
}
func sendMessage(
sessionKey: String,
message: String,
thinking: String,
idempotencyKey: String,
attachments: [MoltbotChatAttachmentPayload]) async throws -> MoltbotChatSendResponse
{
struct Params: Codable {
var sessionKey: String
var message: String
var thinking: String
var attachments: [MoltbotChatAttachmentPayload]?
var timeoutMs: Int
var idempotencyKey: String
}
let params = Params(
sessionKey: sessionKey,
message: message,
thinking: thinking,
attachments: attachments.isEmpty ? nil : attachments,
timeoutMs: 30000,
idempotencyKey: idempotencyKey)
let data = try JSONEncoder().encode(params)
let json = String(data: data, encoding: .utf8)
let res = try await self.gateway.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 35)
return try JSONDecoder().decode(MoltbotChatSendResponse.self, from: res)
}
func requestHealth(timeoutMs: Int) async throws -> Bool {
let seconds = max(1, Int(ceil(Double(timeoutMs) / 1000.0)))
let res = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: seconds)
return (try? JSONDecoder().decode(MoltbotGatewayHealthOK.self, from: res))?.ok ?? true
}
func events() -> AsyncStream<MoltbotChatTransportEvent> {
AsyncStream { continuation in
let task = Task {
let stream = await self.gateway.subscribeServerEvents()
for await evt in stream {
if Task.isCancelled { return }
switch evt.event {
case "tick":
continuation.yield(.tick)
case "seqGap":
continuation.yield(.seqGap)
case "health":
guard let payload = evt.payload else { break }
let ok = (try? GatewayPayloadDecoding.decode(
payload,
as: MoltbotGatewayHealthOK.self))?.ok ?? true
continuation.yield(.health(ok: ok))
case "chat":
guard let payload = evt.payload else { break }
if let chatPayload = try? GatewayPayloadDecoding.decode(
payload,
as: MoltbotChatEventPayload.self)
{
continuation.yield(.chat(chatPayload))
}
case "agent":
guard let payload = evt.payload else { break }
if let agentPayload = try? GatewayPayloadDecoding.decode(
payload,
as: MoltbotAgentEventPayload.self)
{
continuation.yield(.agent(agentPayload))
}
default:
break
}
}
}
continuation.onTermination = { @Sendable _ in
task.cancel()
}
}
}
}

View File

@@ -0,0 +1,31 @@
import SwiftUI
@main
struct MoltbotApp: App {
@State private var appModel: NodeAppModel
@State private var gatewayController: GatewayConnectionController
@Environment(\.scenePhase) private var scenePhase
init() {
GatewaySettingsStore.bootstrapPersistence()
let appModel = NodeAppModel()
_appModel = State(initialValue: appModel)
_gatewayController = State(initialValue: GatewayConnectionController(appModel: appModel))
}
var body: some Scene {
WindowGroup {
RootCanvas()
.environment(self.appModel)
.environment(self.appModel.voiceWake)
.environment(self.gatewayController)
.onOpenURL { url in
Task { await self.appModel.handleDeepLink(url: url) }
}
.onChange(of: self.scenePhase) { _, newValue in
self.appModel.setScenePhase(newValue)
self.gatewayController.setScenePhase(newValue)
}
}
}
}

View File

@@ -0,0 +1,434 @@
import MoltbotKit
import Darwin
import Foundation
import Network
import Observation
import SwiftUI
import UIKit
@MainActor
@Observable
final class GatewayConnectionController {
private(set) var gateways: [GatewayDiscoveryModel.DiscoveredGateway] = []
private(set) var discoveryStatusText: String = "Idle"
private(set) var discoveryDebugLog: [GatewayDiscoveryModel.DebugLogEntry] = []
private let discovery = GatewayDiscoveryModel()
private weak var appModel: NodeAppModel?
private var didAutoConnect = false
init(appModel: NodeAppModel, startDiscovery: Bool = true) {
self.appModel = appModel
GatewaySettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard
self.discovery.setDebugLoggingEnabled(defaults.bool(forKey: "gateway.discovery.debugLogs"))
self.updateFromDiscovery()
self.observeDiscovery()
if startDiscovery {
self.discovery.start()
}
}
func setDiscoveryDebugLoggingEnabled(_ enabled: Bool) {
self.discovery.setDebugLoggingEnabled(enabled)
}
func setScenePhase(_ phase: ScenePhase) {
switch phase {
case .background:
self.discovery.stop()
case .active, .inactive:
self.discovery.start()
@unknown default:
self.discovery.start()
}
}
func connect(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) async {
let instanceId = UserDefaults.standard.string(forKey: "node.instanceId")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId)
let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId)
guard let host = self.resolveGatewayHost(gateway) else { return }
let port = gateway.gatewayPort ?? 18789
let tlsParams = self.resolveDiscoveredTLSParams(gateway: gateway)
guard let url = self.buildGatewayURL(
host: host,
port: port,
useTLS: tlsParams?.required == true)
else { return }
self.didAutoConnect = true
self.startAutoConnect(
url: url,
gatewayStableID: gateway.stableID,
tls: tlsParams,
token: token,
password: password)
}
func connectManual(host: String, port: Int, useTLS: Bool) async {
let instanceId = UserDefaults.standard.string(forKey: "node.instanceId")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId)
let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId)
let stableID = self.manualStableID(host: host, port: port)
let tlsParams = self.resolveManualTLSParams(stableID: stableID, tlsEnabled: useTLS)
guard let url = self.buildGatewayURL(
host: host,
port: port,
useTLS: tlsParams?.required == true)
else { return }
self.didAutoConnect = true
self.startAutoConnect(
url: url,
gatewayStableID: stableID,
tls: tlsParams,
token: token,
password: password)
}
private func updateFromDiscovery() {
let newGateways = self.discovery.gateways
self.gateways = newGateways
self.discoveryStatusText = self.discovery.statusText
self.discoveryDebugLog = self.discovery.debugLog
self.updateLastDiscoveredGateway(from: newGateways)
self.maybeAutoConnect()
}
private func observeDiscovery() {
withObservationTracking {
_ = self.discovery.gateways
_ = self.discovery.statusText
_ = self.discovery.debugLog
} onChange: { [weak self] in
Task { @MainActor in
guard let self else { return }
self.updateFromDiscovery()
self.observeDiscovery()
}
}
}
private func maybeAutoConnect() {
guard !self.didAutoConnect else { return }
guard let appModel = self.appModel else { return }
guard appModel.gatewayServerName == nil else { return }
let defaults = UserDefaults.standard
let manualEnabled = defaults.bool(forKey: "gateway.manual.enabled")
let instanceId = defaults.string(forKey: "node.instanceId")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !instanceId.isEmpty else { return }
let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId)
let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId)
if manualEnabled {
let manualHost = defaults.string(forKey: "gateway.manual.host")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard !manualHost.isEmpty else { return }
let manualPort = defaults.integer(forKey: "gateway.manual.port")
let resolvedPort = manualPort > 0 ? manualPort : 18789
let manualTLS = defaults.bool(forKey: "gateway.manual.tls")
let stableID = self.manualStableID(host: manualHost, port: resolvedPort)
let tlsParams = self.resolveManualTLSParams(stableID: stableID, tlsEnabled: manualTLS)
guard let url = self.buildGatewayURL(
host: manualHost,
port: resolvedPort,
useTLS: tlsParams?.required == true)
else { return }
self.didAutoConnect = true
self.startAutoConnect(
url: url,
gatewayStableID: stableID,
tls: tlsParams,
token: token,
password: password)
return
}
let preferredStableID = defaults.string(forKey: "gateway.preferredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let lastDiscoveredStableID = defaults.string(forKey: "gateway.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let candidates = [preferredStableID, lastDiscoveredStableID].filter { !$0.isEmpty }
guard let targetStableID = candidates.first(where: { id in
self.gateways.contains(where: { $0.stableID == id })
}) else { return }
guard let target = self.gateways.first(where: { $0.stableID == targetStableID }) else { return }
guard let host = self.resolveGatewayHost(target) else { return }
let port = target.gatewayPort ?? 18789
let tlsParams = self.resolveDiscoveredTLSParams(gateway: target)
guard let url = self.buildGatewayURL(host: host, port: port, useTLS: tlsParams?.required == true)
else { return }
self.didAutoConnect = true
self.startAutoConnect(
url: url,
gatewayStableID: target.stableID,
tls: tlsParams,
token: token,
password: password)
}
private func updateLastDiscoveredGateway(from gateways: [GatewayDiscoveryModel.DiscoveredGateway]) {
let defaults = UserDefaults.standard
let preferred = defaults.string(forKey: "gateway.preferredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let existingLast = defaults.string(forKey: "gateway.lastDiscoveredStableID")?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
// Avoid overriding user intent (preferred/lastDiscovered are also set on manual Connect).
guard preferred.isEmpty, existingLast.isEmpty else { return }
guard let first = gateways.first else { return }
defaults.set(first.stableID, forKey: "gateway.lastDiscoveredStableID")
GatewaySettingsStore.saveLastDiscoveredGatewayStableID(first.stableID)
}
private func startAutoConnect(
url: URL,
gatewayStableID: String,
tls: GatewayTLSParams?,
token: String?,
password: String?)
{
guard let appModel else { return }
let connectOptions = self.makeConnectOptions()
Task { [weak self] in
guard let self else { return }
await MainActor.run {
appModel.gatewayStatusText = "Connecting…"
}
appModel.connectToGateway(
url: url,
gatewayStableID: gatewayStableID,
tls: tls,
token: token,
password: password,
connectOptions: connectOptions)
}
}
private func resolveDiscoveredTLSParams(gateway: GatewayDiscoveryModel.DiscoveredGateway) -> GatewayTLSParams? {
let stableID = gateway.stableID
let stored = GatewayTLSStore.loadFingerprint(stableID: stableID)
if gateway.tlsEnabled || gateway.tlsFingerprintSha256 != nil || stored != nil {
return GatewayTLSParams(
required: true,
expectedFingerprint: gateway.tlsFingerprintSha256 ?? stored,
allowTOFU: stored == nil,
storeKey: stableID)
}
return nil
}
private func resolveManualTLSParams(stableID: String, tlsEnabled: Bool) -> GatewayTLSParams? {
let stored = GatewayTLSStore.loadFingerprint(stableID: stableID)
if tlsEnabled || stored != nil {
return GatewayTLSParams(
required: true,
expectedFingerprint: stored,
allowTOFU: stored == nil,
storeKey: stableID)
}
return nil
}
private func resolveGatewayHost(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> String? {
if let lanHost = gateway.lanHost?.trimmingCharacters(in: .whitespacesAndNewlines), !lanHost.isEmpty {
return lanHost
}
if let tailnet = gateway.tailnetDns?.trimmingCharacters(in: .whitespacesAndNewlines), !tailnet.isEmpty {
return tailnet
}
return nil
}
private func buildGatewayURL(host: String, port: Int, useTLS: Bool) -> URL? {
let scheme = useTLS ? "wss" : "ws"
var components = URLComponents()
components.scheme = scheme
components.host = host
components.port = port
return components.url
}
private func manualStableID(host: String, port: Int) -> String {
"manual|\(host.lowercased())|\(port)"
}
private func makeConnectOptions() -> GatewayConnectOptions {
let defaults = UserDefaults.standard
let displayName = self.resolvedDisplayName(defaults: defaults)
return GatewayConnectOptions(
role: "node",
scopes: [],
caps: self.currentCaps(),
commands: self.currentCommands(),
permissions: [:],
clientId: "moltbot-ios",
clientMode: "node",
clientDisplayName: displayName)
}
private func resolvedDisplayName(defaults: UserDefaults) -> String {
let key = "node.displayName"
let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !existing.isEmpty, existing != "iOS Node" { return existing }
let deviceName = UIDevice.current.name.trimmingCharacters(in: .whitespacesAndNewlines)
let candidate = deviceName.isEmpty ? "iOS Node" : deviceName
if existing.isEmpty || existing == "iOS Node" {
defaults.set(candidate, forKey: key)
}
return candidate
}
private func currentCaps() -> [String] {
var caps = [MoltbotCapability.canvas.rawValue, MoltbotCapability.screen.rawValue]
// Default-on: if the key doesn't exist yet, treat it as enabled.
let cameraEnabled =
UserDefaults.standard.object(forKey: "camera.enabled") == nil
? true
: UserDefaults.standard.bool(forKey: "camera.enabled")
if cameraEnabled { caps.append(MoltbotCapability.camera.rawValue) }
let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey)
if voiceWakeEnabled { caps.append(MoltbotCapability.voiceWake.rawValue) }
let locationModeRaw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
let locationMode = MoltbotLocationMode(rawValue: locationModeRaw) ?? .off
if locationMode != .off { caps.append(MoltbotCapability.location.rawValue) }
return caps
}
private func currentCommands() -> [String] {
var commands: [String] = [
MoltbotCanvasCommand.present.rawValue,
MoltbotCanvasCommand.hide.rawValue,
MoltbotCanvasCommand.navigate.rawValue,
MoltbotCanvasCommand.evalJS.rawValue,
MoltbotCanvasCommand.snapshot.rawValue,
MoltbotCanvasA2UICommand.push.rawValue,
MoltbotCanvasA2UICommand.pushJSONL.rawValue,
MoltbotCanvasA2UICommand.reset.rawValue,
MoltbotScreenCommand.record.rawValue,
MoltbotSystemCommand.notify.rawValue,
MoltbotSystemCommand.which.rawValue,
MoltbotSystemCommand.run.rawValue,
MoltbotSystemCommand.execApprovalsGet.rawValue,
MoltbotSystemCommand.execApprovalsSet.rawValue,
]
let caps = Set(self.currentCaps())
if caps.contains(MoltbotCapability.camera.rawValue) {
commands.append(MoltbotCameraCommand.list.rawValue)
commands.append(MoltbotCameraCommand.snap.rawValue)
commands.append(MoltbotCameraCommand.clip.rawValue)
}
if caps.contains(MoltbotCapability.location.rawValue) {
commands.append(MoltbotLocationCommand.get.rawValue)
}
return commands
}
private func platformString() -> String {
let v = ProcessInfo.processInfo.operatingSystemVersion
let name = switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPadOS"
case .phone:
"iOS"
default:
"iOS"
}
return "\(name) \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
}
private func deviceFamily() -> String {
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPad"
case .phone:
"iPhone"
default:
"iOS"
}
}
private func modelIdentifier() -> String {
var systemInfo = utsname()
uname(&systemInfo)
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8)
}
let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? "unknown" : trimmed
}
private func appVersion() -> String {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
}
}
#if DEBUG
extension GatewayConnectionController {
func _test_resolvedDisplayName(defaults: UserDefaults) -> String {
self.resolvedDisplayName(defaults: defaults)
}
func _test_currentCaps() -> [String] {
self.currentCaps()
}
func _test_currentCommands() -> [String] {
self.currentCommands()
}
func _test_platformString() -> String {
self.platformString()
}
func _test_deviceFamily() -> String {
self.deviceFamily()
}
func _test_modelIdentifier() -> String {
self.modelIdentifier()
}
func _test_appVersion() -> String {
self.appVersion()
}
func _test_setGateways(_ gateways: [GatewayDiscoveryModel.DiscoveredGateway]) {
self.gateways = gateways
}
func _test_triggerAutoConnect() {
self.maybeAutoConnect()
}
}
#endif

View File

@@ -0,0 +1,68 @@
import SwiftUI
import UIKit
struct GatewayDiscoveryDebugLogView: View {
@Environment(GatewayConnectionController.self) private var gatewayController
@AppStorage("gateway.discovery.debugLogs") private var debugLogsEnabled: Bool = false
var body: some View {
List {
if !self.debugLogsEnabled {
Text("Enable “Discovery Debug Logs” to start collecting events.")
.foregroundStyle(.secondary)
}
if self.gatewayController.discoveryDebugLog.isEmpty {
Text("No log entries yet.")
.foregroundStyle(.secondary)
} else {
ForEach(self.gatewayController.discoveryDebugLog) { entry in
VStack(alignment: .leading, spacing: 2) {
Text(Self.formatTime(entry.ts))
.font(.caption)
.foregroundStyle(.secondary)
Text(entry.message)
.font(.callout)
.textSelection(.enabled)
}
.padding(.vertical, 4)
}
}
}
.navigationTitle("Discovery Logs")
.toolbar {
ToolbarItem(placement: .topBarTrailing) {
Button("Copy") {
UIPasteboard.general.string = self.formattedLog()
}
.disabled(self.gatewayController.discoveryDebugLog.isEmpty)
}
}
}
private func formattedLog() -> String {
self.gatewayController.discoveryDebugLog
.map { "\(Self.formatISO($0.ts)) \($0.message)" }
.joined(separator: "\n")
}
private static let timeFormatter: DateFormatter = {
let formatter = DateFormatter()
formatter.dateFormat = "HH:mm:ss"
return formatter
}()
private static let isoFormatter: ISO8601DateFormatter = {
let formatter = ISO8601DateFormatter()
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
return formatter
}()
private static func formatTime(_ date: Date) -> String {
self.timeFormatter.string(from: date)
}
private static func formatISO(_ date: Date) -> String {
self.isoFormatter.string(from: date)
}
}

View File

@@ -0,0 +1,224 @@
import MoltbotKit
import Foundation
import Network
import Observation
@MainActor
@Observable
final class GatewayDiscoveryModel {
struct DebugLogEntry: Identifiable, Equatable {
var id = UUID()
var ts: Date
var message: String
}
struct DiscoveredGateway: Identifiable, Equatable {
var id: String { self.stableID }
var name: String
var endpoint: NWEndpoint
var stableID: String
var debugID: String
var lanHost: String?
var tailnetDns: String?
var gatewayPort: Int?
var canvasPort: Int?
var tlsEnabled: Bool
var tlsFingerprintSha256: String?
var cliPath: String?
}
var gateways: [DiscoveredGateway] = []
var statusText: String = "Idle"
private(set) var debugLog: [DebugLogEntry] = []
private var browsers: [String: NWBrowser] = [:]
private var gatewaysByDomain: [String: [DiscoveredGateway]] = [:]
private var statesByDomain: [String: NWBrowser.State] = [:]
private var debugLoggingEnabled = false
private var lastStableIDs = Set<String>()
func setDebugLoggingEnabled(_ enabled: Bool) {
let wasEnabled = self.debugLoggingEnabled
self.debugLoggingEnabled = enabled
if !enabled {
self.debugLog = []
} else if !wasEnabled {
self.appendDebugLog("debug logging enabled")
self.appendDebugLog("snapshot: status=\(self.statusText) gateways=\(self.gateways.count)")
}
}
func start() {
if !self.browsers.isEmpty { return }
self.appendDebugLog("start()")
for domain in MoltbotBonjour.gatewayServiceDomains {
let params = NWParameters.tcp
params.includePeerToPeer = true
let browser = NWBrowser(
for: .bonjour(type: MoltbotBonjour.gatewayServiceType, domain: domain),
using: params)
browser.stateUpdateHandler = { [weak self] state in
Task { @MainActor in
guard let self else { return }
self.statesByDomain[domain] = state
self.updateStatusText()
self.appendDebugLog("state[\(domain)]: \(Self.prettyState(state))")
}
}
browser.browseResultsChangedHandler = { [weak self] results, _ in
Task { @MainActor in
guard let self else { return }
self.gatewaysByDomain[domain] = results.compactMap { result -> DiscoveredGateway? in
switch result.endpoint {
case let .service(name, _, _, _):
let decodedName = BonjourEscapes.decode(name)
let txt = result.endpoint.txtRecord?.dictionary ?? [:]
let advertisedName = txt["displayName"]
let prettyAdvertised = advertisedName
.map(Self.prettifyInstanceName)
.flatMap { $0.isEmpty ? nil : $0 }
let prettyName = prettyAdvertised ?? Self.prettifyInstanceName(decodedName)
return DiscoveredGateway(
name: prettyName,
endpoint: result.endpoint,
stableID: GatewayEndpointID.stableID(result.endpoint),
debugID: GatewayEndpointID.prettyDescription(result.endpoint),
lanHost: Self.txtValue(txt, key: "lanHost"),
tailnetDns: Self.txtValue(txt, key: "tailnetDns"),
gatewayPort: Self.txtIntValue(txt, key: "gatewayPort"),
canvasPort: Self.txtIntValue(txt, key: "canvasPort"),
tlsEnabled: Self.txtBoolValue(txt, key: "gatewayTls"),
tlsFingerprintSha256: Self.txtValue(txt, key: "gatewayTlsSha256"),
cliPath: Self.txtValue(txt, key: "cliPath"))
default:
return nil
}
}
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
self.recomputeGateways()
}
}
self.browsers[domain] = browser
browser.start(queue: DispatchQueue(label: "bot.molt.ios.gateway-discovery.\(domain)"))
}
}
func stop() {
self.appendDebugLog("stop()")
for browser in self.browsers.values {
browser.cancel()
}
self.browsers = [:]
self.gatewaysByDomain = [:]
self.statesByDomain = [:]
self.gateways = []
self.statusText = "Stopped"
}
private func recomputeGateways() {
let next = self.gatewaysByDomain.values
.flatMap(\.self)
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
let nextIDs = Set(next.map(\.stableID))
let added = nextIDs.subtracting(self.lastStableIDs)
let removed = self.lastStableIDs.subtracting(nextIDs)
if !added.isEmpty || !removed.isEmpty {
self.appendDebugLog("results: total=\(next.count) added=\(added.count) removed=\(removed.count)")
}
self.lastStableIDs = nextIDs
self.gateways = next
}
private func updateStatusText() {
let states = Array(self.statesByDomain.values)
if states.isEmpty {
self.statusText = self.browsers.isEmpty ? "Idle" : "Setup"
return
}
if let failed = states.first(where: { state in
if case .failed = state { return true }
return false
}) {
if case let .failed(err) = failed {
self.statusText = "Failed: \(err)"
return
}
}
if let waiting = states.first(where: { state in
if case .waiting = state { return true }
return false
}) {
if case let .waiting(err) = waiting {
self.statusText = "Waiting: \(err)"
return
}
}
if states.contains(where: { if case .ready = $0 { true } else { false } }) {
self.statusText = "Searching…"
return
}
if states.contains(where: { if case .setup = $0 { true } else { false } }) {
self.statusText = "Setup"
return
}
self.statusText = "Searching…"
}
private static func prettyState(_ state: NWBrowser.State) -> String {
switch state {
case .setup:
"setup"
case .ready:
"ready"
case let .failed(err):
"failed (\(err))"
case .cancelled:
"cancelled"
case let .waiting(err):
"waiting (\(err))"
@unknown default:
"unknown"
}
}
private func appendDebugLog(_ message: String) {
guard self.debugLoggingEnabled else { return }
self.debugLog.append(DebugLogEntry(ts: Date(), message: message))
if self.debugLog.count > 200 {
self.debugLog.removeFirst(self.debugLog.count - 200)
}
}
private static func prettifyInstanceName(_ decodedName: String) -> String {
let normalized = decodedName.split(whereSeparator: \.isWhitespace).joined(separator: " ")
let stripped = normalized.replacingOccurrences(of: " (Moltbot)", with: "")
.replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression)
return stripped.trimmingCharacters(in: .whitespacesAndNewlines)
}
private static func txtValue(_ dict: [String: String], key: String) -> String? {
let raw = dict[key]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? nil : raw
}
private static func txtIntValue(_ dict: [String: String], key: String) -> Int? {
guard let raw = self.txtValue(dict, key: key) else { return nil }
return Int(raw)
}
private static func txtBoolValue(_ dict: [String: String], key: String) -> Bool {
guard let raw = self.txtValue(dict, key: key)?.lowercased() else { return false }
return raw == "1" || raw == "true" || raw == "yes"
}
}

View File

@@ -0,0 +1,284 @@
import Foundation
enum GatewaySettingsStore {
private static let gatewayService = "bot.molt.gateway"
private static let legacyGatewayService = "com.clawdbot.gateway"
private static let legacyBridgeService = "com.clawdbot.bridge"
private static let nodeService = "bot.molt.node"
private static let legacyNodeService = "com.clawdbot.node"
private static let instanceIdDefaultsKey = "node.instanceId"
private static let preferredGatewayStableIDDefaultsKey = "gateway.preferredStableID"
private static let lastDiscoveredGatewayStableIDDefaultsKey = "gateway.lastDiscoveredStableID"
private static let manualEnabledDefaultsKey = "gateway.manual.enabled"
private static let manualHostDefaultsKey = "gateway.manual.host"
private static let manualPortDefaultsKey = "gateway.manual.port"
private static let manualTlsDefaultsKey = "gateway.manual.tls"
private static let discoveryDebugLogsDefaultsKey = "gateway.discovery.debugLogs"
private static let legacyPreferredBridgeStableIDDefaultsKey = "bridge.preferredStableID"
private static let legacyLastDiscoveredBridgeStableIDDefaultsKey = "bridge.lastDiscoveredStableID"
private static let legacyManualEnabledDefaultsKey = "bridge.manual.enabled"
private static let legacyManualHostDefaultsKey = "bridge.manual.host"
private static let legacyManualPortDefaultsKey = "bridge.manual.port"
private static let legacyDiscoveryDebugLogsDefaultsKey = "bridge.discovery.debugLogs"
private static let instanceIdAccount = "instanceId"
private static let preferredGatewayStableIDAccount = "preferredStableID"
private static let lastDiscoveredGatewayStableIDAccount = "lastDiscoveredStableID"
static func bootstrapPersistence() {
self.ensureStableInstanceID()
self.ensurePreferredGatewayStableID()
self.ensureLastDiscoveredGatewayStableID()
self.migrateLegacyDefaults()
}
static func loadStableInstanceID() -> String? {
if let value = KeychainStore.loadString(service: self.nodeService, account: self.instanceIdAccount)?
.trimmingCharacters(in: .whitespacesAndNewlines),
!value.isEmpty
{
return value
}
if let legacy = KeychainStore.loadString(service: self.legacyNodeService, account: self.instanceIdAccount)?
.trimmingCharacters(in: .whitespacesAndNewlines),
!legacy.isEmpty
{
_ = KeychainStore.saveString(legacy, service: self.nodeService, account: self.instanceIdAccount)
return legacy
}
return nil
}
static func saveStableInstanceID(_ instanceId: String) {
_ = KeychainStore.saveString(instanceId, service: self.nodeService, account: self.instanceIdAccount)
}
static func loadPreferredGatewayStableID() -> String? {
if let value = KeychainStore.loadString(
service: self.gatewayService,
account: self.preferredGatewayStableIDAccount
)?.trimmingCharacters(in: .whitespacesAndNewlines),
!value.isEmpty
{
return value
}
if let legacy = KeychainStore.loadString(
service: self.legacyGatewayService,
account: self.preferredGatewayStableIDAccount
)?.trimmingCharacters(in: .whitespacesAndNewlines),
!legacy.isEmpty
{
_ = KeychainStore.saveString(
legacy,
service: self.gatewayService,
account: self.preferredGatewayStableIDAccount)
return legacy
}
return nil
}
static func savePreferredGatewayStableID(_ stableID: String) {
_ = KeychainStore.saveString(
stableID,
service: self.gatewayService,
account: self.preferredGatewayStableIDAccount)
}
static func loadLastDiscoveredGatewayStableID() -> String? {
if let value = KeychainStore.loadString(
service: self.gatewayService,
account: self.lastDiscoveredGatewayStableIDAccount
)?.trimmingCharacters(in: .whitespacesAndNewlines),
!value.isEmpty
{
return value
}
if let legacy = KeychainStore.loadString(
service: self.legacyGatewayService,
account: self.lastDiscoveredGatewayStableIDAccount
)?.trimmingCharacters(in: .whitespacesAndNewlines),
!legacy.isEmpty
{
_ = KeychainStore.saveString(
legacy,
service: self.gatewayService,
account: self.lastDiscoveredGatewayStableIDAccount)
return legacy
}
return nil
}
static func saveLastDiscoveredGatewayStableID(_ stableID: String) {
_ = KeychainStore.saveString(
stableID,
service: self.gatewayService,
account: self.lastDiscoveredGatewayStableIDAccount)
}
static func loadGatewayToken(instanceId: String) -> String? {
let account = self.gatewayTokenAccount(instanceId: instanceId)
let token = KeychainStore.loadString(service: self.gatewayService, account: account)?
.trimmingCharacters(in: .whitespacesAndNewlines)
if token?.isEmpty == false { return token }
let legacyAccount = self.legacyBridgeTokenAccount(instanceId: instanceId)
let legacy = KeychainStore.loadString(service: self.legacyBridgeService, account: legacyAccount)?
.trimmingCharacters(in: .whitespacesAndNewlines)
if let legacy, !legacy.isEmpty {
_ = KeychainStore.saveString(legacy, service: self.gatewayService, account: account)
return legacy
}
return nil
}
static func saveGatewayToken(_ token: String, instanceId: String) {
_ = KeychainStore.saveString(
token,
service: self.gatewayService,
account: self.gatewayTokenAccount(instanceId: instanceId))
}
static func loadGatewayPassword(instanceId: String) -> String? {
KeychainStore.loadString(
service: self.gatewayService,
account: self.gatewayPasswordAccount(instanceId: instanceId))?
.trimmingCharacters(in: .whitespacesAndNewlines)
}
static func saveGatewayPassword(_ password: String, instanceId: String) {
_ = KeychainStore.saveString(
password,
service: self.gatewayService,
account: self.gatewayPasswordAccount(instanceId: instanceId))
}
private static func gatewayTokenAccount(instanceId: String) -> String {
"gateway-token.\(instanceId)"
}
private static func legacyBridgeTokenAccount(instanceId: String) -> String {
"bridge-token.\(instanceId)"
}
private static func gatewayPasswordAccount(instanceId: String) -> String {
"gateway-password.\(instanceId)"
}
private static func ensureStableInstanceID() {
let defaults = UserDefaults.standard
if let existing = defaults.string(forKey: self.instanceIdDefaultsKey)?
.trimmingCharacters(in: .whitespacesAndNewlines),
!existing.isEmpty
{
if self.loadStableInstanceID() == nil {
self.saveStableInstanceID(existing)
}
return
}
if let stored = self.loadStableInstanceID(), !stored.isEmpty {
defaults.set(stored, forKey: self.instanceIdDefaultsKey)
return
}
let fresh = UUID().uuidString
self.saveStableInstanceID(fresh)
defaults.set(fresh, forKey: self.instanceIdDefaultsKey)
}
private static func ensurePreferredGatewayStableID() {
let defaults = UserDefaults.standard
if let existing = defaults.string(forKey: self.preferredGatewayStableIDDefaultsKey)?
.trimmingCharacters(in: .whitespacesAndNewlines),
!existing.isEmpty
{
if self.loadPreferredGatewayStableID() == nil {
self.savePreferredGatewayStableID(existing)
}
return
}
if let stored = self.loadPreferredGatewayStableID(), !stored.isEmpty {
defaults.set(stored, forKey: self.preferredGatewayStableIDDefaultsKey)
}
}
private static func ensureLastDiscoveredGatewayStableID() {
let defaults = UserDefaults.standard
if let existing = defaults.string(forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)?
.trimmingCharacters(in: .whitespacesAndNewlines),
!existing.isEmpty
{
if self.loadLastDiscoveredGatewayStableID() == nil {
self.saveLastDiscoveredGatewayStableID(existing)
}
return
}
if let stored = self.loadLastDiscoveredGatewayStableID(), !stored.isEmpty {
defaults.set(stored, forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)
}
}
private static func migrateLegacyDefaults() {
let defaults = UserDefaults.standard
if defaults.string(forKey: self.preferredGatewayStableIDDefaultsKey)?.isEmpty != false,
let legacy = defaults.string(forKey: self.legacyPreferredBridgeStableIDDefaultsKey),
!legacy.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
{
defaults.set(legacy, forKey: self.preferredGatewayStableIDDefaultsKey)
self.savePreferredGatewayStableID(legacy)
}
if defaults.string(forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)?.isEmpty != false,
let legacy = defaults.string(forKey: self.legacyLastDiscoveredBridgeStableIDDefaultsKey),
!legacy.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
{
defaults.set(legacy, forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)
self.saveLastDiscoveredGatewayStableID(legacy)
}
if defaults.object(forKey: self.manualEnabledDefaultsKey) == nil,
defaults.object(forKey: self.legacyManualEnabledDefaultsKey) != nil
{
defaults.set(
defaults.bool(forKey: self.legacyManualEnabledDefaultsKey),
forKey: self.manualEnabledDefaultsKey)
}
if defaults.string(forKey: self.manualHostDefaultsKey)?.isEmpty != false,
let legacy = defaults.string(forKey: self.legacyManualHostDefaultsKey),
!legacy.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
{
defaults.set(legacy, forKey: self.manualHostDefaultsKey)
}
if defaults.integer(forKey: self.manualPortDefaultsKey) == 0,
defaults.integer(forKey: self.legacyManualPortDefaultsKey) > 0
{
defaults.set(
defaults.integer(forKey: self.legacyManualPortDefaultsKey),
forKey: self.manualPortDefaultsKey)
}
if defaults.object(forKey: self.discoveryDebugLogsDefaultsKey) == nil,
defaults.object(forKey: self.legacyDiscoveryDebugLogsDefaultsKey) != nil
{
defaults.set(
defaults.bool(forKey: self.legacyDiscoveryDebugLogsDefaultsKey),
forKey: self.discoveryDebugLogsDefaultsKey)
}
}
}

View File

@@ -0,0 +1,48 @@
import Foundation
import Security
enum KeychainStore {
static func loadString(service: String, account: String) -> String? {
let query: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrService as String: service,
kSecAttrAccount as String: account,
kSecReturnData as String: true,
kSecMatchLimit as String: kSecMatchLimitOne,
]
var item: CFTypeRef?
let status = SecItemCopyMatching(query as CFDictionary, &item)
guard status == errSecSuccess, let data = item as? Data else { return nil }
return String(data: data, encoding: .utf8)
}
static func saveString(_ value: String, service: String, account: String) -> Bool {
let data = Data(value.utf8)
let query: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrService as String: service,
kSecAttrAccount as String: account,
]
let update: [String: Any] = [kSecValueData as String: data]
let status = SecItemUpdate(query as CFDictionary, update as CFDictionary)
if status == errSecSuccess { return true }
if status != errSecItemNotFound { return false }
var insert = query
insert[kSecValueData as String] = data
insert[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly
return SecItemAdd(insert as CFDictionary, nil) == errSecSuccess
}
static func delete(service: String, account: String) -> Bool {
let query: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrService as String: service,
kSecAttrAccount as String: account,
]
let status = SecItemDelete(query as CFDictionary)
return status == errSecSuccess || status == errSecItemNotFound
}
}

View File

@@ -0,0 +1,72 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
<string>Moltbot</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIconName</key>
<string>AppIcon</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>2026.1.26</string>
<key>CFBundleVersion</key>
<string>20260126</string>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsArbitraryLoadsInWebContent</key>
<true/>
</dict>
<key>NSBonjourServices</key>
<array>
<string>_moltbot-gw._tcp</string>
</array>
<key>NSCameraUsageDescription</key>
<string>Moltbot can capture photos or short video clips when requested via the gateway.</string>
<key>NSLocalNetworkUsageDescription</key>
<string>Moltbot discovers and connects to your Moltbot gateway on the local network.</string>
<key>NSLocationAlwaysAndWhenInUseUsageDescription</key>
<string>Moltbot can share your location in the background when you enable Always.</string>
<key>NSLocationWhenInUseUsageDescription</key>
<string>Moltbot uses your location when you allow location sharing.</string>
<key>NSMicrophoneUsageDescription</key>
<string>Moltbot needs microphone access for voice wake.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>Moltbot uses on-device speech recognition for voice wake.</string>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
</dict>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
</array>
<key>UILaunchScreen</key>
<dict/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>

View File

@@ -0,0 +1,138 @@
import MoltbotKit
import CoreLocation
import Foundation
@MainActor
final class LocationService: NSObject, CLLocationManagerDelegate {
enum Error: Swift.Error {
case timeout
case unavailable
}
private let manager = CLLocationManager()
private var authContinuation: CheckedContinuation<CLAuthorizationStatus, Never>?
private var locationContinuation: CheckedContinuation<CLLocation, Swift.Error>?
override init() {
super.init()
self.manager.delegate = self
self.manager.desiredAccuracy = kCLLocationAccuracyBest
}
func authorizationStatus() -> CLAuthorizationStatus {
self.manager.authorizationStatus
}
func accuracyAuthorization() -> CLAccuracyAuthorization {
if #available(iOS 14.0, *) {
return self.manager.accuracyAuthorization
}
return .fullAccuracy
}
func ensureAuthorization(mode: MoltbotLocationMode) async -> CLAuthorizationStatus {
guard CLLocationManager.locationServicesEnabled() else { return .denied }
let status = self.manager.authorizationStatus
if status == .notDetermined {
self.manager.requestWhenInUseAuthorization()
let updated = await self.awaitAuthorizationChange()
if mode != .always { return updated }
}
if mode == .always {
let current = self.manager.authorizationStatus
if current == .authorizedWhenInUse {
self.manager.requestAlwaysAuthorization()
return await self.awaitAuthorizationChange()
}
return current
}
return self.manager.authorizationStatus
}
func currentLocation(
params: MoltbotLocationGetParams,
desiredAccuracy: MoltbotLocationAccuracy,
maxAgeMs: Int?,
timeoutMs: Int?) async throws -> CLLocation
{
let now = Date()
if let maxAgeMs,
let cached = self.manager.location,
now.timeIntervalSince(cached.timestamp) * 1000 <= Double(maxAgeMs)
{
return cached
}
self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy)
let timeout = max(0, timeoutMs ?? 10000)
return try await self.withTimeout(timeoutMs: timeout) {
try await self.requestLocation()
}
}
private func requestLocation() async throws -> CLLocation {
try await withCheckedThrowingContinuation { cont in
self.locationContinuation = cont
self.manager.requestLocation()
}
}
private func awaitAuthorizationChange() async -> CLAuthorizationStatus {
await withCheckedContinuation { cont in
self.authContinuation = cont
}
}
private func withTimeout<T: Sendable>(
timeoutMs: Int,
operation: @escaping @Sendable () async throws -> T) async throws -> T
{
try await AsyncTimeout.withTimeoutMs(timeoutMs: timeoutMs, onTimeout: { Error.timeout }, operation: operation)
}
private static func accuracyValue(_ accuracy: MoltbotLocationAccuracy) -> CLLocationAccuracy {
switch accuracy {
case .coarse:
kCLLocationAccuracyKilometer
case .balanced:
kCLLocationAccuracyHundredMeters
case .precise:
kCLLocationAccuracyBest
}
}
nonisolated func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) {
let status = manager.authorizationStatus
Task { @MainActor in
if let cont = self.authContinuation {
self.authContinuation = nil
cont.resume(returning: status)
}
}
}
nonisolated func locationManager(_ manager: CLLocationManager, didUpdateLocations locations: [CLLocation]) {
let locs = locations
Task { @MainActor in
guard let cont = self.locationContinuation else { return }
self.locationContinuation = nil
if let latest = locs.last {
cont.resume(returning: latest)
} else {
cont.resume(throwing: Error.unavailable)
}
}
}
nonisolated func locationManager(_ manager: CLLocationManager, didFailWithError error: Swift.Error) {
let err = error
Task { @MainActor in
guard let cont = self.locationContinuation else { return }
self.locationContinuation = nil
cont.resume(throwing: err)
}
}
}

View File

@@ -0,0 +1,959 @@
import MoltbotKit
import Network
import Observation
import SwiftUI
import UIKit
@MainActor
@Observable
final class NodeAppModel {
enum CameraHUDKind {
case photo
case recording
case success
case error
}
var isBackgrounded: Bool = false
let screen = ScreenController()
let camera = CameraController()
private let screenRecorder = ScreenRecordService()
var gatewayStatusText: String = "Offline"
var gatewayServerName: String?
var gatewayRemoteAddress: String?
var connectedGatewayID: String?
var seamColorHex: String?
var mainSessionKey: String = "main"
private let gateway = GatewayNodeSession()
private var gatewayTask: Task<Void, Never>?
private var voiceWakeSyncTask: Task<Void, Never>?
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
let voiceWake = VoiceWakeManager()
let talkMode = TalkModeManager()
private let locationService = LocationService()
private var lastAutoA2uiURL: String?
private var gatewayConnected = false
var gatewaySession: GatewayNodeSession { self.gateway }
var cameraHUDText: String?
var cameraHUDKind: CameraHUDKind?
var cameraFlashNonce: Int = 0
var screenRecordActive: Bool = false
init() {
self.voiceWake.configure { [weak self] cmd in
guard let self else { return }
let sessionKey = await MainActor.run { self.mainSessionKey }
do {
try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey)
} catch {
// Best-effort only.
}
}
let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled")
self.voiceWake.setEnabled(enabled)
self.talkMode.attachGateway(self.gateway)
let talkEnabled = UserDefaults.standard.bool(forKey: "talk.enabled")
self.talkMode.setEnabled(talkEnabled)
// Wire up deep links from canvas taps
self.screen.onDeepLink = { [weak self] url in
guard let self else { return }
Task { @MainActor in
await self.handleDeepLink(url: url)
}
}
// Wire up A2UI action clicks (buttons, etc.)
self.screen.onA2UIAction = { [weak self] body in
guard let self else { return }
Task { @MainActor in
await self.handleCanvasA2UIAction(body: body)
}
}
}
private func handleCanvasA2UIAction(body: [String: Any]) async {
let userActionAny = body["userAction"] ?? body
let userAction: [String: Any] = {
if let dict = userActionAny as? [String: Any] { return dict }
if let dict = userActionAny as? [AnyHashable: Any] {
return dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
}
return [:]
}()
guard !userAction.isEmpty else { return }
guard let name = MoltbotCanvasA2UIAction.extractActionName(userAction) else { return }
let actionId: String = {
let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return id.isEmpty ? UUID().uuidString : id
}()
let surfaceId: String = {
let raw = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? "main" : raw
}()
let sourceComponentId: String = {
let raw = (userAction[
"sourceComponentId",
] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return raw.isEmpty ? "-" : raw
}()
let host = UserDefaults.standard.string(forKey: "node.displayName") ?? UIDevice.current.name
let instanceId = (UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node").lowercased()
let contextJSON = MoltbotCanvasA2UIAction.compactJSON(userAction["context"])
let sessionKey = self.mainSessionKey
let messageContext = MoltbotCanvasA2UIAction.AgentMessageContext(
actionName: name,
session: .init(key: sessionKey, surfaceId: surfaceId),
component: .init(id: sourceComponentId, host: host, instanceId: instanceId),
contextJSON: contextJSON)
let message = MoltbotCanvasA2UIAction.formatAgentMessage(messageContext)
let ok: Bool
var errorText: String?
if await !self.isGatewayConnected() {
ok = false
errorText = "gateway not connected"
} else {
do {
try await self.sendAgentRequest(link: AgentDeepLink(
message: message,
sessionKey: sessionKey,
thinking: "low",
deliver: false,
to: nil,
channel: nil,
timeoutSeconds: nil,
key: actionId))
ok = true
} catch {
ok = false
errorText = error.localizedDescription
}
}
let js = MoltbotCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId: actionId, ok: ok, error: errorText)
do {
_ = try await self.screen.eval(javaScript: js)
} catch {
// ignore
}
}
private func resolveA2UIHostURL() async -> String? {
guard let raw = await self.gateway.currentCanvasHostUrl() else { return nil }
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
return base.appendingPathComponent("__moltbot__/a2ui/").absoluteString + "?platform=ios"
}
private func showA2UIOnConnectIfNeeded() async {
guard let a2uiUrl = await self.resolveA2UIHostURL() else { return }
let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
if current.isEmpty || current == self.lastAutoA2uiURL {
self.screen.navigate(to: a2uiUrl)
self.lastAutoA2uiURL = a2uiUrl
}
}
private func showLocalCanvasOnDisconnect() {
self.lastAutoA2uiURL = nil
self.screen.showDefaultCanvas()
}
func setScenePhase(_ phase: ScenePhase) {
switch phase {
case .background:
self.isBackgrounded = true
case .active, .inactive:
self.isBackgrounded = false
@unknown default:
self.isBackgrounded = false
}
}
func setVoiceWakeEnabled(_ enabled: Bool) {
self.voiceWake.setEnabled(enabled)
}
func setTalkEnabled(_ enabled: Bool) {
self.talkMode.setEnabled(enabled)
}
func requestLocationPermissions(mode: MoltbotLocationMode) async -> Bool {
guard mode != .off else { return true }
let status = await self.locationService.ensureAuthorization(mode: mode)
switch status {
case .authorizedAlways:
return true
case .authorizedWhenInUse:
return mode != .always
default:
return false
}
}
func connectToGateway(
url: URL,
gatewayStableID: String,
tls: GatewayTLSParams?,
token: String?,
password: String?,
connectOptions: GatewayConnectOptions)
{
self.gatewayTask?.cancel()
self.gatewayServerName = nil
self.gatewayRemoteAddress = nil
let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines)
self.connectedGatewayID = id.isEmpty ? url.absoluteString : id
self.gatewayConnected = false
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = nil
let sessionBox = tls.map { WebSocketSessionBox(session: GatewayTLSPinningSession(params: $0)) }
self.gatewayTask = Task {
var attempt = 0
while !Task.isCancelled {
await MainActor.run {
if attempt == 0 {
self.gatewayStatusText = "Connecting…"
} else {
self.gatewayStatusText = "Reconnecting…"
}
self.gatewayServerName = nil
self.gatewayRemoteAddress = nil
}
do {
try await self.gateway.connect(
url: url,
token: token,
password: password,
connectOptions: connectOptions,
sessionBox: sessionBox,
onConnected: { [weak self] in
guard let self else { return }
await MainActor.run {
self.gatewayStatusText = "Connected"
self.gatewayServerName = url.host ?? "gateway"
self.gatewayConnected = true
}
if let addr = await self.gateway.currentRemoteAddress() {
await MainActor.run {
self.gatewayRemoteAddress = addr
}
}
await self.refreshBrandingFromGateway()
await self.startVoiceWakeSync()
await self.showA2UIOnConnectIfNeeded()
},
onDisconnected: { [weak self] reason in
guard let self else { return }
await MainActor.run {
self.gatewayStatusText = "Disconnected"
self.gatewayRemoteAddress = nil
self.gatewayConnected = false
self.showLocalCanvasOnDisconnect()
self.gatewayStatusText = "Disconnected: \(reason)"
}
},
onInvoke: { [weak self] req in
guard let self else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "UNAVAILABLE: node not ready"))
}
return await self.handleInvoke(req)
})
if Task.isCancelled { break }
attempt = 0
try? await Task.sleep(nanoseconds: 1_000_000_000)
} catch {
if Task.isCancelled { break }
attempt += 1
await MainActor.run {
self.gatewayStatusText = "Gateway error: \(error.localizedDescription)"
self.gatewayServerName = nil
self.gatewayRemoteAddress = nil
self.gatewayConnected = false
self.showLocalCanvasOnDisconnect()
}
let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt)))
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
}
}
await MainActor.run {
self.gatewayStatusText = "Offline"
self.gatewayServerName = nil
self.gatewayRemoteAddress = nil
self.connectedGatewayID = nil
self.gatewayConnected = false
self.seamColorHex = nil
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
self.mainSessionKey = "main"
self.talkMode.updateMainSessionKey(self.mainSessionKey)
}
self.showLocalCanvasOnDisconnect()
}
}
}
func disconnectGateway() {
self.gatewayTask?.cancel()
self.gatewayTask = nil
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = nil
Task { await self.gateway.disconnect() }
self.gatewayStatusText = "Offline"
self.gatewayServerName = nil
self.gatewayRemoteAddress = nil
self.connectedGatewayID = nil
self.gatewayConnected = false
self.seamColorHex = nil
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
self.mainSessionKey = "main"
self.talkMode.updateMainSessionKey(self.mainSessionKey)
}
self.showLocalCanvasOnDisconnect()
}
private func applyMainSessionKey(_ key: String?) {
let trimmed = (key ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
let current = self.mainSessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
if SessionKey.isCanonicalMainSessionKey(current) { return }
if trimmed == current { return }
self.mainSessionKey = trimmed
self.talkMode.updateMainSessionKey(trimmed)
}
var seamColor: Color {
Self.color(fromHex: self.seamColorHex) ?? Self.defaultSeamColor
}
private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0)
private static func color(fromHex raw: String?) -> Color? {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed
guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil }
let r = Double((value >> 16) & 0xFF) / 255.0
let g = Double((value >> 8) & 0xFF) / 255.0
let b = Double(value & 0xFF) / 255.0
return Color(red: r, green: g, blue: b)
}
private func refreshBrandingFromGateway() async {
do {
let res = try await self.gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
guard let config = json["config"] as? [String: Any] else { return }
let ui = config["ui"] as? [String: Any]
let raw = (ui?["seamColor"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
let session = config["session"] as? [String: Any]
let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String)
await MainActor.run {
self.seamColorHex = raw.isEmpty ? nil : raw
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
self.mainSessionKey = mainKey
self.talkMode.updateMainSessionKey(mainKey)
}
}
} catch {
// ignore
}
}
func setGlobalWakeWords(_ words: [String]) async {
let sanitized = VoiceWakePreferences.sanitizeTriggerWords(words)
struct Payload: Codable {
var triggers: [String]
}
let payload = Payload(triggers: sanitized)
guard let data = try? JSONEncoder().encode(payload),
let json = String(data: data, encoding: .utf8)
else { return }
do {
_ = try await self.gateway.request(method: "voicewake.set", paramsJSON: json, timeoutSeconds: 12)
} catch {
// Best-effort only.
}
}
private func startVoiceWakeSync() async {
self.voiceWakeSyncTask?.cancel()
self.voiceWakeSyncTask = Task { [weak self] in
guard let self else { return }
await self.refreshWakeWordsFromGateway()
let stream = await self.gateway.subscribeServerEvents(bufferingNewest: 200)
for await evt in stream {
if Task.isCancelled { return }
guard evt.event == "voicewake.changed" else { continue }
guard let payload = evt.payload else { continue }
struct Payload: Decodable { var triggers: [String] }
guard let decoded = try? GatewayPayloadDecoding.decode(payload, as: Payload.self) else { continue }
let triggers = VoiceWakePreferences.sanitizeTriggerWords(decoded.triggers)
VoiceWakePreferences.saveTriggerWords(triggers)
}
}
}
private func refreshWakeWordsFromGateway() async {
do {
let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: data) else { return }
VoiceWakePreferences.saveTriggerWords(triggers)
} catch {
// Best-effort only.
}
}
func sendVoiceTranscript(text: String, sessionKey: String?) async throws {
if await !self.isGatewayConnected() {
throw NSError(domain: "Gateway", code: 10, userInfo: [
NSLocalizedDescriptionKey: "Gateway not connected",
])
}
struct Payload: Codable {
var text: String
var sessionKey: String?
}
let payload = Payload(text: text, sessionKey: sessionKey)
let data = try JSONEncoder().encode(payload)
guard let json = String(bytes: data, encoding: .utf8) else {
throw NSError(domain: "NodeAppModel", code: 1, userInfo: [
NSLocalizedDescriptionKey: "Failed to encode voice transcript payload as UTF-8",
])
}
await self.gateway.sendEvent(event: "voice.transcript", payloadJSON: json)
}
func handleDeepLink(url: URL) async {
guard let route = DeepLinkParser.parse(url) else { return }
switch route {
case let .agent(link):
await self.handleAgentDeepLink(link, originalURL: url)
}
}
private func handleAgentDeepLink(_ link: AgentDeepLink, originalURL: URL) async {
let message = link.message.trimmingCharacters(in: .whitespacesAndNewlines)
guard !message.isEmpty else { return }
if message.count > 20000 {
self.screen.errorText = "Deep link too large (message exceeds 20,000 characters)."
return
}
guard await self.isGatewayConnected() else {
self.screen.errorText = "Gateway not connected (cannot forward deep link)."
return
}
do {
try await self.sendAgentRequest(link: link)
self.screen.errorText = nil
} catch {
self.screen.errorText = "Agent request failed: \(error.localizedDescription)"
}
}
private func sendAgentRequest(link: AgentDeepLink) async throws {
if link.message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
throw NSError(domain: "DeepLink", code: 1, userInfo: [
NSLocalizedDescriptionKey: "invalid agent message",
])
}
// iOS gateway forwards to the gateway; no local auth prompts here.
// (Key-based unattended auth is handled on macOS for moltbot:// links.)
let data = try JSONEncoder().encode(link)
guard let json = String(bytes: data, encoding: .utf8) else {
throw NSError(domain: "NodeAppModel", code: 2, userInfo: [
NSLocalizedDescriptionKey: "Failed to encode agent request payload as UTF-8",
])
}
await self.gateway.sendEvent(event: "agent.request", payloadJSON: json)
}
private func isGatewayConnected() async -> Bool {
self.gatewayConnected
}
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
let command = req.command
if self.isBackgrounded, self.isBackgroundRestricted(command) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .backgroundUnavailable,
message: "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground"))
}
if command.hasPrefix("camera."), !self.isCameraEnabled() {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "CAMERA_DISABLED: enable Camera in iOS Settings → Camera → Allow Camera"))
}
do {
switch command {
case MoltbotLocationCommand.get.rawValue:
return try await self.handleLocationInvoke(req)
case MoltbotCanvasCommand.present.rawValue,
MoltbotCanvasCommand.hide.rawValue,
MoltbotCanvasCommand.navigate.rawValue,
MoltbotCanvasCommand.evalJS.rawValue,
MoltbotCanvasCommand.snapshot.rawValue:
return try await self.handleCanvasInvoke(req)
case MoltbotCanvasA2UICommand.reset.rawValue,
MoltbotCanvasA2UICommand.push.rawValue,
MoltbotCanvasA2UICommand.pushJSONL.rawValue:
return try await self.handleCanvasA2UIInvoke(req)
case MoltbotCameraCommand.list.rawValue,
MoltbotCameraCommand.snap.rawValue,
MoltbotCameraCommand.clip.rawValue:
return try await self.handleCameraInvoke(req)
case MoltbotScreenCommand.record.rawValue:
return try await self.handleScreenRecordInvoke(req)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
} catch {
if command.hasPrefix("camera.") {
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
}
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(code: .unavailable, message: error.localizedDescription))
}
}
private func isBackgroundRestricted(_ command: String) -> Bool {
command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen.")
}
private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let mode = self.locationMode()
guard mode != .off else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "LOCATION_DISABLED: enable Location in Settings"))
}
if self.isBackgrounded, mode != .always {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .backgroundUnavailable,
message: "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always"))
}
let params = (try? Self.decodeParams(MoltbotLocationGetParams.self, from: req.paramsJSON)) ??
MoltbotLocationGetParams()
let desired = params.desiredAccuracy ??
(self.isLocationPreciseEnabled() ? .precise : .balanced)
let status = self.locationService.authorizationStatus()
if status != .authorizedAlways, status != .authorizedWhenInUse {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "LOCATION_PERMISSION_REQUIRED: grant Location permission"))
}
if self.isBackgrounded, status != .authorizedAlways {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "LOCATION_PERMISSION_REQUIRED: enable Always for background access"))
}
let location = try await self.locationService.currentLocation(
params: params,
desiredAccuracy: desired,
maxAgeMs: params.maxAgeMs,
timeoutMs: params.timeoutMs)
let isPrecise = self.locationService.accuracyAuthorization() == .fullAccuracy
let payload = MoltbotLocationPayload(
lat: location.coordinate.latitude,
lon: location.coordinate.longitude,
accuracyMeters: location.horizontalAccuracy,
altitudeMeters: location.verticalAccuracy >= 0 ? location.altitude : nil,
speedMps: location.speed >= 0 ? location.speed : nil,
headingDeg: location.course >= 0 ? location.course : nil,
timestamp: ISO8601DateFormatter().string(from: location.timestamp),
isPrecise: isPrecise,
source: nil)
let json = try Self.encodePayload(payload)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
}
private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
switch req.command {
case MoltbotCanvasCommand.present.rawValue:
let params = (try? Self.decodeParams(MoltbotCanvasPresentParams.self, from: req.paramsJSON)) ??
MoltbotCanvasPresentParams()
let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if url.isEmpty {
self.screen.showDefaultCanvas()
} else {
self.screen.navigate(to: url)
}
return BridgeInvokeResponse(id: req.id, ok: true)
case MoltbotCanvasCommand.hide.rawValue:
return BridgeInvokeResponse(id: req.id, ok: true)
case MoltbotCanvasCommand.navigate.rawValue:
let params = try Self.decodeParams(MoltbotCanvasNavigateParams.self, from: req.paramsJSON)
self.screen.navigate(to: params.url)
return BridgeInvokeResponse(id: req.id, ok: true)
case MoltbotCanvasCommand.evalJS.rawValue:
let params = try Self.decodeParams(MoltbotCanvasEvalParams.self, from: req.paramsJSON)
let result = try await self.screen.eval(javaScript: params.javaScript)
let payload = try Self.encodePayload(["result": result])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case MoltbotCanvasCommand.snapshot.rawValue:
let params = try? Self.decodeParams(MoltbotCanvasSnapshotParams.self, from: req.paramsJSON)
let format = params?.format ?? .jpeg
let maxWidth: CGFloat? = {
if let raw = params?.maxWidth, raw > 0 { return CGFloat(raw) }
// Keep default snapshots comfortably below the gateway client's maxPayload.
// For full-res, clients should explicitly request a larger maxWidth.
return switch format {
case .png: 900
case .jpeg: 1600
}
}()
let base64 = try await self.screen.snapshotBase64(
maxWidth: maxWidth,
format: format,
quality: params?.quality)
let payload = try Self.encodePayload([
"format": format == .jpeg ? "jpeg" : "png",
"base64": base64,
])
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
}
private func handleCanvasA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let command = req.command
switch command {
case MoltbotCanvasA2UICommand.reset.rawValue:
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
}
self.screen.navigate(to: a2uiUrl)
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
}
let json = try await self.screen.eval(javaScript: """
(() => {
if (!globalThis.clawdbotA2UI) return JSON.stringify({ ok: false, error: "missing moltbotA2UI" });
return JSON.stringify(globalThis.clawdbotA2UI.reset());
})()
""")
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
case MoltbotCanvasA2UICommand.push.rawValue, MoltbotCanvasA2UICommand.pushJSONL.rawValue:
let messages: [AnyCodable]
if command == MoltbotCanvasA2UICommand.pushJSONL.rawValue {
let params = try Self.decodeParams(MoltbotCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try MoltbotCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
} else {
do {
let params = try Self.decodeParams(MoltbotCanvasA2UIPushParams.self, from: req.paramsJSON)
messages = params.messages
} catch {
// Be forgiving: some clients still send JSONL payloads to `canvas.a2ui.push`.
let params = try Self.decodeParams(MoltbotCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
messages = try MoltbotCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
}
}
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
}
self.screen.navigate(to: a2uiUrl)
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(
code: .unavailable,
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
}
let messagesJSON = try MoltbotCanvasA2UIJSONL.encodeMessagesJSONArray(messages)
let js = """
(() => {
try {
if (!globalThis.clawdbotA2UI) return JSON.stringify({ ok: false, error: "missing moltbotA2UI" });
const messages = \(messagesJSON);
return JSON.stringify(globalThis.clawdbotA2UI.applyMessages(messages));
} catch (e) {
return JSON.stringify({ ok: false, error: String(e?.message ?? e) });
}
})()
"""
let resultJSON = try await self.screen.eval(javaScript: js)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
}
private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
switch req.command {
case MoltbotCameraCommand.list.rawValue:
let devices = await self.camera.listDevices()
struct Payload: Codable {
var devices: [CameraController.CameraDeviceInfo]
}
let payload = try Self.encodePayload(Payload(devices: devices))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case MoltbotCameraCommand.snap.rawValue:
self.showCameraHUD(text: "Taking photo…", kind: .photo)
self.triggerCameraFlash()
let params = (try? Self.decodeParams(MoltbotCameraSnapParams.self, from: req.paramsJSON)) ??
MoltbotCameraSnapParams()
let res = try await self.camera.snap(params: params)
struct Payload: Codable {
var format: String
var base64: String
var width: Int
var height: Int
}
let payload = try Self.encodePayload(Payload(
format: res.format,
base64: res.base64,
width: res.width,
height: res.height))
self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
case MoltbotCameraCommand.clip.rawValue:
let params = (try? Self.decodeParams(MoltbotCameraClipParams.self, from: req.paramsJSON)) ??
MoltbotCameraClipParams()
let suspended = (params.includeAudio ?? true) ? self.voiceWake.suspendForExternalAudioCapture() : false
defer { self.voiceWake.resumeAfterExternalAudioCapture(wasSuspended: suspended) }
self.showCameraHUD(text: "Recording…", kind: .recording)
let res = try await self.camera.clip(params: params)
struct Payload: Codable {
var format: String
var base64: String
var durationMs: Int
var hasAudio: Bool
}
let payload = try Self.encodePayload(Payload(
format: res.format,
base64: res.base64,
durationMs: res.durationMs,
hasAudio: res.hasAudio))
self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8)
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
default:
return BridgeInvokeResponse(
id: req.id,
ok: false,
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
}
}
private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
let params = (try? Self.decodeParams(MoltbotScreenRecordParams.self, from: req.paramsJSON)) ??
MoltbotScreenRecordParams()
if let format = params.format, format.lowercased() != "mp4" {
throw NSError(domain: "Screen", code: 30, userInfo: [
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
])
}
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
self.screenRecordActive = true
defer { self.screenRecordActive = false }
let path = try await self.screenRecorder.record(
screenIndex: params.screenIndex,
durationMs: params.durationMs,
fps: params.fps,
includeAudio: params.includeAudio,
outPath: nil)
defer { try? FileManager().removeItem(atPath: path) }
let data = try Data(contentsOf: URL(fileURLWithPath: path))
struct Payload: Codable {
var format: String
var base64: String
var durationMs: Int?
var fps: Double?
var screenIndex: Int?
var hasAudio: Bool
}
let payload = try Self.encodePayload(Payload(
format: "mp4",
base64: data.base64EncodedString(),
durationMs: params.durationMs,
fps: params.fps,
screenIndex: params.screenIndex,
hasAudio: params.includeAudio ?? true))
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
}
}
private extension NodeAppModel {
func locationMode() -> MoltbotLocationMode {
let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
return MoltbotLocationMode(rawValue: raw) ?? .off
}
func isLocationPreciseEnabled() -> Bool {
if UserDefaults.standard.object(forKey: "location.preciseEnabled") == nil { return true }
return UserDefaults.standard.bool(forKey: "location.preciseEnabled")
}
static func decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
guard let json, let data = json.data(using: .utf8) else {
throw NSError(domain: "Gateway", code: 20, userInfo: [
NSLocalizedDescriptionKey: "INVALID_REQUEST: paramsJSON required",
])
}
return try JSONDecoder().decode(type, from: data)
}
static func encodePayload(_ obj: some Encodable) throws -> String {
let data = try JSONEncoder().encode(obj)
guard let json = String(bytes: data, encoding: .utf8) else {
throw NSError(domain: "NodeAppModel", code: 21, userInfo: [
NSLocalizedDescriptionKey: "Failed to encode payload as UTF-8",
])
}
return json
}
func isCameraEnabled() -> Bool {
// Default-on: if the key doesn't exist yet, treat it as enabled.
if UserDefaults.standard.object(forKey: "camera.enabled") == nil { return true }
return UserDefaults.standard.bool(forKey: "camera.enabled")
}
func triggerCameraFlash() {
self.cameraFlashNonce &+= 1
}
func showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.cameraHUDDismissTask?.cancel()
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
self.cameraHUDText = text
self.cameraHUDKind = kind
}
guard let autoHideSeconds else { return }
self.cameraHUDDismissTask = Task { @MainActor in
try? await Task.sleep(nanoseconds: UInt64(autoHideSeconds * 1_000_000_000))
withAnimation(.easeOut(duration: 0.25)) {
self.cameraHUDText = nil
self.cameraHUDKind = nil
}
}
}
}
#if DEBUG
extension NodeAppModel {
func _test_handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
await self.handleInvoke(req)
}
static func _test_decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
try self.decodeParams(type, from: json)
}
static func _test_encodePayload(_ obj: some Encodable) throws -> String {
try self.encodePayload(obj)
}
func _test_isCameraEnabled() -> Bool {
self.isCameraEnabled()
}
func _test_triggerCameraFlash() {
self.triggerCameraFlash()
}
func _test_showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
self.showCameraHUD(text: text, kind: kind, autoHideSeconds: autoHideSeconds)
}
func _test_handleCanvasA2UIAction(body: [String: Any]) async {
await self.handleCanvasA2UIAction(body: body)
}
func _test_resolveA2UIHostURL() async -> String? {
await self.resolveA2UIHostURL()
}
func _test_showLocalCanvasOnDisconnect() {
self.showLocalCanvasOnDisconnect()
}
}
#endif

View File

@@ -0,0 +1,341 @@
import SwiftUI
import UIKit
struct RootCanvas: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@Environment(\.colorScheme) private var systemColorScheme
@Environment(\.scenePhase) private var scenePhase
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var presentedSheet: PresentedSheet?
@State private var voiceWakeToastText: String?
@State private var toastDismissTask: Task<Void, Never>?
private enum PresentedSheet: Identifiable {
case settings
case chat
var id: Int {
switch self {
case .settings: 0
case .chat: 1
}
}
}
var body: some View {
ZStack {
CanvasContent(
systemColorScheme: self.systemColorScheme,
gatewayStatus: self.gatewayStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
voiceWakeToastText: self.voiceWakeToastText,
cameraHUDText: self.appModel.cameraHUDText,
cameraHUDKind: self.appModel.cameraHUDKind,
openChat: {
self.presentedSheet = .chat
},
openSettings: {
self.presentedSheet = .settings
})
.preferredColorScheme(.dark)
if self.appModel.cameraFlashNonce != 0 {
CameraFlashOverlay(nonce: self.appModel.cameraFlashNonce)
}
}
.sheet(item: self.$presentedSheet) { sheet in
switch sheet {
case .settings:
SettingsTab()
case .chat:
ChatSheet(
gateway: self.appModel.gatewaySession,
sessionKey: self.appModel.mainSessionKey,
userAccent: self.appModel.seamColor)
}
}
.onAppear { self.updateIdleTimer() }
.onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() }
.onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() }
.onAppear { self.updateCanvasDebugStatus() }
.onChange(of: self.canvasDebugStatusEnabled) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.gatewayStatusText) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.gatewayServerName) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.appModel.gatewayRemoteAddress) { _, _ in self.updateCanvasDebugStatus() }
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
guard let newValue else { return }
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
self.toastDismissTask?.cancel()
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
self.voiceWakeToastText = trimmed
}
self.toastDismissTask = Task {
try? await Task.sleep(nanoseconds: 2_300_000_000)
await MainActor.run {
withAnimation(.easeOut(duration: 0.25)) {
self.voiceWakeToastText = nil
}
}
}
}
.onDisappear {
UIApplication.shared.isIdleTimerDisabled = false
self.toastDismissTask?.cancel()
self.toastDismissTask = nil
}
}
private var gatewayStatus: StatusPill.GatewayState {
if self.appModel.gatewayServerName != nil { return .connected }
let text = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
if text.localizedCaseInsensitiveContains("connecting") ||
text.localizedCaseInsensitiveContains("reconnecting")
{
return .connecting
}
if text.localizedCaseInsensitiveContains("error") {
return .error
}
return .disconnected
}
private func updateIdleTimer() {
UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep)
}
private func updateCanvasDebugStatus() {
self.appModel.screen.setDebugStatusEnabled(self.canvasDebugStatusEnabled)
guard self.canvasDebugStatusEnabled else { return }
let title = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let subtitle = self.appModel.gatewayServerName ?? self.appModel.gatewayRemoteAddress
self.appModel.screen.updateDebugStatus(title: title, subtitle: subtitle)
}
}
private struct CanvasContent: View {
@Environment(NodeAppModel.self) private var appModel
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
var systemColorScheme: ColorScheme
var gatewayStatus: StatusPill.GatewayState
var voiceWakeEnabled: Bool
var voiceWakeToastText: String?
var cameraHUDText: String?
var cameraHUDKind: NodeAppModel.CameraHUDKind?
var openChat: () -> Void
var openSettings: () -> Void
private var brightenButtons: Bool { self.systemColorScheme == .light }
var body: some View {
ZStack(alignment: .topTrailing) {
ScreenTab()
VStack(spacing: 10) {
OverlayButton(systemImage: "text.bubble.fill", brighten: self.brightenButtons) {
self.openChat()
}
.accessibilityLabel("Chat")
if self.talkButtonEnabled {
// Talk mode lives on a side bubble so it doesn't get buried in settings.
OverlayButton(
systemImage: self.appModel.talkMode.isEnabled ? "waveform.circle.fill" : "waveform.circle",
brighten: self.brightenButtons,
tint: self.appModel.seamColor,
isActive: self.appModel.talkMode.isEnabled)
{
let next = !self.appModel.talkMode.isEnabled
self.talkEnabled = next
self.appModel.setTalkEnabled(next)
}
.accessibilityLabel("Talk Mode")
}
OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) {
self.openSettings()
}
.accessibilityLabel("Settings")
}
.padding(.top, 10)
.padding(.trailing, 10)
}
.overlay(alignment: .center) {
if self.appModel.talkMode.isEnabled {
TalkOrbOverlay()
.transition(.opacity)
}
}
.overlay(alignment: .topLeading) {
StatusPill(
gateway: self.gatewayStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
activity: self.statusActivity,
brighten: self.brightenButtons,
onTap: {
self.openSettings()
})
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
}
.overlay(alignment: .topLeading) {
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
VoiceWakeToast(
command: voiceWakeToastText,
brighten: self.brightenButtons)
.padding(.leading, 10)
.safeAreaPadding(.top, 58)
.transition(.move(edge: .top).combined(with: .opacity))
}
}
}
private var statusActivity: StatusPill.Activity? {
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let gatewayStatus = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let gatewayLower = gatewayStatus.lowercased()
if gatewayLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if gatewayLower.contains("approval") || gatewayLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
// Avoid duplicating the primary gateway status ("Connecting") in the activity slot.
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
}
private struct OverlayButton: View {
let systemImage: String
let brighten: Bool
var tint: Color?
var isActive: Bool = false
let action: () -> Void
var body: some View {
Button(action: self.action) {
Image(systemName: self.systemImage)
.font(.system(size: 16, weight: .semibold))
.foregroundStyle(self.isActive ? (self.tint ?? .primary) : .primary)
.padding(10)
.background {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.fill(
LinearGradient(
colors: [
.white.opacity(self.brighten ? 0.26 : 0.18),
.white.opacity(self.brighten ? 0.08 : 0.04),
.clear,
],
startPoint: .topLeading,
endPoint: .bottomTrailing))
.blendMode(.overlay)
}
.overlay {
if let tint {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.fill(
LinearGradient(
colors: [
tint.opacity(self.isActive ? 0.22 : 0.14),
tint.opacity(self.isActive ? 0.10 : 0.06),
.clear,
],
startPoint: .topLeading,
endPoint: .bottomTrailing))
.blendMode(.overlay)
}
}
.overlay {
RoundedRectangle(cornerRadius: 12, style: .continuous)
.strokeBorder(
(self.tint ?? .white).opacity(self.isActive ? 0.34 : (self.brighten ? 0.24 : 0.18)),
lineWidth: self.isActive ? 0.7 : 0.5)
}
.shadow(color: .black.opacity(0.35), radius: 12, y: 6)
}
}
.buttonStyle(.plain)
}
}
private struct CameraFlashOverlay: View {
var nonce: Int
@State private var opacity: CGFloat = 0
@State private var task: Task<Void, Never>?
var body: some View {
Color.white
.opacity(self.opacity)
.ignoresSafeArea()
.allowsHitTesting(false)
.onChange(of: self.nonce) { _, _ in
self.task?.cancel()
self.task = Task { @MainActor in
withAnimation(.easeOut(duration: 0.08)) {
self.opacity = 0.85
}
try? await Task.sleep(nanoseconds: 110_000_000)
withAnimation(.easeOut(duration: 0.32)) {
self.opacity = 0
}
}
}
}
}

View File

@@ -0,0 +1,143 @@
import SwiftUI
struct RootTabs: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
@State private var selectedTab: Int = 0
@State private var voiceWakeToastText: String?
@State private var toastDismissTask: Task<Void, Never>?
var body: some View {
TabView(selection: self.$selectedTab) {
ScreenTab()
.tabItem { Label("Screen", systemImage: "rectangle.and.hand.point.up.left") }
.tag(0)
VoiceTab()
.tabItem { Label("Voice", systemImage: "mic") }
.tag(1)
SettingsTab()
.tabItem { Label("Settings", systemImage: "gearshape") }
.tag(2)
}
.overlay(alignment: .topLeading) {
StatusPill(
gateway: self.gatewayStatus,
voiceWakeEnabled: self.voiceWakeEnabled,
activity: self.statusActivity,
onTap: { self.selectedTab = 2 })
.padding(.leading, 10)
.safeAreaPadding(.top, 10)
}
.overlay(alignment: .topLeading) {
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
VoiceWakeToast(command: voiceWakeToastText)
.padding(.leading, 10)
.safeAreaPadding(.top, 58)
.transition(.move(edge: .top).combined(with: .opacity))
}
}
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
guard let newValue else { return }
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
self.toastDismissTask?.cancel()
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
self.voiceWakeToastText = trimmed
}
self.toastDismissTask = Task {
try? await Task.sleep(nanoseconds: 2_300_000_000)
await MainActor.run {
withAnimation(.easeOut(duration: 0.25)) {
self.voiceWakeToastText = nil
}
}
}
}
.onDisappear {
self.toastDismissTask?.cancel()
self.toastDismissTask = nil
}
}
private var gatewayStatus: StatusPill.GatewayState {
if self.appModel.gatewayServerName != nil { return .connected }
let text = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
if text.localizedCaseInsensitiveContains("connecting") ||
text.localizedCaseInsensitiveContains("reconnecting")
{
return .connecting
}
if text.localizedCaseInsensitiveContains("error") {
return .error
}
return .disconnected
}
private var statusActivity: StatusPill.Activity? {
// Keep the top pill consistent across tabs (camera + voice wake + pairing states).
if self.appModel.isBackgrounded {
return StatusPill.Activity(
title: "Foreground required",
systemImage: "exclamationmark.triangle.fill",
tint: .orange)
}
let gatewayStatus = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
let gatewayLower = gatewayStatus.lowercased()
if gatewayLower.contains("repair") {
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
}
if gatewayLower.contains("approval") || gatewayLower.contains("pairing") {
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
}
// Avoid duplicating the primary gateway status ("Connecting") in the activity slot.
if self.appModel.screenRecordActive {
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
}
if let cameraHUDText = self.appModel.cameraHUDText,
let cameraHUDKind = self.appModel.cameraHUDKind,
!cameraHUDText.isEmpty
{
let systemImage: String
let tint: Color?
switch cameraHUDKind {
case .photo:
systemImage = "camera.fill"
tint = nil
case .recording:
systemImage = "video.fill"
tint = .red
case .success:
systemImage = "checkmark.circle.fill"
tint = .green
case .error:
systemImage = "exclamationmark.triangle.fill"
tint = .red
}
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
}
if self.voiceWakeEnabled {
let voiceStatus = self.appModel.voiceWake.statusText
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
}
if voiceStatus == "Paused" {
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
}
}
return nil
}
}

View File

@@ -0,0 +1,411 @@
import MoltbotKit
import Observation
import SwiftUI
import WebKit
@MainActor
@Observable
final class ScreenController {
let webView: WKWebView
private let navigationDelegate: ScreenNavigationDelegate
private let a2uiActionHandler: CanvasA2UIActionMessageHandler
var urlString: String = ""
var errorText: String?
/// Callback invoked when a moltbot:// deep link is tapped in the canvas
var onDeepLink: ((URL) -> Void)?
/// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI.
var onA2UIAction: (([String: Any]) -> Void)?
private var debugStatusEnabled: Bool = false
private var debugStatusTitle: String?
private var debugStatusSubtitle: String?
init() {
let config = WKWebViewConfiguration()
config.websiteDataStore = .nonPersistent()
let a2uiActionHandler = CanvasA2UIActionMessageHandler()
let userContentController = WKUserContentController()
for name in CanvasA2UIActionMessageHandler.handlerNames {
userContentController.add(a2uiActionHandler, name: name)
}
config.userContentController = userContentController
self.navigationDelegate = ScreenNavigationDelegate()
self.a2uiActionHandler = a2uiActionHandler
self.webView = WKWebView(frame: .zero, configuration: config)
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
self.webView.isOpaque = true
self.webView.backgroundColor = .black
self.webView.scrollView.backgroundColor = .black
self.webView.scrollView.contentInsetAdjustmentBehavior = .never
self.webView.scrollView.contentInset = .zero
self.webView.scrollView.scrollIndicatorInsets = .zero
self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false
self.applyScrollBehavior()
self.webView.navigationDelegate = self.navigationDelegate
self.navigationDelegate.controller = self
a2uiActionHandler.controller = self
self.reload()
}
func navigate(to urlString: String) {
let trimmed = urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.urlString = (trimmed == "/" ? "" : trimmed)
self.reload()
}
func reload() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
self.applyScrollBehavior()
if trimmed.isEmpty {
guard let url = Self.canvasScaffoldURL else { return }
self.errorText = nil
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
return
} else {
guard let url = URL(string: trimmed) else {
self.errorText = "Invalid URL: \(trimmed)"
return
}
self.errorText = nil
if url.isFileURL {
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
} else {
self.webView.load(URLRequest(url: url))
}
}
}
func showDefaultCanvas() {
self.urlString = ""
self.reload()
}
func setDebugStatusEnabled(_ enabled: Bool) {
self.debugStatusEnabled = enabled
self.applyDebugStatusIfNeeded()
}
func updateDebugStatus(title: String?, subtitle: String?) {
self.debugStatusTitle = title
self.debugStatusSubtitle = subtitle
self.applyDebugStatusIfNeeded()
}
fileprivate func applyDebugStatusIfNeeded() {
let enabled = self.debugStatusEnabled
let title = self.debugStatusTitle
let subtitle = self.debugStatusSubtitle
let js = """
(() => {
try {
const api = globalThis.__moltbot;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(\(enabled ? "true" : "false"));
}
if (!\(enabled ? "true" : "false")) return;
if (typeof api.setStatus === 'function') {
api.setStatus(\(Self.jsValue(title)), \(Self.jsValue(subtitle)));
}
} catch (_) {}
})()
"""
self.webView.evaluateJavaScript(js) { _, _ in }
}
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
let clock = ContinuousClock()
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
while clock.now < deadline {
do {
let res = try await self.eval(javaScript: """
(() => {
try {
return !!globalThis.clawdbotA2UI && typeof globalThis.clawdbotA2UI.applyMessages === 'function';
} catch (_) { return false; }
})()
""")
let trimmed = res.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
if trimmed == "true" || trimmed == "1" { return true }
} catch {
// ignore; page likely still loading
}
try? await Task.sleep(nanoseconds: 120_000_000)
}
return false
}
func eval(javaScript: String) async throws -> String {
try await withCheckedThrowingContinuation { cont in
self.webView.evaluateJavaScript(javaScript) { result, error in
if let error {
cont.resume(throwing: error)
return
}
if let result {
cont.resume(returning: String(describing: result))
} else {
cont.resume(returning: "")
}
}
}
}
func snapshotPNGBase64(maxWidth: CGFloat? = nil) async throws -> String {
let config = WKSnapshotConfiguration()
if let maxWidth {
config.snapshotWidth = NSNumber(value: Double(maxWidth))
}
let image: UIImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: config) { image, error in
if let error {
cont.resume(throwing: error)
return
}
guard let image else {
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
NSLocalizedDescriptionKey: "snapshot failed",
]))
return
}
cont.resume(returning: image)
}
}
guard let data = image.pngData() else {
throw NSError(domain: "Screen", code: 1, userInfo: [
NSLocalizedDescriptionKey: "snapshot encode failed",
])
}
return data.base64EncodedString()
}
func snapshotBase64(
maxWidth: CGFloat? = nil,
format: MoltbotCanvasSnapshotFormat,
quality: Double? = nil) async throws -> String
{
let config = WKSnapshotConfiguration()
if let maxWidth {
config.snapshotWidth = NSNumber(value: Double(maxWidth))
}
let image: UIImage = try await withCheckedThrowingContinuation { cont in
self.webView.takeSnapshot(with: config) { image, error in
if let error {
cont.resume(throwing: error)
return
}
guard let image else {
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
NSLocalizedDescriptionKey: "snapshot failed",
]))
return
}
cont.resume(returning: image)
}
}
let data: Data?
switch format {
case .png:
data = image.pngData()
case .jpeg:
let q = (quality ?? 0.82).clamped(to: 0.1...1.0)
data = image.jpegData(compressionQuality: q)
}
guard let data else {
throw NSError(domain: "Screen", code: 1, userInfo: [
NSLocalizedDescriptionKey: "snapshot encode failed",
])
}
return data.base64EncodedString()
}
private static func bundledResourceURL(
name: String,
ext: String,
subdirectory: String)
-> URL?
{
let bundle = MoltbotKitResources.bundle
return bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory)
?? bundle.url(forResource: name, withExtension: ext)
}
private static let canvasScaffoldURL: URL? = ScreenController.bundledResourceURL(
name: "scaffold",
ext: "html",
subdirectory: "CanvasScaffold")
func isTrustedCanvasUIURL(_ url: URL) -> Bool {
guard url.isFileURL else { return false }
let std = url.standardizedFileURL
if let expected = Self.canvasScaffoldURL,
std == expected.standardizedFileURL
{
return true
}
return false
}
private func applyScrollBehavior() {
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
let allowScroll = !trimmed.isEmpty
let scrollView = self.webView.scrollView
// Default canvas needs raw touch events; external pages should scroll.
scrollView.isScrollEnabled = allowScroll
scrollView.bounces = allowScroll
}
private static func jsValue(_ value: String?) -> String {
guard let value else { return "null" }
if let data = try? JSONSerialization.data(withJSONObject: [value]),
let encoded = String(data: data, encoding: .utf8),
encoded.count >= 2
{
return String(encoded.dropFirst().dropLast())
}
return "null"
}
func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
return false
}
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
return false
}
if host == "localhost" { return true }
if host.hasSuffix(".local") { return true }
if host.hasSuffix(".ts.net") { return true }
if host.hasSuffix(".tailscale.net") { return true }
// Allow MagicDNS / LAN hostnames like "peters-mac-studio-1".
if !host.contains("."), !host.contains(":") { return true }
if let ipv4 = Self.parseIPv4(host) {
return Self.isLocalNetworkIPv4(ipv4)
}
return false
}
private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 4 else { return nil }
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
guard bytes.count == 4 else { return nil }
return (bytes[0], bytes[1], bytes[2], bytes[3])
}
private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
let (a, b, _, _) = ip
// 10.0.0.0/8
if a == 10 { return true }
// 172.16.0.0/12
if a == 172, (16...31).contains(Int(b)) { return true }
// 192.168.0.0/16
if a == 192, b == 168 { return true }
// 127.0.0.0/8
if a == 127 { return true }
// 169.254.0.0/16 (link-local)
if a == 169, b == 254 { return true }
// Tailscale: 100.64.0.0/10
if a == 100, (64...127).contains(Int(b)) { return true }
return false
}
nonisolated static func parseA2UIActionBody(_ body: Any) -> [String: Any]? {
if let dict = body as? [String: Any] { return dict.isEmpty ? nil : dict }
if let str = body as? String,
let data = str.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
{
return json.isEmpty ? nil : json
}
if let dict = body as? [AnyHashable: Any] {
let mapped = dict.reduce(into: [String: Any]()) { acc, pair in
guard let key = pair.key as? String else { return }
acc[key] = pair.value
}
return mapped.isEmpty ? nil : mapped
}
return nil
}
}
extension Double {
fileprivate func clamped(to range: ClosedRange<Double>) -> Double {
if self < range.lowerBound { return range.lowerBound }
if self > range.upperBound { return range.upperBound }
return self
}
}
// MARK: - Navigation Delegate
/// Handles navigation policy to intercept moltbot:// deep links from canvas
@MainActor
private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
weak var controller: ScreenController?
func webView(
_ webView: WKWebView,
decidePolicyFor navigationAction: WKNavigationAction,
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
{
guard let url = navigationAction.request.url else {
decisionHandler(.allow)
return
}
// Intercept moltbot:// deep links
if url.scheme == "moltbot" {
decisionHandler(.cancel)
self.controller?.onDeepLink?(url)
return
}
decisionHandler(.allow)
}
func webView(
_: WKWebView,
didFailProvisionalNavigation _: WKNavigation?,
withError error: any Error)
{
self.controller?.errorText = error.localizedDescription
}
func webView(_: WKWebView, didFinish _: WKNavigation?) {
self.controller?.errorText = nil
self.controller?.applyDebugStatusIfNeeded()
}
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
self.controller?.errorText = error.localizedDescription
}
}
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
static let messageName = "moltbotCanvasA2UIAction"
static let legacyMessageNames = ["canvas", "a2ui", "userAction", "action"]
static let handlerNames = [messageName] + legacyMessageNames
weak var controller: ScreenController?
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
guard message.name == Self.messageName else { return }
guard let controller else { return }
guard let url = message.webView?.url else { return }
if url.isFileURL {
guard controller.isTrustedCanvasUIURL(url) else { return }
} else {
// For security, only accept actions from local-network pages (e.g. the canvas host).
guard controller.isLocalNetworkCanvasURL(url) else { return }
}
guard let body = ScreenController.parseA2UIActionBody(message.body) else { return }
controller.onA2UIAction?(body)
}
}

View File

@@ -0,0 +1,360 @@
import AVFoundation
import ReplayKit
final class ScreenRecordService: @unchecked Sendable {
private struct UncheckedSendableBox<T>: @unchecked Sendable {
let value: T
}
private final class CaptureState: @unchecked Sendable {
private let lock = NSLock()
var writer: AVAssetWriter?
var videoInput: AVAssetWriterInput?
var audioInput: AVAssetWriterInput?
var started = false
var sawVideo = false
var lastVideoTime: CMTime?
var handlerError: Error?
func withLock<T>(_ body: (CaptureState) -> T) -> T {
self.lock.lock()
defer { lock.unlock() }
return body(self)
}
}
enum ScreenRecordError: LocalizedError {
case invalidScreenIndex(Int)
case captureFailed(String)
case writeFailed(String)
var errorDescription: String? {
switch self {
case let .invalidScreenIndex(idx):
"Invalid screen index \(idx)"
case let .captureFailed(msg):
msg
case let .writeFailed(msg):
msg
}
}
}
func record(
screenIndex: Int?,
durationMs: Int?,
fps: Double?,
includeAudio: Bool?,
outPath: String?) async throws -> String
{
let config = try self.makeRecordConfig(
screenIndex: screenIndex,
durationMs: durationMs,
fps: fps,
includeAudio: includeAudio,
outPath: outPath)
let state = CaptureState()
let recordQueue = DispatchQueue(label: "bot.molt.screenrecord")
try await self.startCapture(state: state, config: config, recordQueue: recordQueue)
try await Task.sleep(nanoseconds: UInt64(config.durationMs) * 1_000_000)
try await self.stopCapture()
try self.finalizeCapture(state: state)
try await self.finishWriting(state: state)
return config.outURL.path
}
private struct RecordConfig {
let durationMs: Int
let fpsValue: Double
let includeAudio: Bool
let outURL: URL
}
private func makeRecordConfig(
screenIndex: Int?,
durationMs: Int?,
fps: Double?,
includeAudio: Bool?,
outPath: String?) throws -> RecordConfig
{
if let idx = screenIndex, idx != 0 {
throw ScreenRecordError.invalidScreenIndex(idx)
}
let durationMs = Self.clampDurationMs(durationMs)
let fps = Self.clampFps(fps)
let fpsInt = Int32(fps.rounded())
let fpsValue = Double(fpsInt)
let includeAudio = includeAudio ?? true
let outURL = self.makeOutputURL(outPath: outPath)
try? FileManager().removeItem(at: outURL)
return RecordConfig(
durationMs: durationMs,
fpsValue: fpsValue,
includeAudio: includeAudio,
outURL: outURL)
}
private func makeOutputURL(outPath: String?) -> URL {
if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return URL(fileURLWithPath: outPath)
}
return FileManager().temporaryDirectory
.appendingPathComponent("moltbot-screen-record-\(UUID().uuidString).mp4")
}
private func startCapture(
state: CaptureState,
config: RecordConfig,
recordQueue: DispatchQueue) async throws
{
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
let handler = self.makeCaptureHandler(
state: state,
config: config,
recordQueue: recordQueue)
let completion: @Sendable (Error?) -> Void = { error in
if let error { cont.resume(throwing: error) } else { cont.resume() }
}
Task { @MainActor in
startReplayKitCapture(
includeAudio: config.includeAudio,
handler: handler,
completion: completion)
}
}
}
private func makeCaptureHandler(
state: CaptureState,
config: RecordConfig,
recordQueue: DispatchQueue) -> @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void
{
{ sample, type, error in
let sampleBox = UncheckedSendableBox(value: sample)
// ReplayKit can call the capture handler on a background queue.
// Serialize writes to avoid queue asserts.
recordQueue.async {
let sample = sampleBox.value
if let error {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
return
}
guard CMSampleBufferDataIsReady(sample) else { return }
switch type {
case .video:
self.handleVideoSample(sample, state: state, config: config)
case .audioApp, .audioMic:
self.handleAudioSample(sample, state: state, includeAudio: config.includeAudio)
@unknown default:
break
}
}
}
}
private func handleVideoSample(
_ sample: CMSampleBuffer,
state: CaptureState,
config: RecordConfig)
{
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
let shouldSkip = state.withLock { state in
if let lastVideoTime = state.lastVideoTime {
let delta = CMTimeSubtract(pts, lastVideoTime)
return delta.seconds < (1.0 / config.fpsValue)
}
return false
}
if shouldSkip { return }
if state.withLock({ $0.writer == nil }) {
self.prepareWriter(sample: sample, state: state, config: config, pts: pts)
}
let vInput = state.withLock { $0.videoInput }
let isStarted = state.withLock { $0.started }
guard let vInput, isStarted else { return }
if vInput.isReadyForMoreMediaData {
if vInput.append(sample) {
state.withLock { state in
state.sawVideo = true
state.lastVideoTime = pts
}
} else {
let err = state.withLock { $0.writer?.error }
if let err {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.writeFailed(err.localizedDescription)
}
}
}
}
}
}
private func prepareWriter(
sample: CMSampleBuffer,
state: CaptureState,
config: RecordConfig,
pts: CMTime)
{
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
state.withLock { state in
if state.handlerError == nil {
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
}
}
return
}
let width = CVPixelBufferGetWidth(imageBuffer)
let height = CVPixelBufferGetHeight(imageBuffer)
do {
let writer = try AVAssetWriter(outputURL: config.outURL, fileType: .mp4)
let settings: [String: Any] = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoWidthKey: width,
AVVideoHeightKey: height,
]
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
vInput.expectsMediaDataInRealTime = true
guard writer.canAdd(vInput) else {
throw ScreenRecordError.writeFailed("Cannot add video input")
}
writer.add(vInput)
if config.includeAudio {
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
aInput.expectsMediaDataInRealTime = true
if writer.canAdd(aInput) {
writer.add(aInput)
state.withLock { state in
state.audioInput = aInput
}
}
}
guard writer.startWriting() else {
throw ScreenRecordError.writeFailed(
writer.error?.localizedDescription ?? "Failed to start writer")
}
writer.startSession(atSourceTime: pts)
state.withLock { state in
state.writer = writer
state.videoInput = vInput
state.started = true
}
} catch {
state.withLock { state in
if state.handlerError == nil { state.handlerError = error }
}
}
}
private func handleAudioSample(
_ sample: CMSampleBuffer,
state: CaptureState,
includeAudio: Bool)
{
let aInput = state.withLock { $0.audioInput }
let isStarted = state.withLock { $0.started }
guard includeAudio, let aInput, isStarted else { return }
if aInput.isReadyForMoreMediaData {
_ = aInput.append(sample)
}
}
private func stopCapture() async throws {
let stopError = await withCheckedContinuation { cont in
Task { @MainActor in
stopReplayKitCapture { error in cont.resume(returning: error) }
}
}
if let stopError { throw stopError }
}
private func finalizeCapture(state: CaptureState) throws {
if let handlerErrorSnapshot = state.withLock({ $0.handlerError }) {
throw handlerErrorSnapshot
}
let writerSnapshot = state.withLock { $0.writer }
let videoInputSnapshot = state.withLock { $0.videoInput }
let audioInputSnapshot = state.withLock { $0.audioInput }
let sawVideoSnapshot = state.withLock { $0.sawVideo }
guard let writerSnapshot, let videoInputSnapshot, sawVideoSnapshot else {
throw ScreenRecordError.captureFailed("No frames captured")
}
videoInputSnapshot.markAsFinished()
audioInputSnapshot?.markAsFinished()
_ = writerSnapshot
}
private func finishWriting(state: CaptureState) async throws {
guard let writerSnapshot = state.withLock({ $0.writer }) else {
throw ScreenRecordError.captureFailed("Missing writer")
}
let writerBox = UncheckedSendableBox(value: writerSnapshot)
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
writerBox.value.finishWriting {
let writer = writerBox.value
if let err = writer.error {
cont.resume(throwing: ScreenRecordError.writeFailed(err.localizedDescription))
} else if writer.status != .completed {
cont.resume(throwing: ScreenRecordError.writeFailed("Failed to finalize video"))
} else {
cont.resume()
}
}
}
}
private nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
let v = ms ?? 10000
return min(60000, max(250, v))
}
private nonisolated static func clampFps(_ fps: Double?) -> Double {
let v = fps ?? 10
if !v.isFinite { return 10 }
return min(30, max(1, v))
}
}
@MainActor
private func startReplayKitCapture(
includeAudio: Bool,
handler: @escaping @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void,
completion: @escaping @Sendable (Error?) -> Void)
{
let recorder = RPScreenRecorder.shared()
recorder.isMicrophoneEnabled = includeAudio
recorder.startCapture(handler: handler, completionHandler: completion)
}
@MainActor
private func stopReplayKitCapture(_ completion: @escaping @Sendable (Error?) -> Void) {
RPScreenRecorder.shared().stopCapture { error in completion(error) }
}
#if DEBUG
extension ScreenRecordService {
nonisolated static func _test_clampDurationMs(_ ms: Int?) -> Int {
self.clampDurationMs(ms)
}
nonisolated static func _test_clampFps(_ fps: Double?) -> Double {
self.clampFps(fps)
}
}
#endif

View File

@@ -0,0 +1,25 @@
import MoltbotKit
import SwiftUI
struct ScreenTab: View {
@Environment(NodeAppModel.self) private var appModel
var body: some View {
ZStack(alignment: .top) {
ScreenWebView(controller: self.appModel.screen)
.ignoresSafeArea()
.overlay(alignment: .top) {
if let errorText = self.appModel.screen.errorText {
Text(errorText)
.font(.footnote)
.padding(10)
.background(.thinMaterial)
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
.padding()
}
}
}
}
// Navigation is agent-driven; no local URL bar here.
}

View File

@@ -0,0 +1,15 @@
import MoltbotKit
import SwiftUI
import WebKit
struct ScreenWebView: UIViewRepresentable {
var controller: ScreenController
func makeUIView(context: Context) -> WKWebView {
self.controller.webView
}
func updateUIView(_ webView: WKWebView, context: Context) {
// State changes are driven by ScreenController.
}
}

View File

@@ -0,0 +1,15 @@
import Foundation
enum SessionKey {
static func normalizeMainKey(_ raw: String?) -> String {
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return trimmed.isEmpty ? "main" : trimmed
}
static func isCanonicalMainSessionKey(_ value: String?) -> Bool {
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
if trimmed.isEmpty { return false }
if trimmed == "global" { return true }
return trimmed.hasPrefix("agent:")
}
}

View File

@@ -0,0 +1,40 @@
import Foundation
struct SettingsHostPort: Equatable {
var host: String
var port: Int
}
enum SettingsNetworkingHelpers {
static func parseHostPort(from address: String) -> SettingsHostPort? {
let trimmed = address.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
if trimmed.hasPrefix("["),
let close = trimmed.firstIndex(of: "]"),
close < trimmed.endIndex
{
let host = String(trimmed[trimmed.index(after: trimmed.startIndex)..<close])
let portStart = trimmed.index(after: close)
guard portStart < trimmed.endIndex, trimmed[portStart] == ":" else { return nil }
let portString = String(trimmed[trimmed.index(after: portStart)...])
guard let port = Int(portString) else { return nil }
return SettingsHostPort(host: host, port: port)
}
guard let colon = trimmed.lastIndex(of: ":") else { return nil }
let host = String(trimmed[..<colon])
let portString = String(trimmed[trimmed.index(after: colon)...])
guard !host.isEmpty, let port = Int(portString) else { return nil }
return SettingsHostPort(host: host, port: port)
}
static func httpURLString(host: String?, port: Int?, fallback: String) -> String {
if let host, let port {
let needsBrackets = host.contains(":") && !host.hasPrefix("[") && !host.hasSuffix("]")
let hostPart = needsBrackets ? "[\(host)]" : host
return "http://\(hostPart):\(port)"
}
return "http://\(fallback)"
}
}

View File

@@ -0,0 +1,466 @@
import MoltbotKit
import Network
import Observation
import SwiftUI
import UIKit
@MainActor
@Observable
private final class ConnectStatusStore {
var text: String?
}
extension ConnectStatusStore: @unchecked Sendable {}
struct SettingsTab: View {
@Environment(NodeAppModel.self) private var appModel: NodeAppModel
@Environment(VoiceWakeManager.self) private var voiceWake: VoiceWakeManager
@Environment(GatewayConnectionController.self) private var gatewayController: GatewayConnectionController
@Environment(\.dismiss) private var dismiss
@AppStorage("node.displayName") private var displayName: String = "iOS Node"
@AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
@AppStorage("camera.enabled") private var cameraEnabled: Bool = true
@AppStorage("location.enabledMode") private var locationEnabledModeRaw: String = MoltbotLocationMode.off.rawValue
@AppStorage("location.preciseEnabled") private var locationPreciseEnabled: Bool = true
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
@AppStorage("gateway.preferredStableID") private var preferredGatewayStableID: String = ""
@AppStorage("gateway.lastDiscoveredStableID") private var lastDiscoveredGatewayStableID: String = ""
@AppStorage("gateway.manual.enabled") private var manualGatewayEnabled: Bool = false
@AppStorage("gateway.manual.host") private var manualGatewayHost: String = ""
@AppStorage("gateway.manual.port") private var manualGatewayPort: Int = 18789
@AppStorage("gateway.manual.tls") private var manualGatewayTLS: Bool = true
@AppStorage("gateway.discovery.debugLogs") private var discoveryDebugLogsEnabled: Bool = false
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
@State private var connectStatus = ConnectStatusStore()
@State private var connectingGatewayID: String?
@State private var localIPAddress: String?
@State private var lastLocationModeRaw: String = MoltbotLocationMode.off.rawValue
@State private var gatewayToken: String = ""
@State private var gatewayPassword: String = ""
var body: some View {
NavigationStack {
Form {
Section("Node") {
TextField("Name", text: self.$displayName)
Text(self.instanceId)
.font(.footnote)
.foregroundStyle(.secondary)
LabeledContent("IP", value: self.localIPAddress ?? "")
.contextMenu {
if let ip = self.localIPAddress {
Button {
UIPasteboard.general.string = ip
} label: {
Label("Copy", systemImage: "doc.on.doc")
}
}
}
LabeledContent("Platform", value: self.platformString())
LabeledContent("Version", value: self.appVersion())
LabeledContent("Model", value: self.modelIdentifier())
}
Section("Gateway") {
LabeledContent("Discovery", value: self.gatewayController.discoveryStatusText)
LabeledContent("Status", value: self.appModel.gatewayStatusText)
if let serverName = self.appModel.gatewayServerName {
LabeledContent("Server", value: serverName)
if let addr = self.appModel.gatewayRemoteAddress {
let parts = Self.parseHostPort(from: addr)
let urlString = Self.httpURLString(host: parts?.host, port: parts?.port, fallback: addr)
LabeledContent("Address") {
Text(urlString)
}
.contextMenu {
Button {
UIPasteboard.general.string = urlString
} label: {
Label("Copy URL", systemImage: "doc.on.doc")
}
if let parts {
Button {
UIPasteboard.general.string = parts.host
} label: {
Label("Copy Host", systemImage: "doc.on.doc")
}
Button {
UIPasteboard.general.string = "\(parts.port)"
} label: {
Label("Copy Port", systemImage: "doc.on.doc")
}
}
}
}
Button("Disconnect", role: .destructive) {
self.appModel.disconnectGateway()
}
self.gatewayList(showing: .availableOnly)
} else {
self.gatewayList(showing: .all)
}
if let text = self.connectStatus.text {
Text(text)
.font(.footnote)
.foregroundStyle(.secondary)
}
DisclosureGroup("Advanced") {
Toggle("Use Manual Gateway", isOn: self.$manualGatewayEnabled)
TextField("Host", text: self.$manualGatewayHost)
.textInputAutocapitalization(.never)
.autocorrectionDisabled()
TextField("Port", value: self.$manualGatewayPort, format: .number)
.keyboardType(.numberPad)
Toggle("Use TLS", isOn: self.$manualGatewayTLS)
Button {
Task { await self.connectManual() }
} label: {
if self.connectingGatewayID == "manual" {
HStack(spacing: 8) {
ProgressView()
.progressViewStyle(.circular)
Text("Connecting…")
}
} else {
Text("Connect (Manual)")
}
}
.disabled(self.connectingGatewayID != nil || self.manualGatewayHost
.trimmingCharacters(in: .whitespacesAndNewlines)
.isEmpty || self.manualGatewayPort <= 0 || self.manualGatewayPort > 65535)
Text(
"Use this when mDNS/Bonjour discovery is blocked. "
+ "The gateway WebSocket listens on port 18789 by default.")
.font(.footnote)
.foregroundStyle(.secondary)
Toggle("Discovery Debug Logs", isOn: self.$discoveryDebugLogsEnabled)
.onChange(of: self.discoveryDebugLogsEnabled) { _, newValue in
self.gatewayController.setDiscoveryDebugLoggingEnabled(newValue)
}
NavigationLink("Discovery Logs") {
GatewayDiscoveryDebugLogView()
}
Toggle("Debug Canvas Status", isOn: self.$canvasDebugStatusEnabled)
TextField("Gateway Token", text: self.$gatewayToken)
.textInputAutocapitalization(.never)
.autocorrectionDisabled()
SecureField("Gateway Password", text: self.$gatewayPassword)
}
}
Section("Voice") {
Toggle("Voice Wake", isOn: self.$voiceWakeEnabled)
.onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue)
}
Toggle("Talk Mode", isOn: self.$talkEnabled)
.onChange(of: self.talkEnabled) { _, newValue in
self.appModel.setTalkEnabled(newValue)
}
// Keep this separate so users can hide the side bubble without disabling Talk Mode.
Toggle("Show Talk Button", isOn: self.$talkButtonEnabled)
NavigationLink {
VoiceWakeWordsSettingsView()
} label: {
LabeledContent(
"Wake Words",
value: VoiceWakePreferences.displayString(for: self.voiceWake.triggerWords))
}
}
Section("Camera") {
Toggle("Allow Camera", isOn: self.$cameraEnabled)
Text("Allows the gateway to request photos or short video clips (foreground only).")
.font(.footnote)
.foregroundStyle(.secondary)
}
Section("Location") {
Picker("Location Access", selection: self.$locationEnabledModeRaw) {
Text("Off").tag(MoltbotLocationMode.off.rawValue)
Text("While Using").tag(MoltbotLocationMode.whileUsing.rawValue)
Text("Always").tag(MoltbotLocationMode.always.rawValue)
}
.pickerStyle(.segmented)
Toggle("Precise Location", isOn: self.$locationPreciseEnabled)
.disabled(self.locationMode == .off)
Text("Always requires system permission and may prompt to open Settings.")
.font(.footnote)
.foregroundStyle(.secondary)
}
Section("Screen") {
Toggle("Prevent Sleep", isOn: self.$preventSleep)
Text("Keeps the screen awake while Moltbot is open.")
.font(.footnote)
.foregroundStyle(.secondary)
}
}
.navigationTitle("Settings")
.toolbar {
ToolbarItem(placement: .topBarTrailing) {
Button {
self.dismiss()
} label: {
Image(systemName: "xmark")
}
.accessibilityLabel("Close")
}
}
.onAppear {
self.localIPAddress = Self.primaryIPv4Address()
self.lastLocationModeRaw = self.locationEnabledModeRaw
let trimmedInstanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmedInstanceId.isEmpty {
self.gatewayToken = GatewaySettingsStore.loadGatewayToken(instanceId: trimmedInstanceId) ?? ""
self.gatewayPassword = GatewaySettingsStore.loadGatewayPassword(instanceId: trimmedInstanceId) ?? ""
}
}
.onChange(of: self.preferredGatewayStableID) { _, newValue in
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
GatewaySettingsStore.savePreferredGatewayStableID(trimmed)
}
.onChange(of: self.gatewayToken) { _, newValue in
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
let instanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines)
guard !instanceId.isEmpty else { return }
GatewaySettingsStore.saveGatewayToken(trimmed, instanceId: instanceId)
}
.onChange(of: self.gatewayPassword) { _, newValue in
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
let instanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines)
guard !instanceId.isEmpty else { return }
GatewaySettingsStore.saveGatewayPassword(trimmed, instanceId: instanceId)
}
.onChange(of: self.appModel.gatewayServerName) { _, _ in
self.connectStatus.text = nil
}
.onChange(of: self.locationEnabledModeRaw) { _, newValue in
let previous = self.lastLocationModeRaw
self.lastLocationModeRaw = newValue
guard let mode = MoltbotLocationMode(rawValue: newValue) else { return }
Task {
let granted = await self.appModel.requestLocationPermissions(mode: mode)
if !granted {
await MainActor.run {
self.locationEnabledModeRaw = previous
self.lastLocationModeRaw = previous
}
}
}
}
}
}
@ViewBuilder
private func gatewayList(showing: GatewayListMode) -> some View {
if self.gatewayController.gateways.isEmpty {
Text("No gateways found yet.")
.foregroundStyle(.secondary)
} else {
let connectedID = self.appModel.connectedGatewayID
let rows = self.gatewayController.gateways.filter { gateway in
let isConnected = gateway.stableID == connectedID
switch showing {
case .all:
return true
case .availableOnly:
return !isConnected
}
}
if rows.isEmpty, showing == .availableOnly {
Text("No other gateways found.")
.foregroundStyle(.secondary)
} else {
ForEach(rows) { gateway in
HStack {
VStack(alignment: .leading, spacing: 2) {
Text(gateway.name)
let detailLines = self.gatewayDetailLines(gateway)
ForEach(detailLines, id: \.self) { line in
Text(line)
.font(.footnote)
.foregroundStyle(.secondary)
}
}
Spacer()
Button {
Task { await self.connect(gateway) }
} label: {
if self.connectingGatewayID == gateway.id {
ProgressView()
.progressViewStyle(.circular)
} else {
Text("Connect")
}
}
.disabled(self.connectingGatewayID != nil)
}
}
}
}
}
private enum GatewayListMode: Equatable {
case all
case availableOnly
}
private func platformString() -> String {
let v = ProcessInfo.processInfo.operatingSystemVersion
return "iOS \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
}
private var locationMode: MoltbotLocationMode {
MoltbotLocationMode(rawValue: self.locationEnabledModeRaw) ?? .off
}
private func appVersion() -> String {
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
}
private func deviceFamily() -> String {
switch UIDevice.current.userInterfaceIdiom {
case .pad:
"iPad"
case .phone:
"iPhone"
default:
"iOS"
}
}
private func modelIdentifier() -> String {
var systemInfo = utsname()
uname(&systemInfo)
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8)
}
let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? "unknown" : trimmed
}
private func connect(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) async {
self.connectingGatewayID = gateway.id
self.manualGatewayEnabled = false
self.preferredGatewayStableID = gateway.stableID
GatewaySettingsStore.savePreferredGatewayStableID(gateway.stableID)
self.lastDiscoveredGatewayStableID = gateway.stableID
GatewaySettingsStore.saveLastDiscoveredGatewayStableID(gateway.stableID)
defer { self.connectingGatewayID = nil }
await self.gatewayController.connect(gateway)
}
private func connectManual() async {
let host = self.manualGatewayHost.trimmingCharacters(in: .whitespacesAndNewlines)
guard !host.isEmpty else {
self.connectStatus.text = "Failed: host required"
return
}
guard self.manualGatewayPort > 0, self.manualGatewayPort <= 65535 else {
self.connectStatus.text = "Failed: invalid port"
return
}
self.connectingGatewayID = "manual"
self.manualGatewayEnabled = true
defer { self.connectingGatewayID = nil }
await self.gatewayController.connectManual(
host: host,
port: self.manualGatewayPort,
useTLS: self.manualGatewayTLS)
}
private static func primaryIPv4Address() -> String? {
var addrList: UnsafeMutablePointer<ifaddrs>?
guard getifaddrs(&addrList) == 0, let first = addrList else { return nil }
defer { freeifaddrs(addrList) }
var fallback: String?
var en0: String?
for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) {
let flags = Int32(ptr.pointee.ifa_flags)
let isUp = (flags & IFF_UP) != 0
let isLoopback = (flags & IFF_LOOPBACK) != 0
let name = String(cString: ptr.pointee.ifa_name)
let family = ptr.pointee.ifa_addr.pointee.sa_family
if !isUp || isLoopback || family != UInt8(AF_INET) { continue }
var addr = ptr.pointee.ifa_addr.pointee
var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST))
let result = getnameinfo(
&addr,
socklen_t(ptr.pointee.ifa_addr.pointee.sa_len),
&buffer,
socklen_t(buffer.count),
nil,
0,
NI_NUMERICHOST)
guard result == 0 else { continue }
let len = buffer.prefix { $0 != 0 }
let bytes = len.map { UInt8(bitPattern: $0) }
guard let ip = String(bytes: bytes, encoding: .utf8) else { continue }
if name == "en0" { en0 = ip; break }
if fallback == nil { fallback = ip }
}
return en0 ?? fallback
}
private static func parseHostPort(from address: String) -> SettingsHostPort? {
SettingsNetworkingHelpers.parseHostPort(from: address)
}
private static func httpURLString(host: String?, port: Int?, fallback: String) -> String {
SettingsNetworkingHelpers.httpURLString(host: host, port: port, fallback: fallback)
}
private func gatewayDetailLines(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> [String] {
var lines: [String] = []
if let lanHost = gateway.lanHost { lines.append("LAN: \(lanHost)") }
if let tailnet = gateway.tailnetDns { lines.append("Tailnet: \(tailnet)") }
let gatewayPort = gateway.gatewayPort
let canvasPort = gateway.canvasPort
if gatewayPort != nil || canvasPort != nil {
let gw = gatewayPort.map(String.init) ?? ""
let canvas = canvasPort.map(String.init) ?? ""
lines.append("Ports: gateway \(gw) · canvas \(canvas)")
}
if lines.isEmpty {
lines.append(gateway.debugID)
}
return lines
}
}

View File

@@ -0,0 +1,98 @@
import SwiftUI
import Combine
struct VoiceWakeWordsSettingsView: View {
@Environment(NodeAppModel.self) private var appModel
@State private var triggerWords: [String] = VoiceWakePreferences.loadTriggerWords()
@FocusState private var focusedTriggerIndex: Int?
@State private var syncTask: Task<Void, Never>?
var body: some View {
Form {
Section {
ForEach(self.triggerWords.indices, id: \.self) { index in
TextField("Wake word", text: self.binding(for: index))
.textInputAutocapitalization(.never)
.autocorrectionDisabled()
.focused(self.$focusedTriggerIndex, equals: index)
.onSubmit {
self.commitTriggerWords()
}
}
.onDelete(perform: self.removeWords)
Button {
self.addWord()
} label: {
Label("Add word", systemImage: "plus")
}
.disabled(self.triggerWords
.contains(where: { $0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }))
Button("Reset defaults") {
self.triggerWords = VoiceWakePreferences.defaultTriggerWords
}
} header: {
Text("Wake Words")
} footer: {
Text(
"Moltbot reacts when any trigger appears in a transcription. "
+ "Keep them short to avoid false positives.")
}
}
.navigationTitle("Wake Words")
.toolbar { EditButton() }
.onAppear {
if self.triggerWords.isEmpty {
self.triggerWords = VoiceWakePreferences.defaultTriggerWords
self.commitTriggerWords()
}
}
.onChange(of: self.focusedTriggerIndex) { oldValue, newValue in
guard oldValue != nil, oldValue != newValue else { return }
self.commitTriggerWords()
}
.onReceive(NotificationCenter.default.publisher(for: UserDefaults.didChangeNotification)) { _ in
guard self.focusedTriggerIndex == nil else { return }
let updated = VoiceWakePreferences.loadTriggerWords()
if updated != self.triggerWords {
self.triggerWords = updated
}
}
}
private func addWord() {
self.triggerWords.append("")
}
private func removeWords(at offsets: IndexSet) {
self.triggerWords.remove(atOffsets: offsets)
if self.triggerWords.isEmpty {
self.triggerWords = VoiceWakePreferences.defaultTriggerWords
}
self.commitTriggerWords()
}
private func binding(for index: Int) -> Binding<String> {
Binding(
get: {
guard self.triggerWords.indices.contains(index) else { return "" }
return self.triggerWords[index]
},
set: { newValue in
guard self.triggerWords.indices.contains(index) else { return }
self.triggerWords[index] = newValue
})
}
private func commitTriggerWords() {
VoiceWakePreferences.saveTriggerWords(self.triggerWords)
let snapshot = VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords)
self.syncTask?.cancel()
self.syncTask = Task { [snapshot, weak appModel = self.appModel] in
try? await Task.sleep(nanoseconds: 650_000_000)
await appModel?.setGlobalWakeWords(snapshot)
}
}
}

View File

@@ -0,0 +1,127 @@
import SwiftUI
struct StatusPill: View {
@Environment(\.scenePhase) private var scenePhase
enum GatewayState: Equatable {
case connected
case connecting
case error
case disconnected
var title: String {
switch self {
case .connected: "Connected"
case .connecting: "Connecting…"
case .error: "Error"
case .disconnected: "Offline"
}
}
var color: Color {
switch self {
case .connected: .green
case .connecting: .yellow
case .error: .red
case .disconnected: .gray
}
}
}
struct Activity: Equatable {
var title: String
var systemImage: String
var tint: Color?
}
var gateway: GatewayState
var voiceWakeEnabled: Bool
var activity: Activity?
var brighten: Bool = false
var onTap: () -> Void
@State private var pulse: Bool = false
var body: some View {
Button(action: self.onTap) {
HStack(spacing: 10) {
HStack(spacing: 8) {
Circle()
.fill(self.gateway.color)
.frame(width: 9, height: 9)
.scaleEffect(self.gateway == .connecting ? (self.pulse ? 1.15 : 0.85) : 1.0)
.opacity(self.gateway == .connecting ? (self.pulse ? 1.0 : 0.6) : 1.0)
Text(self.gateway.title)
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(.primary)
}
Divider()
.frame(height: 14)
.opacity(0.35)
if let activity {
HStack(spacing: 6) {
Image(systemName: activity.systemImage)
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(activity.tint ?? .primary)
Text(activity.title)
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
}
.transition(.opacity.combined(with: .move(edge: .top)))
} else {
Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash")
.font(.system(size: 13, weight: .semibold))
.foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary)
.accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled")
.transition(.opacity.combined(with: .move(edge: .top)))
}
}
.padding(.vertical, 8)
.padding(.horizontal, 12)
.background {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
}
.buttonStyle(.plain)
.accessibilityLabel("Status")
.accessibilityValue(self.accessibilityValue)
.onAppear { self.updatePulse(for: self.gateway, scenePhase: self.scenePhase) }
.onDisappear { self.pulse = false }
.onChange(of: self.gateway) { _, newValue in
self.updatePulse(for: newValue, scenePhase: self.scenePhase)
}
.onChange(of: self.scenePhase) { _, newValue in
self.updatePulse(for: self.gateway, scenePhase: newValue)
}
.animation(.easeInOut(duration: 0.18), value: self.activity?.title)
}
private var accessibilityValue: String {
if let activity {
return "\(self.gateway.title), \(activity.title)"
}
return "\(self.gateway.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")"
}
private func updatePulse(for gateway: GatewayState, scenePhase: ScenePhase) {
guard gateway == .connecting, scenePhase == .active else {
withAnimation(.easeOut(duration: 0.2)) { self.pulse = false }
return
}
guard !self.pulse else { return }
withAnimation(.easeInOut(duration: 0.9).repeatForever(autoreverses: true)) {
self.pulse = true
}
}
}

View File

@@ -0,0 +1,33 @@
import SwiftUI
struct VoiceWakeToast: View {
var command: String
var brighten: Bool = false
var body: some View {
HStack(spacing: 10) {
Image(systemName: "mic.fill")
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
Text(self.command)
.font(.system(size: 14, weight: .semibold))
.foregroundStyle(.primary)
.lineLimit(1)
.truncationMode(.tail)
}
.padding(.vertical, 10)
.padding(.horizontal, 12)
.background {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.fill(.ultraThinMaterial)
.overlay {
RoundedRectangle(cornerRadius: 14, style: .continuous)
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
}
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
}
.accessibilityLabel("Voice Wake")
.accessibilityValue(self.command)
}
}

View File

@@ -0,0 +1,729 @@
import AVFAudio
import MoltbotKit
import MoltbotProtocol
import Foundation
import Observation
import OSLog
import Speech
@MainActor
@Observable
final class TalkModeManager: NSObject {
private typealias SpeechRequest = SFSpeechAudioBufferRecognitionRequest
private static let defaultModelIdFallback = "eleven_v3"
var isEnabled: Bool = false
var isListening: Bool = false
var isSpeaking: Bool = false
var statusText: String = "Off"
private let audioEngine = AVAudioEngine()
private var speechRecognizer: SFSpeechRecognizer?
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
private var recognitionTask: SFSpeechRecognitionTask?
private var silenceTask: Task<Void, Never>?
private var lastHeard: Date?
private var lastTranscript: String = ""
private var lastSpokenText: String?
private var lastInterruptedAtSeconds: Double?
private var defaultVoiceId: String?
private var currentVoiceId: String?
private var defaultModelId: String?
private var currentModelId: String?
private var voiceOverrideActive = false
private var modelOverrideActive = false
private var defaultOutputFormat: String?
private var apiKey: String?
private var voiceAliases: [String: String] = [:]
private var interruptOnSpeech: Bool = true
private var mainSessionKey: String = "main"
private var fallbackVoiceId: String?
private var lastPlaybackWasPCM: Bool = false
var pcmPlayer: PCMStreamingAudioPlaying = PCMStreamingAudioPlayer.shared
var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared
private var gateway: GatewayNodeSession?
private let silenceWindow: TimeInterval = 0.7
private var chatSubscribedSessionKeys = Set<String>()
private let logger = Logger(subsystem: "bot.molt", category: "TalkMode")
func attachGateway(_ gateway: GatewayNodeSession) {
self.gateway = gateway
}
func updateMainSessionKey(_ sessionKey: String?) {
let trimmed = (sessionKey ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
if SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { return }
self.mainSessionKey = trimmed
}
func setEnabled(_ enabled: Bool) {
self.isEnabled = enabled
if enabled {
self.logger.info("enabled")
Task { await self.start() }
} else {
self.logger.info("disabled")
self.stop()
}
}
func start() async {
guard self.isEnabled else { return }
if self.isListening { return }
self.logger.info("start")
self.statusText = "Requesting permissions…"
let micOk = await Self.requestMicrophonePermission()
guard micOk else {
self.logger.warning("start blocked: microphone permission denied")
self.statusText = "Microphone permission denied"
return
}
let speechOk = await Self.requestSpeechPermission()
guard speechOk else {
self.logger.warning("start blocked: speech permission denied")
self.statusText = "Speech recognition permission denied"
return
}
await self.reloadConfig()
do {
try Self.configureAudioSession()
try self.startRecognition()
self.isListening = true
self.statusText = "Listening"
self.startSilenceMonitor()
await self.subscribeChatIfNeeded(sessionKey: self.mainSessionKey)
self.logger.info("listening")
} catch {
self.isListening = false
self.statusText = "Start failed: \(error.localizedDescription)"
self.logger.error("start failed: \(error.localizedDescription, privacy: .public)")
}
}
func stop() {
self.isEnabled = false
self.isListening = false
self.statusText = "Off"
self.lastTranscript = ""
self.lastHeard = nil
self.silenceTask?.cancel()
self.silenceTask = nil
self.stopRecognition()
self.stopSpeaking()
self.lastInterruptedAtSeconds = nil
TalkSystemSpeechSynthesizer.shared.stop()
do {
try AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation])
} catch {
self.logger.warning("audio session deactivate failed: \(error.localizedDescription, privacy: .public)")
}
Task { await self.unsubscribeAllChats() }
}
func userTappedOrb() {
self.stopSpeaking()
}
private func startRecognition() throws {
#if targetEnvironment(simulator)
throw NSError(domain: "TalkMode", code: 2, userInfo: [
NSLocalizedDescriptionKey: "Talk mode is not supported on the iOS simulator",
])
#endif
self.stopRecognition()
self.speechRecognizer = SFSpeechRecognizer()
guard let recognizer = self.speechRecognizer else {
throw NSError(domain: "TalkMode", code: 1, userInfo: [
NSLocalizedDescriptionKey: "Speech recognizer unavailable",
])
}
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
self.recognitionRequest?.shouldReportPartialResults = true
guard let request = self.recognitionRequest else { return }
let input = self.audioEngine.inputNode
let format = input.outputFormat(forBus: 0)
guard format.sampleRate > 0, format.channelCount > 0 else {
throw NSError(domain: "TalkMode", code: 3, userInfo: [
NSLocalizedDescriptionKey: "Invalid audio input format",
])
}
input.removeTap(onBus: 0)
let tapBlock = Self.makeAudioTapAppendCallback(request: request)
input.installTap(onBus: 0, bufferSize: 2048, format: format, block: tapBlock)
self.audioEngine.prepare()
try self.audioEngine.start()
self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in
guard let self else { return }
if let error {
if !self.isSpeaking {
self.statusText = "Speech error: \(error.localizedDescription)"
}
self.logger.debug("speech recognition error: \(error.localizedDescription, privacy: .public)")
}
guard let result else { return }
let transcript = result.bestTranscription.formattedString
Task { @MainActor in
await self.handleTranscript(transcript: transcript, isFinal: result.isFinal)
}
}
}
private func stopRecognition() {
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest?.endAudio()
self.recognitionRequest = nil
self.audioEngine.inputNode.removeTap(onBus: 0)
self.audioEngine.stop()
self.speechRecognizer = nil
}
private nonisolated static func makeAudioTapAppendCallback(request: SpeechRequest) -> AVAudioNodeTapBlock {
{ buffer, _ in
request.append(buffer)
}
}
private func handleTranscript(transcript: String, isFinal: Bool) async {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
if self.isSpeaking, self.interruptOnSpeech {
if self.shouldInterrupt(with: trimmed) {
self.stopSpeaking()
}
return
}
guard self.isListening else { return }
if !trimmed.isEmpty {
self.lastTranscript = trimmed
self.lastHeard = Date()
}
if isFinal {
self.lastTranscript = trimmed
}
}
private func startSilenceMonitor() {
self.silenceTask?.cancel()
self.silenceTask = Task { [weak self] in
guard let self else { return }
while self.isEnabled {
try? await Task.sleep(nanoseconds: 200_000_000)
await self.checkSilence()
}
}
}
private func checkSilence() async {
guard self.isListening, !self.isSpeaking else { return }
let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
guard !transcript.isEmpty else { return }
guard let lastHeard else { return }
if Date().timeIntervalSince(lastHeard) < self.silenceWindow { return }
await self.finalizeTranscript(transcript)
}
private func finalizeTranscript(_ transcript: String) async {
self.isListening = false
self.statusText = "Thinking…"
self.lastTranscript = ""
self.lastHeard = nil
self.stopRecognition()
await self.reloadConfig()
let prompt = self.buildPrompt(transcript: transcript)
guard let gateway else {
self.statusText = "Gateway not connected"
self.logger.warning("finalize: gateway not connected")
await self.start()
return
}
do {
let startedAt = Date().timeIntervalSince1970
let sessionKey = self.mainSessionKey
await self.subscribeChatIfNeeded(sessionKey: sessionKey)
self.logger.info(
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
let runId = try await self.sendChat(prompt, gateway: gateway)
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120)
if completion == .timeout {
self.logger.warning(
"chat completion timeout runId=\(runId, privacy: .public); attempting history fallback")
} else if completion == .aborted {
self.statusText = "Aborted"
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
await self.start()
return
} else if completion == .error {
self.statusText = "Chat error"
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
await self.start()
return
}
guard let assistantText = try await self.waitForAssistantText(
gateway: gateway,
since: startedAt,
timeoutSeconds: completion == .final ? 12 : 25)
else {
self.statusText = "No reply"
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
await self.start()
return
}
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
await self.playAssistant(text: assistantText)
} catch {
self.statusText = "Talk failed: \(error.localizedDescription)"
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
}
await self.start()
}
private func subscribeChatIfNeeded(sessionKey: String) async {
let key = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
guard !key.isEmpty else { return }
guard let gateway else { return }
guard !self.chatSubscribedSessionKeys.contains(key) else { return }
let payload = "{\"sessionKey\":\"\(key)\"}"
await gateway.sendEvent(event: "chat.subscribe", payloadJSON: payload)
self.chatSubscribedSessionKeys.insert(key)
self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)")
}
private func unsubscribeAllChats() async {
guard let gateway else { return }
let keys = self.chatSubscribedSessionKeys
self.chatSubscribedSessionKeys.removeAll()
for key in keys {
let payload = "{\"sessionKey\":\"\(key)\"}"
await gateway.sendEvent(event: "chat.unsubscribe", payloadJSON: payload)
}
}
private func buildPrompt(transcript: String) -> String {
let interrupted = self.lastInterruptedAtSeconds
self.lastInterruptedAtSeconds = nil
return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted)
}
private enum ChatCompletionState: CustomStringConvertible {
case final
case aborted
case error
case timeout
var description: String {
switch self {
case .final: "final"
case .aborted: "aborted"
case .error: "error"
case .timeout: "timeout"
}
}
}
private func sendChat(_ message: String, gateway: GatewayNodeSession) async throws -> String {
struct SendResponse: Decodable { let runId: String }
let payload: [String: Any] = [
"sessionKey": self.mainSessionKey,
"message": message,
"thinking": "low",
"timeoutMs": 30000,
"idempotencyKey": UUID().uuidString,
]
let data = try JSONSerialization.data(withJSONObject: payload)
guard let json = String(bytes: data, encoding: .utf8) else {
throw NSError(
domain: "TalkModeManager",
code: 1,
userInfo: [NSLocalizedDescriptionKey: "Failed to encode chat payload"])
}
let res = try await gateway.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30)
let decoded = try JSONDecoder().decode(SendResponse.self, from: res)
return decoded.runId
}
private func waitForChatCompletion(
runId: String,
gateway: GatewayNodeSession,
timeoutSeconds: Int = 120) async -> ChatCompletionState
{
let stream = await gateway.subscribeServerEvents(bufferingNewest: 200)
return await withTaskGroup(of: ChatCompletionState.self) { group in
group.addTask { [runId] in
for await evt in stream {
if Task.isCancelled { return .timeout }
guard evt.event == "chat", let payload = evt.payload else { continue }
guard let chatEvent = try? GatewayPayloadDecoding.decode(payload, as: ChatEvent.self) else {
continue
}
guard chatEvent.runid == runId else { continue }
if let state = chatEvent.state.value as? String {
switch state {
case "final": return .final
case "aborted": return .aborted
case "error": return .error
default: break
}
}
}
return .timeout
}
group.addTask {
try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds) * 1_000_000_000)
return .timeout
}
let result = await group.next() ?? .timeout
group.cancelAll()
return result
}
}
private func waitForAssistantText(
gateway: GatewayNodeSession,
since: Double,
timeoutSeconds: Int) async throws -> String?
{
let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds))
while Date() < deadline {
if let text = try await self.fetchLatestAssistantText(gateway: gateway, since: since) {
return text
}
try? await Task.sleep(nanoseconds: 300_000_000)
}
return nil
}
private func fetchLatestAssistantText(gateway: GatewayNodeSession, since: Double? = nil) async throws -> String? {
let res = try await gateway.request(
method: "chat.history",
paramsJSON: "{\"sessionKey\":\"\(self.mainSessionKey)\"}",
timeoutSeconds: 15)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return nil }
guard let messages = json["messages"] as? [[String: Any]] else { return nil }
for msg in messages.reversed() {
guard (msg["role"] as? String) == "assistant" else { continue }
if let since, let timestamp = msg["timestamp"] as? Double,
TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since) == false
{
continue
}
guard let content = msg["content"] as? [[String: Any]] else { continue }
let text = content.compactMap { $0["text"] as? String }.joined(separator: "\n")
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
if !trimmed.isEmpty { return trimmed }
}
return nil
}
private func playAssistant(text: String) async {
let parsed = TalkDirectiveParser.parse(text)
let directive = parsed.directive
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
guard !cleaned.isEmpty else { return }
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
}
if let voice = resolvedVoice {
if directive?.once != true {
self.currentVoiceId = voice
self.voiceOverrideActive = true
}
}
if let model = directive?.modelId {
if directive?.once != true {
self.currentModelId = model
self.modelOverrideActive = true
}
}
self.statusText = "Generating voice…"
self.isSpeaking = true
self.lastSpokenText = cleaned
do {
let started = Date()
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
let resolvedKey =
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
} else {
nil
}
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
if canUseElevenLabs, let voiceId, let apiKey {
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
.trimmingCharacters(in: .whitespacesAndNewlines)
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
if outputFormat == nil, let requestedOutputFormat {
self.logger.warning(
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
}
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
ElevenLabsTTSRequest(
text: cleaned,
modelId: modelId,
outputFormat: outputFormat,
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
style: TalkTTSValidation.validatedUnit(directive?.style),
speakerBoost: directive?.speakerBoost,
seed: TalkTTSValidation.validatedSeed(directive?.seed),
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
language: language,
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
}
let request = makeRequest(outputFormat: outputFormat)
let client = ElevenLabsTTSClient(apiKey: apiKey)
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking…"
let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat)
let result: StreamingPlaybackResult
if let sampleRate {
self.lastPlaybackWasPCM = true
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
if !playback.finished, playback.interruptedAt == nil {
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
self.logger.warning("pcm playback failed; retrying mp3")
self.lastPlaybackWasPCM = false
let mp3Stream = client.streamSynthesize(
voiceId: voiceId,
request: makeRequest(outputFormat: mp3Format))
playback = await self.mp3Player.play(stream: mp3Stream)
}
result = playback
} else {
self.lastPlaybackWasPCM = false
result = await self.mp3Player.play(stream: stream)
}
let duration = Date().timeIntervalSince(started)
self.logger.info("elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(duration, privacy: .public)s")
if !result.finished, let interruptedAt = result.interruptedAt {
self.lastInterruptedAtSeconds = interruptedAt
}
} else {
self.logger.warning("tts unavailable; falling back to system voice (missing key or voiceId)")
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking (System)…"
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
}
} catch {
self.logger.error(
"tts failed: \(error.localizedDescription, privacy: .public); falling back to system voice")
do {
if self.interruptOnSpeech {
do {
try self.startRecognition()
} catch {
self.logger.warning(
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
}
}
self.statusText = "Speaking (System)…"
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
} catch {
self.statusText = "Speak failed: \(error.localizedDescription)"
self.logger.error("system voice failed: \(error.localizedDescription, privacy: .public)")
}
}
self.stopRecognition()
self.isSpeaking = false
}
private func stopSpeaking(storeInterruption: Bool = true) {
guard self.isSpeaking else { return }
let interruptedAt = self.lastPlaybackWasPCM
? self.pcmPlayer.stop()
: self.mp3Player.stop()
if storeInterruption {
self.lastInterruptedAtSeconds = interruptedAt
}
_ = self.lastPlaybackWasPCM
? self.mp3Player.stop()
: self.pcmPlayer.stop()
TalkSystemSpeechSynthesizer.shared.stop()
self.isSpeaking = false
}
private func shouldInterrupt(with transcript: String) -> Bool {
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
guard trimmed.count >= 3 else { return false }
if let spoken = self.lastSpokenText?.lowercased(), spoken.contains(trimmed.lowercased()) {
return false
}
return true
}
private func resolveVoiceAlias(_ value: String?) -> String? {
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return nil }
let normalized = trimmed.lowercased()
if let mapped = self.voiceAliases[normalized] { return mapped }
if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) {
return trimmed
}
return Self.isLikelyVoiceId(trimmed) ? trimmed : nil
}
private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? {
let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
if !trimmed.isEmpty {
if let resolved = self.resolveVoiceAlias(trimmed) { return resolved }
self.logger.warning("unknown voice alias \(trimmed, privacy: .public)")
}
if let fallbackVoiceId { return fallbackVoiceId }
do {
let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices()
guard let first = voices.first else {
self.logger.warning("elevenlabs voices list empty")
return nil
}
self.fallbackVoiceId = first.voiceId
if self.defaultVoiceId == nil {
self.defaultVoiceId = first.voiceId
}
if !self.voiceOverrideActive {
self.currentVoiceId = first.voiceId
}
let name = first.name ?? "unknown"
self.logger
.info("default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))")
return first.voiceId
} catch {
self.logger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)")
return nil
}
}
private static func isLikelyVoiceId(_ value: String) -> Bool {
guard value.count >= 10 else { return false }
return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" }
}
private func reloadConfig() async {
guard let gateway else { return }
do {
let res = try await gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
guard let config = json["config"] as? [String: Any] else { return }
let talk = config["talk"] as? [String: Any]
let session = config["session"] as? [String: Any]
let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String)
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
self.mainSessionKey = mainKey
}
self.defaultVoiceId = (talk?["voiceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let aliases = talk?["voiceAliases"] as? [String: Any] {
var resolved: [String: String] = [:]
for (key, value) in aliases {
guard let id = value as? String else { continue }
let normalizedKey = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
let trimmedId = id.trimmingCharacters(in: .whitespacesAndNewlines)
guard !normalizedKey.isEmpty, !trimmedId.isEmpty else { continue }
resolved[normalizedKey] = trimmedId
}
self.voiceAliases = resolved
} else {
self.voiceAliases = [:]
}
if !self.voiceOverrideActive {
self.currentVoiceId = self.defaultVoiceId
}
let model = (talk?["modelId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
self.defaultModelId = (model?.isEmpty == false) ? model : Self.defaultModelIdFallback
if !self.modelOverrideActive {
self.currentModelId = self.defaultModelId
}
self.defaultOutputFormat = (talk?["outputFormat"] as? String)?
.trimmingCharacters(in: .whitespacesAndNewlines)
self.apiKey = (talk?["apiKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
if let interrupt = talk?["interruptOnSpeech"] as? Bool {
self.interruptOnSpeech = interrupt
}
} catch {
self.defaultModelId = Self.defaultModelIdFallback
if !self.modelOverrideActive {
self.currentModelId = self.defaultModelId
}
}
}
private static func configureAudioSession() throws {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .voiceChat, options: [
.duckOthers,
.mixWithOthers,
.allowBluetoothHFP,
.defaultToSpeaker,
])
try session.setActive(true, options: [])
}
private nonisolated static func requestMicrophonePermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
AVAudioApplication.requestRecordPermission { ok in
cont.resume(returning: ok)
}
}
}
private nonisolated static func requestSpeechPermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
SFSpeechRecognizer.requestAuthorization { status in
cont.resume(returning: status == .authorized)
}
}
}
}

View File

@@ -0,0 +1,70 @@
import SwiftUI
struct TalkOrbOverlay: View {
@Environment(NodeAppModel.self) private var appModel
@State private var pulse: Bool = false
var body: some View {
let seam = self.appModel.seamColor
let status = self.appModel.talkMode.statusText.trimmingCharacters(in: .whitespacesAndNewlines)
VStack(spacing: 14) {
ZStack {
Circle()
.stroke(seam.opacity(0.26), lineWidth: 2)
.frame(width: 320, height: 320)
.scaleEffect(self.pulse ? 1.15 : 0.96)
.opacity(self.pulse ? 0.0 : 1.0)
.animation(.easeOut(duration: 1.3).repeatForever(autoreverses: false), value: self.pulse)
Circle()
.stroke(seam.opacity(0.18), lineWidth: 2)
.frame(width: 320, height: 320)
.scaleEffect(self.pulse ? 1.45 : 1.02)
.opacity(self.pulse ? 0.0 : 0.9)
.animation(.easeOut(duration: 1.9).repeatForever(autoreverses: false).delay(0.2), value: self.pulse)
Circle()
.fill(
RadialGradient(
colors: [
seam.opacity(0.95),
seam.opacity(0.40),
Color.black.opacity(0.55),
],
center: .center,
startRadius: 1,
endRadius: 112))
.frame(width: 190, height: 190)
.overlay(
Circle()
.stroke(seam.opacity(0.35), lineWidth: 1))
.shadow(color: seam.opacity(0.32), radius: 26, x: 0, y: 0)
.shadow(color: Color.black.opacity(0.50), radius: 22, x: 0, y: 10)
}
.contentShape(Circle())
.onTapGesture {
self.appModel.talkMode.userTappedOrb()
}
if !status.isEmpty, status != "Off" {
Text(status)
.font(.system(.footnote, design: .rounded).weight(.semibold))
.foregroundStyle(Color.white.opacity(0.92))
.padding(.horizontal, 12)
.padding(.vertical, 8)
.background(
Capsule()
.fill(Color.black.opacity(0.40))
.overlay(
Capsule().stroke(seam.opacity(0.22), lineWidth: 1)))
}
}
.padding(28)
.onAppear {
self.pulse = true
}
.accessibilityElement(children: .combine)
.accessibilityLabel("Talk Mode \(status)")
}
}

View File

@@ -0,0 +1,46 @@
import SwiftUI
struct VoiceTab: View {
@Environment(NodeAppModel.self) private var appModel
@Environment(VoiceWakeManager.self) private var voiceWake
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
var body: some View {
NavigationStack {
List {
Section("Status") {
LabeledContent("Voice Wake", value: self.voiceWakeEnabled ? "Enabled" : "Disabled")
LabeledContent("Listener", value: self.voiceWake.isListening ? "Listening" : "Idle")
Text(self.voiceWake.statusText)
.font(.footnote)
.foregroundStyle(.secondary)
LabeledContent("Talk Mode", value: self.talkEnabled ? "Enabled" : "Disabled")
}
Section("Notes") {
let triggers = self.voiceWake.activeTriggerWords
Group {
if triggers.isEmpty {
Text("Add wake words in Settings.")
} else if triggers.count == 1 {
Text("Say “\(triggers[0]) …” to trigger.")
} else if triggers.count == 2 {
Text("Say “\(triggers[0]) …” or “\(triggers[1]) …” to trigger.")
} else {
Text("Say “\(triggers.joined(separator: " …”, “")) …” to trigger.")
}
}
.foregroundStyle(.secondary)
}
}
.navigationTitle("Voice")
.onChange(of: self.voiceWakeEnabled) { _, newValue in
self.appModel.setVoiceWakeEnabled(newValue)
}
.onChange(of: self.talkEnabled) { _, newValue in
self.appModel.setTalkEnabled(newValue)
}
}
}
}

View File

@@ -0,0 +1,389 @@
import AVFAudio
import Foundation
import Observation
import Speech
import SwabbleKit
private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
{ buffer, _ in
// This callback is invoked on a realtime audio thread/queue. Keep it tiny and nonisolated.
queue.enqueueCopy(of: buffer)
}
}
private final class AudioBufferQueue: @unchecked Sendable {
private let lock = NSLock()
private var buffers: [AVAudioPCMBuffer] = []
func enqueueCopy(of buffer: AVAudioPCMBuffer) {
guard let copy = buffer.deepCopy() else { return }
self.lock.lock()
self.buffers.append(copy)
self.lock.unlock()
}
func drain() -> [AVAudioPCMBuffer] {
self.lock.lock()
let drained = self.buffers
self.buffers.removeAll(keepingCapacity: true)
self.lock.unlock()
return drained
}
func clear() {
self.lock.lock()
self.buffers.removeAll(keepingCapacity: false)
self.lock.unlock()
}
}
extension AVAudioPCMBuffer {
fileprivate func deepCopy() -> AVAudioPCMBuffer? {
let format = self.format
let frameLength = self.frameLength
guard let copy = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameLength) else {
return nil
}
copy.frameLength = frameLength
if let src = self.floatChannelData, let dst = copy.floatChannelData {
let channels = Int(format.channelCount)
let frames = Int(frameLength)
for ch in 0..<channels {
dst[ch].update(from: src[ch], count: frames)
}
return copy
}
if let src = self.int16ChannelData, let dst = copy.int16ChannelData {
let channels = Int(format.channelCount)
let frames = Int(frameLength)
for ch in 0..<channels {
dst[ch].update(from: src[ch], count: frames)
}
return copy
}
if let src = self.int32ChannelData, let dst = copy.int32ChannelData {
let channels = Int(format.channelCount)
let frames = Int(frameLength)
for ch in 0..<channels {
dst[ch].update(from: src[ch], count: frames)
}
return copy
}
return nil
}
}
@MainActor
@Observable
final class VoiceWakeManager: NSObject {
var isEnabled: Bool = false
var isListening: Bool = false
var statusText: String = "Off"
var triggerWords: [String] = VoiceWakePreferences.loadTriggerWords()
var lastTriggeredCommand: String?
private let audioEngine = AVAudioEngine()
private var speechRecognizer: SFSpeechRecognizer?
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
private var recognitionTask: SFSpeechRecognitionTask?
private var tapQueue: AudioBufferQueue?
private var tapDrainTask: Task<Void, Never>?
private var lastDispatched: String?
private var onCommand: (@Sendable (String) async -> Void)?
private var userDefaultsObserver: NSObjectProtocol?
override init() {
super.init()
self.triggerWords = VoiceWakePreferences.loadTriggerWords()
self.userDefaultsObserver = NotificationCenter.default.addObserver(
forName: UserDefaults.didChangeNotification,
object: UserDefaults.standard,
queue: .main,
using: { [weak self] _ in
Task { @MainActor in
self?.handleUserDefaultsDidChange()
}
})
}
@MainActor deinit {
if let userDefaultsObserver = self.userDefaultsObserver {
NotificationCenter.default.removeObserver(userDefaultsObserver)
}
}
var activeTriggerWords: [String] {
VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords)
}
private func handleUserDefaultsDidChange() {
let updated = VoiceWakePreferences.loadTriggerWords()
if updated != self.triggerWords {
self.triggerWords = updated
}
}
func configure(onCommand: @escaping @Sendable (String) async -> Void) {
self.onCommand = onCommand
}
func setEnabled(_ enabled: Bool) {
self.isEnabled = enabled
if enabled {
Task { await self.start() }
} else {
self.stop()
}
}
func start() async {
guard self.isEnabled else { return }
if self.isListening { return }
if ProcessInfo.processInfo.environment["SIMULATOR_DEVICE_NAME"] != nil ||
ProcessInfo.processInfo.environment["SIMULATOR_UDID"] != nil
{
// The iOS Simulators audio stack is unreliable for long-running microphone capture.
// (Weve observed CoreAudio deadlocks after TCC permission prompts.)
self.isListening = false
self.statusText = "Voice Wake isnt supported on Simulator"
return
}
self.statusText = "Requesting permissions…"
let micOk = await Self.requestMicrophonePermission()
guard micOk else {
self.statusText = "Microphone permission denied"
self.isListening = false
return
}
let speechOk = await Self.requestSpeechPermission()
guard speechOk else {
self.statusText = "Speech recognition permission denied"
self.isListening = false
return
}
self.speechRecognizer = SFSpeechRecognizer()
guard self.speechRecognizer != nil else {
self.statusText = "Speech recognizer unavailable"
self.isListening = false
return
}
do {
try Self.configureAudioSession()
try self.startRecognition()
self.isListening = true
self.statusText = "Listening"
} catch {
self.isListening = false
self.statusText = "Start failed: \(error.localizedDescription)"
}
}
func stop() {
self.isEnabled = false
self.isListening = false
self.statusText = "Off"
self.tapDrainTask?.cancel()
self.tapDrainTask = nil
self.tapQueue?.clear()
self.tapQueue = nil
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest = nil
if self.audioEngine.isRunning {
self.audioEngine.stop()
self.audioEngine.inputNode.removeTap(onBus: 0)
}
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
}
/// Temporarily releases the microphone so other subsystems (e.g. camera video capture) can record audio.
/// Returns `true` when listening was active and was suspended.
func suspendForExternalAudioCapture() -> Bool {
guard self.isEnabled, self.isListening else { return false }
self.isListening = false
self.statusText = "Paused"
self.tapDrainTask?.cancel()
self.tapDrainTask = nil
self.tapQueue?.clear()
self.tapQueue = nil
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.recognitionRequest = nil
if self.audioEngine.isRunning {
self.audioEngine.stop()
self.audioEngine.inputNode.removeTap(onBus: 0)
}
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
return true
}
func resumeAfterExternalAudioCapture(wasSuspended: Bool) {
guard wasSuspended else { return }
Task { await self.start() }
}
private func startRecognition() throws {
self.recognitionTask?.cancel()
self.recognitionTask = nil
self.tapDrainTask?.cancel()
self.tapDrainTask = nil
self.tapQueue?.clear()
self.tapQueue = nil
let request = SFSpeechAudioBufferRecognitionRequest()
request.shouldReportPartialResults = true
self.recognitionRequest = request
let inputNode = self.audioEngine.inputNode
inputNode.removeTap(onBus: 0)
let recordingFormat = inputNode.outputFormat(forBus: 0)
let queue = AudioBufferQueue()
self.tapQueue = queue
let tapBlock: @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void = makeAudioTapEnqueueCallback(queue: queue)
inputNode.installTap(
onBus: 0,
bufferSize: 1024,
format: recordingFormat,
block: tapBlock)
self.audioEngine.prepare()
try self.audioEngine.start()
let handler = self.makeRecognitionResultHandler()
self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request, resultHandler: handler)
self.tapDrainTask = Task { [weak self] in
guard let self, let queue = self.tapQueue else { return }
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: 40_000_000)
let drained = queue.drain()
if drained.isEmpty { continue }
for buf in drained {
request.append(buf)
}
}
}
}
private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void {
{ [weak self] result, error in
let transcript = result?.bestTranscription.formattedString
let segments = result.flatMap { result in
transcript.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) }
} ?? []
let errorText = error?.localizedDescription
Task { @MainActor in
self?.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
}
}
}
private func handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
if let errorText {
self.statusText = "Recognizer error: \(errorText)"
self.isListening = false
let shouldRestart = self.isEnabled
if shouldRestart {
Task {
try? await Task.sleep(nanoseconds: 700_000_000)
await self.start()
}
}
return
}
guard let transcript else { return }
guard let cmd = self.extractCommand(from: transcript, segments: segments) else { return }
if cmd == self.lastDispatched { return }
self.lastDispatched = cmd
self.lastTriggeredCommand = cmd
self.statusText = "Triggered"
Task { [weak self] in
guard let self else { return }
await self.onCommand?(cmd)
await self.startIfEnabled()
}
}
private func startIfEnabled() async {
let shouldRestart = self.isEnabled
if shouldRestart {
await self.start()
}
}
private func extractCommand(from transcript: String, segments: [WakeWordSegment]) -> String? {
Self.extractCommand(from: transcript, segments: segments, triggers: self.activeTriggerWords)
}
nonisolated static func extractCommand(
from transcript: String,
segments: [WakeWordSegment],
triggers: [String],
minPostTriggerGap: TimeInterval = 0.45) -> String?
{
let config = WakeWordGateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap)
return WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command
}
private static func configureAudioSession() throws {
let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .measurement, options: [
.duckOthers,
.mixWithOthers,
.allowBluetoothHFP,
.defaultToSpeaker,
])
try session.setActive(true, options: [])
}
private nonisolated static func requestMicrophonePermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
AVAudioApplication.requestRecordPermission { ok in
cont.resume(returning: ok)
}
}
}
private nonisolated static func requestSpeechPermission() async -> Bool {
await withCheckedContinuation(isolation: nil) { cont in
SFSpeechRecognizer.requestAuthorization { status in
cont.resume(returning: status == .authorized)
}
}
}
}
#if DEBUG
extension VoiceWakeManager {
func _test_handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
self.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
}
}
#endif

View File

@@ -0,0 +1,44 @@
import Foundation
enum VoiceWakePreferences {
static let enabledKey = "voiceWake.enabled"
static let triggerWordsKey = "voiceWake.triggerWords"
// Keep defaults aligned with the mac app.
static let defaultTriggerWords: [String] = ["clawd", "claude"]
static let maxWords = 32
static let maxWordLength = 64
static func decodeGatewayTriggers(from payloadJSON: String) -> [String]? {
guard let data = payloadJSON.data(using: .utf8) else { return nil }
return self.decodeGatewayTriggers(from: data)
}
static func decodeGatewayTriggers(from data: Data) -> [String]? {
struct Payload: Decodable { var triggers: [String] }
guard let decoded = try? JSONDecoder().decode(Payload.self, from: data) else { return nil }
return self.sanitizeTriggerWords(decoded.triggers)
}
static func loadTriggerWords(defaults: UserDefaults = .standard) -> [String] {
defaults.stringArray(forKey: self.triggerWordsKey) ?? self.defaultTriggerWords
}
static func saveTriggerWords(_ words: [String], defaults: UserDefaults = .standard) {
defaults.set(words, forKey: self.triggerWordsKey)
}
static func sanitizeTriggerWords(_ words: [String]) -> [String] {
let cleaned = words
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
.filter { !$0.isEmpty }
.prefix(Self.maxWords)
.map { String($0.prefix(Self.maxWordLength)) }
return cleaned.isEmpty ? Self.defaultTriggerWords : cleaned
}
static func displayString(for words: [String]) -> String {
let sanitized = self.sanitizeTriggerWords(words)
return sanitized.joined(separator: ", ")
}
}

View File

@@ -0,0 +1,60 @@
Sources/Gateway/GatewayConnectionController.swift
Sources/Gateway/GatewayDiscoveryDebugLogView.swift
Sources/Gateway/GatewayDiscoveryModel.swift
Sources/Gateway/GatewaySettingsStore.swift
Sources/Gateway/KeychainStore.swift
Sources/Camera/CameraController.swift
Sources/Chat/ChatSheet.swift
Sources/Chat/IOSGatewayChatTransport.swift
Sources/ClawdbotApp.swift
Sources/Location/LocationService.swift
Sources/Model/NodeAppModel.swift
Sources/RootCanvas.swift
Sources/RootTabs.swift
Sources/Screen/ScreenController.swift
Sources/Screen/ScreenRecordService.swift
Sources/Screen/ScreenTab.swift
Sources/Screen/ScreenWebView.swift
Sources/SessionKey.swift
Sources/Settings/SettingsNetworkingHelpers.swift
Sources/Settings/SettingsTab.swift
Sources/Settings/VoiceWakeWordsSettingsView.swift
Sources/Status/StatusPill.swift
Sources/Status/VoiceWakeToast.swift
Sources/Voice/VoiceTab.swift
Sources/Voice/VoiceWakeManager.swift
Sources/Voice/VoiceWakePreferences.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatComposer.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatMarkdownRenderer.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatMarkdownPreprocessor.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatMessageViews.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatModels.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatPayloadDecoding.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatSessions.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatSheets.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatTheme.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatTransport.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatView.swift
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatViewModel.swift
../shared/MoltbotKit/Sources/MoltbotKit/AnyCodable.swift
../shared/MoltbotKit/Sources/MoltbotKit/BonjourEscapes.swift
../shared/MoltbotKit/Sources/MoltbotKit/BonjourTypes.swift
../shared/MoltbotKit/Sources/MoltbotKit/BridgeFrames.swift
../shared/MoltbotKit/Sources/MoltbotKit/CameraCommands.swift
../shared/MoltbotKit/Sources/MoltbotKit/CanvasA2UIAction.swift
../shared/MoltbotKit/Sources/MoltbotKit/CanvasA2UICommands.swift
../shared/MoltbotKit/Sources/MoltbotKit/CanvasA2UIJSONL.swift
../shared/MoltbotKit/Sources/MoltbotKit/CanvasCommandParams.swift
../shared/MoltbotKit/Sources/MoltbotKit/CanvasCommands.swift
../shared/MoltbotKit/Sources/MoltbotKit/Capabilities.swift
../shared/MoltbotKit/Sources/MoltbotKit/ClawdbotKitResources.swift
../shared/MoltbotKit/Sources/MoltbotKit/DeepLinks.swift
../shared/MoltbotKit/Sources/MoltbotKit/JPEGTranscoder.swift
../shared/MoltbotKit/Sources/MoltbotKit/NodeError.swift
../shared/MoltbotKit/Sources/MoltbotKit/ScreenCommands.swift
../shared/MoltbotKit/Sources/MoltbotKit/StoragePaths.swift
../shared/MoltbotKit/Sources/MoltbotKit/SystemCommands.swift
../shared/MoltbotKit/Sources/MoltbotKit/TalkDirective.swift
../../Swabble/Sources/SwabbleKit/WakeWordGate.swift
Sources/Voice/TalkModeManager.swift
Sources/Voice/TalkOrbOverlay.swift

View File

@@ -0,0 +1,31 @@
import SwiftUI
import Testing
@testable import Moltbot
@Suite struct AppCoverageTests {
@Test @MainActor func nodeAppModelUpdatesBackgroundedState() {
let appModel = NodeAppModel()
appModel.setScenePhase(.background)
#expect(appModel.isBackgrounded == true)
appModel.setScenePhase(.inactive)
#expect(appModel.isBackgrounded == false)
appModel.setScenePhase(.active)
#expect(appModel.isBackgrounded == false)
}
@Test @MainActor func voiceWakeStartReportsUnsupportedOnSimulator() async {
let voiceWake = VoiceWakeManager()
voiceWake.isEnabled = true
await voiceWake.start()
#expect(voiceWake.isListening == false)
#expect(voiceWake.statusText.contains("Simulator"))
voiceWake.stop()
#expect(voiceWake.statusText == "Off")
}
}

View File

@@ -0,0 +1,24 @@
import Testing
@testable import Moltbot
@Suite struct CameraControllerClampTests {
@Test func clampQualityDefaultsAndBounds() {
#expect(CameraController.clampQuality(nil) == 0.9)
#expect(CameraController.clampQuality(0.0) == 0.05)
#expect(CameraController.clampQuality(0.049) == 0.05)
#expect(CameraController.clampQuality(0.05) == 0.05)
#expect(CameraController.clampQuality(0.5) == 0.5)
#expect(CameraController.clampQuality(1.0) == 1.0)
#expect(CameraController.clampQuality(1.1) == 1.0)
}
@Test func clampDurationDefaultsAndBounds() {
#expect(CameraController.clampDurationMs(nil) == 3000)
#expect(CameraController.clampDurationMs(0) == 250)
#expect(CameraController.clampDurationMs(249) == 250)
#expect(CameraController.clampDurationMs(250) == 250)
#expect(CameraController.clampDurationMs(1000) == 1000)
#expect(CameraController.clampDurationMs(60000) == 60000)
#expect(CameraController.clampDurationMs(60001) == 60000)
}
}

View File

@@ -0,0 +1,14 @@
import Testing
@testable import Moltbot
@Suite struct CameraControllerErrorTests {
@Test func errorDescriptionsAreStable() {
#expect(CameraController.CameraError.cameraUnavailable.errorDescription == "Camera unavailable")
#expect(CameraController.CameraError.microphoneUnavailable.errorDescription == "Microphone unavailable")
#expect(CameraController.CameraError.permissionDenied(kind: "Camera")
.errorDescription == "Camera permission denied")
#expect(CameraController.CameraError.invalidParams("bad").errorDescription == "bad")
#expect(CameraController.CameraError.captureFailed("nope").errorDescription == "nope")
#expect(CameraController.CameraError.exportFailed("export").errorDescription == "export")
}
}

View File

@@ -0,0 +1,79 @@
import MoltbotKit
import Foundation
import Testing
@Suite struct DeepLinkParserTests {
@Test func parseRejectsUnknownHost() {
let url = URL(string: "moltbot://nope?message=hi")!
#expect(DeepLinkParser.parse(url) == nil)
}
@Test func parseHostIsCaseInsensitive() {
let url = URL(string: "moltbot://AGENT?message=Hello")!
#expect(DeepLinkParser.parse(url) == .agent(.init(
message: "Hello",
sessionKey: nil,
thinking: nil,
deliver: false,
to: nil,
channel: nil,
timeoutSeconds: nil,
key: nil)))
}
@Test func parseRejectsNonMoltbotScheme() {
let url = URL(string: "https://example.com/agent?message=hi")!
#expect(DeepLinkParser.parse(url) == nil)
}
@Test func parseRejectsEmptyMessage() {
let url = URL(string: "moltbot://agent?message=%20%20%0A")!
#expect(DeepLinkParser.parse(url) == nil)
}
@Test func parseAgentLinkParsesCommonFields() {
let url =
URL(string: "moltbot://agent?message=Hello&deliver=1&sessionKey=node-test&thinking=low&timeoutSeconds=30")!
#expect(
DeepLinkParser.parse(url) == .agent(
.init(
message: "Hello",
sessionKey: "node-test",
thinking: "low",
deliver: true,
to: nil,
channel: nil,
timeoutSeconds: 30,
key: nil)))
}
@Test func parseAgentLinkParsesTargetRoutingFields() {
let url =
URL(
string: "moltbot://agent?message=Hello%20World&deliver=1&to=%2B15551234567&channel=whatsapp&key=secret")!
#expect(
DeepLinkParser.parse(url) == .agent(
.init(
message: "Hello World",
sessionKey: nil,
thinking: nil,
deliver: true,
to: "+15551234567",
channel: "whatsapp",
timeoutSeconds: nil,
key: "secret")))
}
@Test func parseRejectsNegativeTimeoutSeconds() {
let url = URL(string: "moltbot://agent?message=Hello&timeoutSeconds=-1")!
#expect(DeepLinkParser.parse(url) == .agent(.init(
message: "Hello",
sessionKey: nil,
thinking: nil,
deliver: false,
to: nil,
channel: nil,
timeoutSeconds: nil,
key: nil)))
}
}

View File

@@ -0,0 +1,79 @@
import MoltbotKit
import Foundation
import Testing
import UIKit
@testable import Moltbot
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try body()
}
@Suite(.serialized) struct GatewayConnectionControllerTests {
@Test @MainActor func resolvedDisplayNameSetsDefaultWhenMissing() {
let defaults = UserDefaults.standard
let displayKey = "node.displayName"
withUserDefaults([displayKey: nil, "node.instanceId": "ios-test"]) {
let appModel = NodeAppModel()
let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false)
let resolved = controller._test_resolvedDisplayName(defaults: defaults)
#expect(!resolved.isEmpty)
#expect(defaults.string(forKey: displayKey) == resolved)
}
}
@Test @MainActor func currentCapsReflectToggles() {
withUserDefaults([
"node.instanceId": "ios-test",
"node.displayName": "Test Node",
"camera.enabled": true,
"location.enabledMode": MoltbotLocationMode.always.rawValue,
VoiceWakePreferences.enabledKey: true,
]) {
let appModel = NodeAppModel()
let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false)
let caps = Set(controller._test_currentCaps())
#expect(caps.contains(MoltbotCapability.canvas.rawValue))
#expect(caps.contains(MoltbotCapability.screen.rawValue))
#expect(caps.contains(MoltbotCapability.camera.rawValue))
#expect(caps.contains(MoltbotCapability.location.rawValue))
#expect(caps.contains(MoltbotCapability.voiceWake.rawValue))
}
}
@Test @MainActor func currentCommandsIncludeLocationWhenEnabled() {
withUserDefaults([
"node.instanceId": "ios-test",
"location.enabledMode": MoltbotLocationMode.whileUsing.rawValue,
]) {
let appModel = NodeAppModel()
let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false)
let commands = Set(controller._test_currentCommands())
#expect(commands.contains(MoltbotLocationCommand.get.rawValue))
}
}
}

View File

@@ -0,0 +1,22 @@
import Testing
@testable import Moltbot
@Suite(.serialized) struct GatewayDiscoveryModelTests {
@Test @MainActor func debugLoggingCapturesLifecycleAndResets() {
let model = GatewayDiscoveryModel()
#expect(model.debugLog.isEmpty)
#expect(model.statusText == "Idle")
model.setDebugLoggingEnabled(true)
#expect(model.debugLog.count >= 2)
model.stop()
#expect(model.statusText == "Stopped")
#expect(model.gateways.isEmpty)
#expect(model.debugLog.count >= 3)
model.setDebugLoggingEnabled(false)
#expect(model.debugLog.isEmpty)
}
}

View File

@@ -0,0 +1,33 @@
import MoltbotKit
import Network
import Testing
@testable import Moltbot
@Suite struct GatewayEndpointIDTests {
@Test func stableIDForServiceDecodesAndNormalizesName() {
let endpoint = NWEndpoint.service(
name: "Moltbot\\032Gateway \\032 Node\n",
type: "_moltbot-gw._tcp",
domain: "local.",
interface: nil)
#expect(GatewayEndpointID.stableID(endpoint) == "_moltbot-gw._tcp|local.|Moltbot Gateway Node")
}
@Test func stableIDForNonServiceUsesEndpointDescription() {
let endpoint = NWEndpoint.hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 4242)
#expect(GatewayEndpointID.stableID(endpoint) == String(describing: endpoint))
}
@Test func prettyDescriptionDecodesBonjourEscapes() {
let endpoint = NWEndpoint.service(
name: "Moltbot\\032Gateway",
type: "_moltbot-gw._tcp",
domain: "local.",
interface: nil)
let pretty = GatewayEndpointID.prettyDescription(endpoint)
#expect(pretty == BonjourEscapes.decode(String(describing: endpoint)))
#expect(!pretty.localizedCaseInsensitiveContains("\\032"))
}
}

View File

@@ -0,0 +1,127 @@
import Foundation
import Testing
@testable import Moltbot
private struct KeychainEntry: Hashable {
let service: String
let account: String
}
private let gatewayService = "bot.molt.gateway"
private let nodeService = "bot.molt.node"
private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId")
private let preferredGatewayEntry = KeychainEntry(service: gatewayService, account: "preferredStableID")
private let lastGatewayEntry = KeychainEntry(service: gatewayService, account: "lastDiscoveredStableID")
private func snapshotDefaults(_ keys: [String]) -> [String: Any?] {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in keys {
snapshot[key] = defaults.object(forKey: key)
}
return snapshot
}
private func applyDefaults(_ values: [String: Any?]) {
let defaults = UserDefaults.standard
for (key, value) in values {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
private func restoreDefaults(_ snapshot: [String: Any?]) {
applyDefaults(snapshot)
}
private func snapshotKeychain(_ entries: [KeychainEntry]) -> [KeychainEntry: String?] {
var snapshot: [KeychainEntry: String?] = [:]
for entry in entries {
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
}
return snapshot
}
private func applyKeychain(_ values: [KeychainEntry: String?]) {
for (entry, value) in values {
if let value {
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
} else {
_ = KeychainStore.delete(service: entry.service, account: entry.account)
}
}
}
private func restoreKeychain(_ snapshot: [KeychainEntry: String?]) {
applyKeychain(snapshot)
}
@Suite(.serialized) struct GatewaySettingsStoreTests {
@Test func bootstrapCopiesDefaultsToKeychainWhenMissing() {
let defaultsKeys = [
"node.instanceId",
"gateway.preferredStableID",
"gateway.lastDiscoveredStableID",
]
let entries = [instanceIdEntry, preferredGatewayEntry, lastGatewayEntry]
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
let keychainSnapshot = snapshotKeychain(entries)
defer {
restoreDefaults(defaultsSnapshot)
restoreKeychain(keychainSnapshot)
}
applyDefaults([
"node.instanceId": "node-test",
"gateway.preferredStableID": "preferred-test",
"gateway.lastDiscoveredStableID": "last-test",
])
applyKeychain([
instanceIdEntry: nil,
preferredGatewayEntry: nil,
lastGatewayEntry: nil,
])
GatewaySettingsStore.bootstrapPersistence()
#expect(KeychainStore.loadString(service: nodeService, account: "instanceId") == "node-test")
#expect(KeychainStore.loadString(service: gatewayService, account: "preferredStableID") == "preferred-test")
#expect(KeychainStore.loadString(service: gatewayService, account: "lastDiscoveredStableID") == "last-test")
}
@Test func bootstrapCopiesKeychainToDefaultsWhenMissing() {
let defaultsKeys = [
"node.instanceId",
"gateway.preferredStableID",
"gateway.lastDiscoveredStableID",
]
let entries = [instanceIdEntry, preferredGatewayEntry, lastGatewayEntry]
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
let keychainSnapshot = snapshotKeychain(entries)
defer {
restoreDefaults(defaultsSnapshot)
restoreKeychain(keychainSnapshot)
}
applyDefaults([
"node.instanceId": nil,
"gateway.preferredStableID": nil,
"gateway.lastDiscoveredStableID": nil,
])
applyKeychain([
instanceIdEntry: "node-from-keychain",
preferredGatewayEntry: "preferred-from-keychain",
lastGatewayEntry: "last-from-keychain",
])
GatewaySettingsStore.bootstrapPersistence()
let defaults = UserDefaults.standard
#expect(defaults.string(forKey: "node.instanceId") == "node-from-keychain")
#expect(defaults.string(forKey: "gateway.preferredStableID") == "preferred-from-keychain")
#expect(defaults.string(forKey: "gateway.lastDiscoveredStableID") == "last-from-keychain")
}
}

View File

@@ -0,0 +1,30 @@
import MoltbotKit
import Testing
@testable import Moltbot
@Suite struct IOSGatewayChatTransportTests {
@Test func requestsFailFastWhenGatewayNotConnected() async {
let gateway = GatewayNodeSession()
let transport = IOSGatewayChatTransport(gateway: gateway)
do {
_ = try await transport.requestHistory(sessionKey: "node-test")
Issue.record("Expected requestHistory to throw when gateway not connected")
} catch {}
do {
_ = try await transport.sendMessage(
sessionKey: "node-test",
message: "hello",
thinking: "low",
idempotencyKey: "idempotency",
attachments: [])
Issue.record("Expected sendMessage to throw when gateway not connected")
} catch {}
do {
_ = try await transport.requestHealth(timeoutMs: 250)
Issue.record("Expected requestHealth to throw when gateway not connected")
} catch {}
}
}

View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
<string>MoltbotTests</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>BNDL</string>
<key>CFBundleShortVersionString</key>
<string>2026.1.26</string>
<key>CFBundleVersion</key>
<string>20260126</string>
</dict>
</plist>

View File

@@ -0,0 +1,22 @@
import Foundation
import Testing
@testable import Moltbot
@Suite struct KeychainStoreTests {
@Test func saveLoadUpdateDeleteRoundTrip() {
let service = "bot.molt.tests.\(UUID().uuidString)"
let account = "value"
#expect(KeychainStore.delete(service: service, account: account))
#expect(KeychainStore.loadString(service: service, account: account) == nil)
#expect(KeychainStore.saveString("first", service: service, account: account))
#expect(KeychainStore.loadString(service: service, account: account) == "first")
#expect(KeychainStore.saveString("second", service: service, account: account))
#expect(KeychainStore.loadString(service: service, account: account) == "second")
#expect(KeychainStore.delete(service: service, account: account))
#expect(KeychainStore.loadString(service: service, account: account) == nil)
}
}

View File

@@ -0,0 +1,194 @@
import MoltbotKit
import Foundation
import Testing
import UIKit
@testable import Moltbot
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
let defaults = UserDefaults.standard
var snapshot: [String: Any?] = [:]
for key in updates.keys {
snapshot[key] = defaults.object(forKey: key)
}
for (key, value) in updates {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
defer {
for (key, value) in snapshot {
if let value {
defaults.set(value, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
}
return try body()
}
@Suite(.serialized) struct NodeAppModelInvokeTests {
@Test @MainActor func decodeParamsFailsWithoutJSON() {
#expect(throws: Error.self) {
_ = try NodeAppModel._test_decodeParams(MoltbotCanvasNavigateParams.self, from: nil)
}
}
@Test @MainActor func encodePayloadEmitsJSON() throws {
struct Payload: Codable, Equatable {
var value: String
}
let json = try NodeAppModel._test_encodePayload(Payload(value: "ok"))
#expect(json.contains("\"value\""))
}
@Test @MainActor func handleInvokeRejectsBackgroundCommands() async {
let appModel = NodeAppModel()
appModel.setScenePhase(.background)
let req = BridgeInvokeRequest(id: "bg", command: MoltbotCanvasCommand.present.rawValue)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .backgroundUnavailable)
}
@Test @MainActor func handleInvokeRejectsCameraWhenDisabled() async {
let appModel = NodeAppModel()
let req = BridgeInvokeRequest(id: "cam", command: MoltbotCameraCommand.snap.rawValue)
let defaults = UserDefaults.standard
let key = "camera.enabled"
let previous = defaults.object(forKey: key)
defaults.set(false, forKey: key)
defer {
if let previous {
defaults.set(previous, forKey: key)
} else {
defaults.removeObject(forKey: key)
}
}
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .unavailable)
#expect(res.error?.message.contains("CAMERA_DISABLED") == true)
}
@Test @MainActor func handleInvokeRejectsInvalidScreenFormat() async {
let appModel = NodeAppModel()
let params = MoltbotScreenRecordParams(format: "gif")
let data = try? JSONEncoder().encode(params)
let json = data.flatMap { String(data: $0, encoding: .utf8) }
let req = BridgeInvokeRequest(
id: "screen",
command: MoltbotScreenCommand.record.rawValue,
paramsJSON: json)
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.message.contains("screen format must be mp4") == true)
}
@Test @MainActor func handleInvokeCanvasCommandsUpdateScreen() async throws {
let appModel = NodeAppModel()
appModel.screen.navigate(to: "http://example.com")
let present = BridgeInvokeRequest(id: "present", command: MoltbotCanvasCommand.present.rawValue)
let presentRes = await appModel._test_handleInvoke(present)
#expect(presentRes.ok == true)
#expect(appModel.screen.urlString.isEmpty)
let navigateParams = MoltbotCanvasNavigateParams(url: "http://localhost:18789/")
let navData = try JSONEncoder().encode(navigateParams)
let navJSON = String(decoding: navData, as: UTF8.self)
let navigate = BridgeInvokeRequest(
id: "nav",
command: MoltbotCanvasCommand.navigate.rawValue,
paramsJSON: navJSON)
let navRes = await appModel._test_handleInvoke(navigate)
#expect(navRes.ok == true)
#expect(appModel.screen.urlString == "http://localhost:18789/")
let evalParams = MoltbotCanvasEvalParams(javaScript: "1+1")
let evalData = try JSONEncoder().encode(evalParams)
let evalJSON = String(decoding: evalData, as: UTF8.self)
let eval = BridgeInvokeRequest(
id: "eval",
command: MoltbotCanvasCommand.evalJS.rawValue,
paramsJSON: evalJSON)
let evalRes = await appModel._test_handleInvoke(eval)
#expect(evalRes.ok == true)
let payloadData = try #require(evalRes.payloadJSON?.data(using: .utf8))
let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any]
#expect(payload?["result"] as? String == "2")
}
@Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws {
let appModel = NodeAppModel()
let reset = BridgeInvokeRequest(id: "reset", command: MoltbotCanvasA2UICommand.reset.rawValue)
let resetRes = await appModel._test_handleInvoke(reset)
#expect(resetRes.ok == false)
#expect(resetRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
let jsonl = "{\"beginRendering\":{}}"
let pushParams = MoltbotCanvasA2UIPushJSONLParams(jsonl: jsonl)
let pushData = try JSONEncoder().encode(pushParams)
let pushJSON = String(decoding: pushData, as: UTF8.self)
let push = BridgeInvokeRequest(
id: "push",
command: MoltbotCanvasA2UICommand.pushJSONL.rawValue,
paramsJSON: pushJSON)
let pushRes = await appModel._test_handleInvoke(push)
#expect(pushRes.ok == false)
#expect(pushRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
}
@Test @MainActor func handleInvokeUnknownCommandReturnsInvalidRequest() async {
let appModel = NodeAppModel()
let req = BridgeInvokeRequest(id: "unknown", command: "nope")
let res = await appModel._test_handleInvoke(req)
#expect(res.ok == false)
#expect(res.error?.code == .invalidRequest)
}
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
let appModel = NodeAppModel()
let url = URL(string: "moltbot://agent?message=hello")!
await appModel.handleDeepLink(url: url)
#expect(appModel.screen.errorText?.contains("Gateway not connected") == true)
}
@Test @MainActor func handleDeepLinkRejectsOversizedMessage() async {
let appModel = NodeAppModel()
let msg = String(repeating: "a", count: 20001)
let url = URL(string: "moltbot://agent?message=\(msg)")!
await appModel.handleDeepLink(url: url)
#expect(appModel.screen.errorText?.contains("Deep link too large") == true)
}
@Test @MainActor func sendVoiceTranscriptThrowsWhenGatewayOffline() async {
let appModel = NodeAppModel()
await #expect(throws: Error.self) {
try await appModel.sendVoiceTranscript(text: "hello", sessionKey: "main")
}
}
@Test @MainActor func canvasA2UIActionDispatchesStatus() async {
let appModel = NodeAppModel()
let body: [String: Any] = [
"userAction": [
"name": "tap",
"id": "action-1",
"surfaceId": "main",
"sourceComponentId": "button-1",
"context": ["value": "ok"],
],
]
await appModel._test_handleCanvasA2UIAction(body: body)
#expect(appModel.screen.urlString.isEmpty)
}
}

View File

@@ -0,0 +1,71 @@
import Testing
import WebKit
@testable import Moltbot
@Suite struct ScreenControllerTests {
@Test @MainActor func canvasModeConfiguresWebViewForTouch() {
let screen = ScreenController()
#expect(screen.webView.isOpaque == true)
#expect(screen.webView.backgroundColor == .black)
let scrollView = screen.webView.scrollView
#expect(scrollView.backgroundColor == .black)
#expect(scrollView.contentInsetAdjustmentBehavior == .never)
#expect(scrollView.isScrollEnabled == false)
#expect(scrollView.bounces == false)
}
@Test @MainActor func navigateEnablesScrollForWebPages() {
let screen = ScreenController()
screen.navigate(to: "https://example.com")
let scrollView = screen.webView.scrollView
#expect(scrollView.isScrollEnabled == true)
#expect(scrollView.bounces == true)
}
@Test @MainActor func navigateSlashShowsDefaultCanvas() {
let screen = ScreenController()
screen.navigate(to: "/")
#expect(screen.urlString.isEmpty)
}
@Test @MainActor func evalExecutesJavaScript() async throws {
let screen = ScreenController()
let deadline = ContinuousClock().now.advanced(by: .seconds(3))
while true {
do {
let result = try await screen.eval(javaScript: "1+1")
#expect(result == "2")
return
} catch {
if ContinuousClock().now >= deadline {
throw error
}
try? await Task.sleep(nanoseconds: 100_000_000)
}
}
}
@Test @MainActor func localNetworkCanvasURLsAreAllowed() {
let screen = ScreenController()
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://localhost:18789/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://clawd.local:18789/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://peters-mac-studio-1:18789/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "https://peters-mac-studio-1.ts.net:18789/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://192.168.0.10:18789/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://10.0.0.10:18789/")!) == true)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://100.123.224.76:18789/")!) == true) // Tailscale CGNAT
#expect(screen.isLocalNetworkCanvasURL(URL(string: "https://example.com/")!) == false)
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://8.8.8.8/")!) == false)
}
@Test func parseA2UIActionBodyAcceptsJSONString() throws {
let body = ScreenController.parseA2UIActionBody("{\"userAction\":{\"name\":\"hello\"}}")
let userAction = try #require(body?["userAction"] as? [String: Any])
#expect(userAction["name"] as? String == "hello")
}
}

View File

@@ -0,0 +1,32 @@
import Testing
@testable import Moltbot
@Suite(.serialized) struct ScreenRecordServiceTests {
@Test func clampDefaultsAndBounds() {
#expect(ScreenRecordService._test_clampDurationMs(nil) == 10000)
#expect(ScreenRecordService._test_clampDurationMs(0) == 250)
#expect(ScreenRecordService._test_clampDurationMs(60001) == 60000)
#expect(ScreenRecordService._test_clampFps(nil) == 10)
#expect(ScreenRecordService._test_clampFps(0) == 1)
#expect(ScreenRecordService._test_clampFps(120) == 30)
#expect(ScreenRecordService._test_clampFps(.infinity) == 10)
}
@Test @MainActor func recordRejectsInvalidScreenIndex() async {
let recorder = ScreenRecordService()
do {
_ = try await recorder.record(
screenIndex: 1,
durationMs: 250,
fps: 5,
includeAudio: false,
outPath: nil)
Issue.record("Expected invalid screen index to throw")
} catch let error as ScreenRecordService.ScreenRecordError {
#expect(error.localizedDescription.contains("Invalid screen index") == true)
} catch {
Issue.record("Unexpected error type: \(error)")
}
}
}

View File

@@ -0,0 +1,50 @@
import Testing
@testable import Moltbot
@Suite struct SettingsNetworkingHelpersTests {
@Test func parseHostPortParsesIPv4() {
#expect(SettingsNetworkingHelpers.parseHostPort(from: "127.0.0.1:8080") == .init(host: "127.0.0.1", port: 8080))
}
@Test func parseHostPortParsesHostnameAndTrims() {
#expect(SettingsNetworkingHelpers.parseHostPort(from: " example.com:80 \n") == .init(
host: "example.com",
port: 80))
}
@Test func parseHostPortParsesBracketedIPv6() {
#expect(
SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]:443") ==
.init(host: "2001:db8::1", port: 443))
}
@Test func parseHostPortRejectsMissingPort() {
#expect(SettingsNetworkingHelpers.parseHostPort(from: "example.com") == nil)
#expect(SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]") == nil)
}
@Test func parseHostPortRejectsInvalidPort() {
#expect(SettingsNetworkingHelpers.parseHostPort(from: "example.com:lol") == nil)
#expect(SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]:lol") == nil)
}
@Test func httpURLStringFormatsIPv4AndPort() {
#expect(SettingsNetworkingHelpers
.httpURLString(host: "127.0.0.1", port: 8080, fallback: "fallback") == "http://127.0.0.1:8080")
}
@Test func httpURLStringBracketsIPv6() {
#expect(SettingsNetworkingHelpers
.httpURLString(host: "2001:db8::1", port: 8080, fallback: "fallback") == "http://[2001:db8::1]:8080")
}
@Test func httpURLStringLeavesAlreadyBracketedIPv6() {
#expect(SettingsNetworkingHelpers
.httpURLString(host: "[2001:db8::1]", port: 8080, fallback: "fallback") == "http://[2001:db8::1]:8080")
}
@Test func httpURLStringFallsBackWhenMissingHostOrPort() {
#expect(SettingsNetworkingHelpers.httpURLString(host: nil, port: 80, fallback: "x") == "http://x")
#expect(SettingsNetworkingHelpers.httpURLString(host: "example.com", port: nil, fallback: "y") == "http://y")
}
}

View File

@@ -0,0 +1,81 @@
import MoltbotKit
import SwiftUI
import Testing
import UIKit
@testable import Moltbot
@Suite struct SwiftUIRenderSmokeTests {
@MainActor private static func host(_ view: some View) -> UIWindow {
let window = UIWindow(frame: UIScreen.main.bounds)
window.rootViewController = UIHostingController(rootView: view)
window.makeKeyAndVisible()
window.rootViewController?.view.setNeedsLayout()
window.rootViewController?.view.layoutIfNeeded()
return window
}
@Test @MainActor func statusPillConnectingBuildsAViewHierarchy() {
let root = StatusPill(gateway: .connecting, voiceWakeEnabled: true, brighten: true) {}
_ = Self.host(root)
}
@Test @MainActor func statusPillDisconnectedBuildsAViewHierarchy() {
let root = StatusPill(gateway: .disconnected, voiceWakeEnabled: false) {}
_ = Self.host(root)
}
@Test @MainActor func settingsTabBuildsAViewHierarchy() {
let appModel = NodeAppModel()
let gatewayController = GatewayConnectionController(appModel: appModel, startDiscovery: false)
let root = SettingsTab()
.environment(appModel)
.environment(appModel.voiceWake)
.environment(gatewayController)
_ = Self.host(root)
}
@Test @MainActor func rootTabsBuildAViewHierarchy() {
let appModel = NodeAppModel()
let gatewayController = GatewayConnectionController(appModel: appModel, startDiscovery: false)
let root = RootTabs()
.environment(appModel)
.environment(appModel.voiceWake)
.environment(gatewayController)
_ = Self.host(root)
}
@Test @MainActor func voiceTabBuildsAViewHierarchy() {
let appModel = NodeAppModel()
let root = VoiceTab()
.environment(appModel)
.environment(appModel.voiceWake)
_ = Self.host(root)
}
@Test @MainActor func voiceWakeWordsViewBuildsAViewHierarchy() {
let appModel = NodeAppModel()
let root = NavigationStack { VoiceWakeWordsSettingsView() }
.environment(appModel)
_ = Self.host(root)
}
@Test @MainActor func chatSheetBuildsAViewHierarchy() {
let appModel = NodeAppModel()
let gateway = GatewayNodeSession()
let root = ChatSheet(gateway: gateway, sessionKey: "test")
.environment(appModel)
.environment(appModel.voiceWake)
_ = Self.host(root)
}
@Test @MainActor func voiceWakeToastBuildsAViewHierarchy() {
let root = VoiceWakeToast(command: "moltbot: do something")
_ = Self.host(root)
}
}

View File

@@ -0,0 +1,22 @@
import Foundation
import Testing
@testable import Moltbot
@Suite struct VoiceWakeGatewaySyncTests {
@Test func decodeGatewayTriggersFromJSONSanitizes() {
let payload = #"{"triggers":[" clawd ","", "computer"]}"#
let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payload)
#expect(triggers == ["clawd", "computer"])
}
@Test func decodeGatewayTriggersFromJSONFallsBackWhenEmpty() {
let payload = #"{"triggers":[" ",""]}"#
let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payload)
#expect(triggers == VoiceWakePreferences.defaultTriggerWords)
}
@Test func decodeGatewayTriggersFromInvalidJSONReturnsNil() {
let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: "not json")
#expect(triggers == nil)
}
}

View File

@@ -0,0 +1,90 @@
import Foundation
import SwabbleKit
import Testing
@testable import Moltbot
@Suite struct VoiceWakeManagerExtractCommandTests {
@Test func extractCommandReturnsNilWhenNoTriggerFound() {
let transcript = "hello world"
let segments = makeSegments(
transcript: transcript,
words: [("hello", 0.0, 0.1), ("world", 0.2, 0.1)])
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
}
@Test func extractCommandTrimsTokensAndResult() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: [" clawd "],
minPostTriggerGap: 0.3)
#expect(cmd == "do thing")
}
@Test func extractCommandReturnsNilWhenGapTooShort() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.35, 0.1),
("thing", 0.5, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: ["clawd"],
minPostTriggerGap: 0.3)
#expect(cmd == nil)
}
@Test func extractCommandReturnsNilWhenNothingAfterTrigger() {
let transcript = "hey clawd"
let segments = makeSegments(
transcript: transcript,
words: [("hey", 0.0, 0.1), ("clawd", 0.2, 0.1)])
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
}
@Test func extractCommandIgnoresEmptyTriggers() {
let transcript = "hey clawd do thing"
let segments = makeSegments(
transcript: transcript,
words: [
("hey", 0.0, 0.1),
("clawd", 0.2, 0.1),
("do", 0.9, 0.1),
("thing", 1.1, 0.1),
])
let cmd = VoiceWakeManager.extractCommand(
from: transcript,
segments: segments,
triggers: ["", " ", "clawd"],
minPostTriggerGap: 0.3)
#expect(cmd == "do thing")
}
}
private func makeSegments(
transcript: String,
words: [(String, TimeInterval, TimeInterval)])
-> [WakeWordSegment] {
var searchStart = transcript.startIndex
var output: [WakeWordSegment] = []
for (word, start, duration) in words {
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
if let range { searchStart = range.upperBound }
}
return output
}

View File

@@ -0,0 +1,65 @@
import Foundation
import SwabbleKit
import Testing
@testable import Moltbot
@Suite(.serialized) struct VoiceWakeManagerStateTests {
@Test @MainActor func suspendAndResumeCycleUpdatesState() async {
let manager = VoiceWakeManager()
manager.isEnabled = true
manager.isListening = true
manager.statusText = "Listening"
let suspended = manager.suspendForExternalAudioCapture()
#expect(suspended == true)
#expect(manager.isListening == false)
#expect(manager.statusText == "Paused")
manager.resumeAfterExternalAudioCapture(wasSuspended: true)
try? await Task.sleep(nanoseconds: 900_000_000)
#expect(manager.statusText.contains("Voice Wake") == true)
}
@Test @MainActor func handleRecognitionCallbackRestartsOnError() async {
let manager = VoiceWakeManager()
manager.isEnabled = true
manager.isListening = true
manager._test_handleRecognitionCallback(transcript: nil, segments: [], errorText: "boom")
#expect(manager.statusText.contains("Recognizer error") == true)
#expect(manager.isListening == false)
try? await Task.sleep(nanoseconds: 900_000_000)
#expect(manager.statusText.contains("Voice Wake") == true)
}
@Test @MainActor func handleRecognitionCallbackDispatchesCommand() async {
let manager = VoiceWakeManager()
manager.triggerWords = ["clawd"]
manager.isEnabled = true
actor CaptureBox {
var value: String?
func set(_ next: String) { self.value = next }
}
let capture = CaptureBox()
manager.configure { cmd in
await capture.set(cmd)
}
let transcript = "clawd hello"
let clawdRange = transcript.range(of: "clawd")!
let helloRange = transcript.range(of: "hello")!
let segments = [
WakeWordSegment(text: "clawd", start: 0.0, duration: 0.2, range: clawdRange),
WakeWordSegment(text: "hello", start: 0.8, duration: 0.2, range: helloRange),
]
manager._test_handleRecognitionCallback(transcript: transcript, segments: segments, errorText: nil)
#expect(manager.lastTriggeredCommand == "hello")
#expect(manager.statusText == "Triggered")
try? await Task.sleep(nanoseconds: 300_000_000)
#expect(await capture.value == "hello")
}
}

View File

@@ -0,0 +1,38 @@
import Foundation
import Testing
@testable import Moltbot
@Suite struct VoiceWakePreferencesTests {
@Test func sanitizeTriggerWordsTrimsAndDropsEmpty() {
#expect(VoiceWakePreferences.sanitizeTriggerWords([" clawd ", "", " \nclaude\t"]) == ["clawd", "claude"])
}
@Test func sanitizeTriggerWordsFallsBackToDefaultsWhenEmpty() {
#expect(VoiceWakePreferences.sanitizeTriggerWords(["", " "]) == VoiceWakePreferences.defaultTriggerWords)
}
@Test func sanitizeTriggerWordsLimitsWordLength() {
let long = String(repeating: "x", count: VoiceWakePreferences.maxWordLength + 5)
let cleaned = VoiceWakePreferences.sanitizeTriggerWords(["ok", long])
#expect(cleaned[1].count == VoiceWakePreferences.maxWordLength)
}
@Test func sanitizeTriggerWordsLimitsWordCount() {
let words = (1...VoiceWakePreferences.maxWords + 3).map { "w\($0)" }
let cleaned = VoiceWakePreferences.sanitizeTriggerWords(words)
#expect(cleaned.count == VoiceWakePreferences.maxWords)
}
@Test func displayStringUsesSanitizedWords() {
#expect(VoiceWakePreferences.displayString(for: ["", " "]) == "clawd, claude")
}
@Test func loadAndSaveTriggerWordsRoundTrip() {
let suiteName = "VoiceWakePreferencesTests.\(UUID().uuidString)"
let defaults = UserDefaults(suiteName: suiteName)!
#expect(VoiceWakePreferences.loadTriggerWords(defaults: defaults) == VoiceWakePreferences.defaultTriggerWords)
VoiceWakePreferences.saveTriggerWords(["computer"], defaults: defaults)
#expect(VoiceWakePreferences.loadTriggerWords(defaults: defaults) == ["computer"])
}
}

View File

@@ -0,0 +1,21 @@
# App Store Connect API key (pick one approach)
#
# Recommended (use the downloaded .p8 directly):
# ASC_KEY_ID=XXXXXXXXXX
# ASC_ISSUER_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
# ASC_KEY_PATH=/absolute/path/to/AuthKey_XXXXXXXXXX.p8
#
# Or (JSON key file):
# APP_STORE_CONNECT_API_KEY_PATH=/absolute/path/to/AuthKey_XXXXXX.json
#
# Or:
# ASC_KEY_ID=XXXXXXXXXX
# ASC_ISSUER_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
# ASC_KEY_CONTENT=BASE64_P8_CONTENT
# Code signing
# IOS_DEVELOPMENT_TEAM=XXXXXXXXXX
# Deliver toggles (off by default)
# DELIVER_METADATA=1
# DELIVER_SCREENSHOTS=1

View File

@@ -0,0 +1,7 @@
app_identifier("bot.molt.ios")
# Auth is expected via App Store Connect API key.
# Provide either:
# - APP_STORE_CONNECT_API_KEY_PATH=/path/to/AuthKey_XXXXXX.p8.json (recommended)
# or:
# - ASC_KEY_ID, ASC_ISSUER_ID, and ASC_KEY_CONTENT (base64 or raw p8 content)

View File

@@ -0,0 +1,103 @@
require "shellwords"
default_platform(:ios)
def load_env_file(path)
return unless File.exist?(path)
File.foreach(path) do |line|
stripped = line.strip
next if stripped.empty? || stripped.start_with?("#")
key, value = stripped.split("=", 2)
next if key.nil? || key.empty? || value.nil?
ENV[key] = value if ENV[key].nil? || ENV[key].strip.empty?
end
end
platform :ios do
private_lane :asc_api_key do
load_env_file(File.join(__dir__, ".env"))
api_key = nil
key_path = ENV["APP_STORE_CONNECT_API_KEY_PATH"]
if key_path && !key_path.strip.empty?
api_key = app_store_connect_api_key(path: key_path)
else
p8_path = ENV["ASC_KEY_PATH"]
if p8_path && !p8_path.strip.empty?
key_id = ENV["ASC_KEY_ID"]
issuer_id = ENV["ASC_ISSUER_ID"]
UI.user_error!("Missing ASC_KEY_ID or ASC_ISSUER_ID for ASC_KEY_PATH auth.") if [key_id, issuer_id].any? { |v| v.nil? || v.strip.empty? }
api_key = app_store_connect_api_key(
key_id: key_id,
issuer_id: issuer_id,
key_filepath: p8_path
)
else
key_id = ENV["ASC_KEY_ID"]
issuer_id = ENV["ASC_ISSUER_ID"]
key_content = ENV["ASC_KEY_CONTENT"]
UI.user_error!("Missing App Store Connect API key. Set APP_STORE_CONNECT_API_KEY_PATH (json) or ASC_KEY_PATH (p8) or ASC_KEY_ID/ASC_ISSUER_ID/ASC_KEY_CONTENT.") if [key_id, issuer_id, key_content].any? { |v| v.nil? || v.strip.empty? }
is_base64 = key_content.include?("BEGIN PRIVATE KEY") ? false : true
api_key = app_store_connect_api_key(
key_id: key_id,
issuer_id: issuer_id,
key_content: key_content,
is_key_content_base64: is_base64
)
end
end
api_key
end
desc "Build + upload to TestFlight"
lane :beta do
api_key = asc_api_key
team_id = ENV["IOS_DEVELOPMENT_TEAM"]
if team_id.nil? || team_id.strip.empty?
helper_path = File.expand_path("../../scripts/ios-team-id.sh", __dir__)
if File.exist?(helper_path)
team_id = sh("bash #{helper_path.shellescape}").strip
end
end
UI.user_error!("Missing IOS_DEVELOPMENT_TEAM (Apple Team ID). Add it to fastlane/.env or export it in your shell.") if team_id.nil? || team_id.strip.empty?
build_app(
project: "Moltbot.xcodeproj",
scheme: "Moltbot",
export_method: "app-store",
clean: true,
xcargs: "DEVELOPMENT_TEAM=#{team_id} -allowProvisioningUpdates",
export_xcargs: "-allowProvisioningUpdates",
export_options: {
signingStyle: "automatic"
}
)
upload_to_testflight(
api_key: api_key,
skip_waiting_for_build_processing: true
)
end
desc "Upload App Store metadata (and optionally screenshots)"
lane :metadata do
api_key = asc_api_key
deliver(
api_key: api_key,
force: true,
skip_screenshots: ENV["DELIVER_SCREENSHOTS"] != "1",
skip_metadata: ENV["DELIVER_METADATA"] != "1"
)
end
end

View File

@@ -0,0 +1,32 @@
# fastlane setup (Moltbot iOS)
Install:
```bash
brew install fastlane
```
Create an App Store Connect API key:
- App Store Connect → Users and Access → Keys → App Store Connect API → Generate API Key
- Download the `.p8`, note the **Issuer ID** and **Key ID**
Create `apps/ios/fastlane/.env` (gitignored):
```bash
ASC_KEY_ID=YOUR_KEY_ID
ASC_ISSUER_ID=YOUR_ISSUER_ID
ASC_KEY_PATH=/absolute/path/to/AuthKey_XXXXXXXXXX.p8
# Code signing (Apple Team ID / App ID Prefix)
IOS_DEVELOPMENT_TEAM=YOUR_TEAM_ID
```
Tip: run `scripts/ios-team-id.sh` from the repo root to print a Team ID to paste into `.env`. Fastlane falls back to this helper if `IOS_DEVELOPMENT_TEAM` is missing.
Run:
```bash
cd apps/ios
fastlane beta
```

View File

@@ -0,0 +1,134 @@
name: Moltbot
options:
bundleIdPrefix: bot.molt
deploymentTarget:
iOS: "18.0"
xcodeVersion: "16.0"
settings:
base:
SWIFT_VERSION: "6.0"
packages:
MoltbotKit:
path: ../shared/MoltbotKit
Swabble:
path: ../../Swabble
schemes:
Moltbot:
shared: true
build:
targets:
Moltbot: all
test:
targets:
- MoltbotTests
targets:
Moltbot:
type: application
platform: iOS
sources:
- path: Sources
dependencies:
- package: MoltbotKit
- package: MoltbotKit
product: MoltbotChatUI
- package: MoltbotKit
product: MoltbotProtocol
- package: Swabble
product: SwabbleKit
- sdk: AppIntents.framework
preBuildScripts:
- name: SwiftFormat (lint)
basedOnDependencyAnalysis: false
inputFileLists:
- $(SRCROOT)/SwiftSources.input.xcfilelist
script: |
set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
if ! command -v swiftformat >/dev/null 2>&1; then
echo "error: swiftformat not found (brew install swiftformat)" >&2
exit 1
fi
swiftformat --lint --config "$SRCROOT/../../.swiftformat" \
--filelist "$SRCROOT/SwiftSources.input.xcfilelist"
- name: SwiftLint
basedOnDependencyAnalysis: false
inputFileLists:
- $(SRCROOT)/SwiftSources.input.xcfilelist
script: |
set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
if ! command -v swiftlint >/dev/null 2>&1; then
echo "error: swiftlint not found (brew install swiftlint)" >&2
exit 1
fi
swiftlint lint --config "$SRCROOT/.swiftlint.yml" --use-script-input-file-lists
settings:
base:
CODE_SIGN_IDENTITY: "Apple Development"
CODE_SIGN_STYLE: Manual
DEVELOPMENT_TEAM: Y5PE65HELJ
PRODUCT_BUNDLE_IDENTIFIER: bot.molt.ios
PROVISIONING_PROFILE_SPECIFIER: "bot.molt.ios Development"
SWIFT_VERSION: "6.0"
SWIFT_STRICT_CONCURRENCY: complete
ENABLE_APPINTENTS_METADATA: NO
info:
path: Sources/Info.plist
properties:
CFBundleDisplayName: Moltbot
CFBundleIconName: AppIcon
CFBundleShortVersionString: "2026.1.26"
CFBundleVersion: "20260126"
UILaunchScreen: {}
UIApplicationSceneManifest:
UIApplicationSupportsMultipleScenes: false
UIBackgroundModes:
- audio
NSLocalNetworkUsageDescription: Moltbot discovers and connects to your Moltbot gateway on the local network.
NSAppTransportSecurity:
NSAllowsArbitraryLoadsInWebContent: true
NSBonjourServices:
- _moltbot-gw._tcp
NSCameraUsageDescription: Moltbot can capture photos or short video clips when requested via the gateway.
NSLocationWhenInUseUsageDescription: Moltbot uses your location when you allow location sharing.
NSLocationAlwaysAndWhenInUseUsageDescription: Moltbot can share your location in the background when you enable Always.
NSMicrophoneUsageDescription: Moltbot needs microphone access for voice wake.
NSSpeechRecognitionUsageDescription: Moltbot uses on-device speech recognition for voice wake.
UISupportedInterfaceOrientations:
- UIInterfaceOrientationPortrait
- UIInterfaceOrientationPortraitUpsideDown
- UIInterfaceOrientationLandscapeLeft
- UIInterfaceOrientationLandscapeRight
UISupportedInterfaceOrientations~ipad:
- UIInterfaceOrientationPortrait
- UIInterfaceOrientationPortraitUpsideDown
- UIInterfaceOrientationLandscapeLeft
- UIInterfaceOrientationLandscapeRight
MoltbotTests:
type: bundle.unit-test
platform: iOS
sources:
- path: Tests
dependencies:
- target: Moltbot
- package: Swabble
product: SwabbleKit
- sdk: AppIntents.framework
settings:
base:
PRODUCT_BUNDLE_IDENTIFIER: bot.molt.ios.tests
SWIFT_VERSION: "6.0"
SWIFT_STRICT_CONCURRENCY: complete
TEST_HOST: "$(BUILT_PRODUCTS_DIR)/Moltbot.app/Moltbot"
BUNDLE_LOADER: "$(TEST_HOST)"
info:
path: Tests/Info.plist
properties:
CFBundleDisplayName: MoltbotTests
CFBundleShortVersionString: "2026.1.26"
CFBundleVersion: "20260126"