Add ez-assistant and kerberos service folders
5
docker-compose/ez-assistant/apps/ios/.swiftlint.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
parent_config: ../../.swiftlint.yml
|
||||
|
||||
included:
|
||||
- Sources
|
||||
- ../shared/ClawdisNodeKit/Sources
|
||||
28
docker-compose/ez-assistant/apps/ios/README.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Clawdbot (iOS)
|
||||
|
||||
Internal-only SwiftUI app scaffold.
|
||||
|
||||
## Lint/format (required)
|
||||
```bash
|
||||
brew install swiftformat swiftlint
|
||||
```
|
||||
|
||||
## Generate the Xcode project
|
||||
```bash
|
||||
cd apps/ios
|
||||
xcodegen generate
|
||||
open Clawdbot.xcodeproj
|
||||
```
|
||||
|
||||
## Shared packages
|
||||
- `../shared/MoltbotKit` — shared types/constants used by iOS (and later macOS bridge + gateway routing).
|
||||
|
||||
## fastlane
|
||||
```bash
|
||||
brew install fastlane
|
||||
|
||||
cd apps/ios
|
||||
fastlane lanes
|
||||
```
|
||||
|
||||
See `apps/ios/fastlane/SETUP.md` for App Store Connect auth + upload lanes.
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"images" : [
|
||||
{ "filename" : "icon-20@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "20x20" },
|
||||
{ "filename" : "icon-20@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "20x20" },
|
||||
{ "filename" : "icon-20@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "20x20" },
|
||||
{ "filename" : "icon-20@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "20x20" },
|
||||
|
||||
{ "filename" : "icon-29@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "29x29" },
|
||||
{ "filename" : "icon-29@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "29x29" },
|
||||
{ "filename" : "icon-29@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "29x29" },
|
||||
{ "filename" : "icon-29@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "29x29" },
|
||||
|
||||
{ "filename" : "icon-40@1x.png", "idiom" : "ipad", "scale" : "1x", "size" : "40x40" },
|
||||
{ "filename" : "icon-40@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "40x40" },
|
||||
{ "filename" : "icon-40@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "40x40" },
|
||||
{ "filename" : "icon-40@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "40x40" },
|
||||
|
||||
{ "filename" : "icon-60@2x.png", "idiom" : "iphone","scale" : "2x", "size" : "60x60" },
|
||||
{ "filename" : "icon-60@3x.png", "idiom" : "iphone","scale" : "3x", "size" : "60x60" },
|
||||
|
||||
{ "filename" : "icon-76@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "76x76" },
|
||||
|
||||
{ "filename" : "icon-83.5@2x.png", "idiom" : "ipad", "scale" : "2x", "size" : "83.5x83.5" },
|
||||
|
||||
{ "filename" : "icon-1024.png", "idiom" : "ios-marketing", "scale" : "1x", "size" : "1024x1024" }
|
||||
],
|
||||
"info" : {
|
||||
"author" : "xcode",
|
||||
"version" : 1
|
||||
}
|
||||
}
|
||||
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.9 KiB |
|
After Width: | Height: | Size: 4.2 KiB |
|
After Width: | Height: | Size: 7.9 KiB |
|
After Width: | Height: | Size: 2.8 KiB |
|
After Width: | Height: | Size: 7.4 KiB |
|
After Width: | Height: | Size: 15 KiB |
|
After Width: | Height: | Size: 4.2 KiB |
|
After Width: | Height: | Size: 13 KiB |
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 53 KiB |
|
After Width: | Height: | Size: 39 KiB |
|
After Width: | Height: | Size: 46 KiB |
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"info" : {
|
||||
"author" : "xcode",
|
||||
"version" : 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
import AVFoundation
|
||||
import MoltbotKit
|
||||
import Foundation
|
||||
|
||||
actor CameraController {
|
||||
struct CameraDeviceInfo: Codable, Sendable {
|
||||
var id: String
|
||||
var name: String
|
||||
var position: String
|
||||
var deviceType: String
|
||||
}
|
||||
|
||||
enum CameraError: LocalizedError, Sendable {
|
||||
case cameraUnavailable
|
||||
case microphoneUnavailable
|
||||
case permissionDenied(kind: String)
|
||||
case invalidParams(String)
|
||||
case captureFailed(String)
|
||||
case exportFailed(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .cameraUnavailable:
|
||||
"Camera unavailable"
|
||||
case .microphoneUnavailable:
|
||||
"Microphone unavailable"
|
||||
case let .permissionDenied(kind):
|
||||
"\(kind) permission denied"
|
||||
case let .invalidParams(msg):
|
||||
msg
|
||||
case let .captureFailed(msg):
|
||||
msg
|
||||
case let .exportFailed(msg):
|
||||
msg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func snap(params: MoltbotCameraSnapParams) async throws -> (
|
||||
format: String,
|
||||
base64: String,
|
||||
width: Int,
|
||||
height: Int)
|
||||
{
|
||||
let facing = params.facing ?? .front
|
||||
let format = params.format ?? .jpg
|
||||
// Default to a reasonable max width to keep gateway payload sizes manageable.
|
||||
// If you need the full-res photo, explicitly request a larger maxWidth.
|
||||
let maxWidth = params.maxWidth.flatMap { $0 > 0 ? $0 : nil } ?? 1600
|
||||
let quality = Self.clampQuality(params.quality)
|
||||
let delayMs = max(0, params.delayMs ?? 0)
|
||||
|
||||
try await self.ensureAccess(for: .video)
|
||||
|
||||
let session = AVCaptureSession()
|
||||
session.sessionPreset = .photo
|
||||
|
||||
guard let device = Self.pickCamera(facing: facing, deviceId: params.deviceId) else {
|
||||
throw CameraError.cameraUnavailable
|
||||
}
|
||||
|
||||
let input = try AVCaptureDeviceInput(device: device)
|
||||
guard session.canAddInput(input) else {
|
||||
throw CameraError.captureFailed("Failed to add camera input")
|
||||
}
|
||||
session.addInput(input)
|
||||
|
||||
let output = AVCapturePhotoOutput()
|
||||
guard session.canAddOutput(output) else {
|
||||
throw CameraError.captureFailed("Failed to add photo output")
|
||||
}
|
||||
session.addOutput(output)
|
||||
output.maxPhotoQualityPrioritization = .quality
|
||||
|
||||
session.startRunning()
|
||||
defer { session.stopRunning() }
|
||||
await Self.warmUpCaptureSession()
|
||||
await Self.sleepDelayMs(delayMs)
|
||||
|
||||
let settings: AVCapturePhotoSettings = {
|
||||
if output.availablePhotoCodecTypes.contains(.jpeg) {
|
||||
return AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg])
|
||||
}
|
||||
return AVCapturePhotoSettings()
|
||||
}()
|
||||
settings.photoQualityPrioritization = .quality
|
||||
|
||||
var delegate: PhotoCaptureDelegate?
|
||||
let rawData: Data = try await withCheckedThrowingContinuation { cont in
|
||||
let d = PhotoCaptureDelegate(cont)
|
||||
delegate = d
|
||||
output.capturePhoto(with: settings, delegate: d)
|
||||
}
|
||||
withExtendedLifetime(delegate) {}
|
||||
|
||||
let maxPayloadBytes = 5 * 1024 * 1024
|
||||
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
|
||||
let maxEncodedBytes = (maxPayloadBytes / 4) * 3
|
||||
let res = try JPEGTranscoder.transcodeToJPEG(
|
||||
imageData: rawData,
|
||||
maxWidthPx: maxWidth,
|
||||
quality: quality,
|
||||
maxBytes: maxEncodedBytes)
|
||||
|
||||
return (
|
||||
format: format.rawValue,
|
||||
base64: res.data.base64EncodedString(),
|
||||
width: res.widthPx,
|
||||
height: res.heightPx)
|
||||
}
|
||||
|
||||
func clip(params: MoltbotCameraClipParams) async throws -> (
|
||||
format: String,
|
||||
base64: String,
|
||||
durationMs: Int,
|
||||
hasAudio: Bool)
|
||||
{
|
||||
let facing = params.facing ?? .front
|
||||
let durationMs = Self.clampDurationMs(params.durationMs)
|
||||
let includeAudio = params.includeAudio ?? true
|
||||
let format = params.format ?? .mp4
|
||||
|
||||
try await self.ensureAccess(for: .video)
|
||||
if includeAudio {
|
||||
try await self.ensureAccess(for: .audio)
|
||||
}
|
||||
|
||||
let session = AVCaptureSession()
|
||||
session.sessionPreset = .high
|
||||
|
||||
guard let camera = Self.pickCamera(facing: facing, deviceId: params.deviceId) else {
|
||||
throw CameraError.cameraUnavailable
|
||||
}
|
||||
let cameraInput = try AVCaptureDeviceInput(device: camera)
|
||||
guard session.canAddInput(cameraInput) else {
|
||||
throw CameraError.captureFailed("Failed to add camera input")
|
||||
}
|
||||
session.addInput(cameraInput)
|
||||
|
||||
if includeAudio {
|
||||
guard let mic = AVCaptureDevice.default(for: .audio) else {
|
||||
throw CameraError.microphoneUnavailable
|
||||
}
|
||||
let micInput = try AVCaptureDeviceInput(device: mic)
|
||||
if session.canAddInput(micInput) {
|
||||
session.addInput(micInput)
|
||||
} else {
|
||||
throw CameraError.captureFailed("Failed to add microphone input")
|
||||
}
|
||||
}
|
||||
|
||||
let output = AVCaptureMovieFileOutput()
|
||||
guard session.canAddOutput(output) else {
|
||||
throw CameraError.captureFailed("Failed to add movie output")
|
||||
}
|
||||
session.addOutput(output)
|
||||
output.maxRecordedDuration = CMTime(value: Int64(durationMs), timescale: 1000)
|
||||
|
||||
session.startRunning()
|
||||
defer { session.stopRunning() }
|
||||
await Self.warmUpCaptureSession()
|
||||
|
||||
let movURL = FileManager().temporaryDirectory
|
||||
.appendingPathComponent("moltbot-camera-\(UUID().uuidString).mov")
|
||||
let mp4URL = FileManager().temporaryDirectory
|
||||
.appendingPathComponent("moltbot-camera-\(UUID().uuidString).mp4")
|
||||
|
||||
defer {
|
||||
try? FileManager().removeItem(at: movURL)
|
||||
try? FileManager().removeItem(at: mp4URL)
|
||||
}
|
||||
|
||||
var delegate: MovieFileDelegate?
|
||||
let recordedURL: URL = try await withCheckedThrowingContinuation { cont in
|
||||
let d = MovieFileDelegate(cont)
|
||||
delegate = d
|
||||
output.startRecording(to: movURL, recordingDelegate: d)
|
||||
}
|
||||
withExtendedLifetime(delegate) {}
|
||||
|
||||
// Transcode .mov -> .mp4 for easier downstream handling.
|
||||
try await Self.exportToMP4(inputURL: recordedURL, outputURL: mp4URL)
|
||||
|
||||
let data = try Data(contentsOf: mp4URL)
|
||||
return (
|
||||
format: format.rawValue,
|
||||
base64: data.base64EncodedString(),
|
||||
durationMs: durationMs,
|
||||
hasAudio: includeAudio)
|
||||
}
|
||||
|
||||
func listDevices() -> [CameraDeviceInfo] {
|
||||
return Self.discoverVideoDevices().map { device in
|
||||
CameraDeviceInfo(
|
||||
id: device.uniqueID,
|
||||
name: device.localizedName,
|
||||
position: Self.positionLabel(device.position),
|
||||
deviceType: device.deviceType.rawValue)
|
||||
}
|
||||
}
|
||||
|
||||
private func ensureAccess(for mediaType: AVMediaType) async throws {
|
||||
let status = AVCaptureDevice.authorizationStatus(for: mediaType)
|
||||
switch status {
|
||||
case .authorized:
|
||||
return
|
||||
case .notDetermined:
|
||||
let ok = await withCheckedContinuation(isolation: nil) { cont in
|
||||
AVCaptureDevice.requestAccess(for: mediaType) { granted in
|
||||
cont.resume(returning: granted)
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone")
|
||||
}
|
||||
case .denied, .restricted:
|
||||
throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone")
|
||||
@unknown default:
|
||||
throw CameraError.permissionDenied(kind: mediaType == .video ? "Camera" : "Microphone")
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func pickCamera(
|
||||
facing: MoltbotCameraFacing,
|
||||
deviceId: String?) -> AVCaptureDevice?
|
||||
{
|
||||
if let deviceId, !deviceId.isEmpty {
|
||||
if let match = Self.discoverVideoDevices().first(where: { $0.uniqueID == deviceId }) {
|
||||
return match
|
||||
}
|
||||
}
|
||||
let position: AVCaptureDevice.Position = (facing == .front) ? .front : .back
|
||||
if let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: position) {
|
||||
return device
|
||||
}
|
||||
// Fall back to any default camera (e.g. simulator / unusual device configurations).
|
||||
return AVCaptureDevice.default(for: .video)
|
||||
}
|
||||
|
||||
private nonisolated static func positionLabel(_ position: AVCaptureDevice.Position) -> String {
|
||||
switch position {
|
||||
case .front: "front"
|
||||
case .back: "back"
|
||||
default: "unspecified"
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func discoverVideoDevices() -> [AVCaptureDevice] {
|
||||
let types: [AVCaptureDevice.DeviceType] = [
|
||||
.builtInWideAngleCamera,
|
||||
.builtInUltraWideCamera,
|
||||
.builtInTelephotoCamera,
|
||||
.builtInDualCamera,
|
||||
.builtInDualWideCamera,
|
||||
.builtInTripleCamera,
|
||||
.builtInTrueDepthCamera,
|
||||
.builtInLiDARDepthCamera,
|
||||
]
|
||||
let session = AVCaptureDevice.DiscoverySession(
|
||||
deviceTypes: types,
|
||||
mediaType: .video,
|
||||
position: .unspecified)
|
||||
return session.devices
|
||||
}
|
||||
|
||||
nonisolated static func clampQuality(_ quality: Double?) -> Double {
|
||||
let q = quality ?? 0.9
|
||||
return min(1.0, max(0.05, q))
|
||||
}
|
||||
|
||||
nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
|
||||
let v = ms ?? 3000
|
||||
// Keep clips short by default; avoid huge base64 payloads on the gateway.
|
||||
return min(60000, max(250, v))
|
||||
}
|
||||
|
||||
private nonisolated static func exportToMP4(inputURL: URL, outputURL: URL) async throws {
|
||||
let asset = AVURLAsset(url: inputURL)
|
||||
guard let exporter = AVAssetExportSession(asset: asset, presetName: AVAssetExportPresetMediumQuality) else {
|
||||
throw CameraError.exportFailed("Failed to create export session")
|
||||
}
|
||||
exporter.shouldOptimizeForNetworkUse = true
|
||||
|
||||
if #available(iOS 18.0, tvOS 18.0, visionOS 2.0, *) {
|
||||
do {
|
||||
try await exporter.export(to: outputURL, as: .mp4)
|
||||
return
|
||||
} catch {
|
||||
throw CameraError.exportFailed(error.localizedDescription)
|
||||
}
|
||||
} else {
|
||||
exporter.outputURL = outputURL
|
||||
exporter.outputFileType = .mp4
|
||||
|
||||
try await withCheckedThrowingContinuation(isolation: nil) { (cont: CheckedContinuation<Void, Error>) in
|
||||
exporter.exportAsynchronously {
|
||||
cont.resume(returning: ())
|
||||
}
|
||||
}
|
||||
|
||||
switch exporter.status {
|
||||
case .completed:
|
||||
return
|
||||
case .failed:
|
||||
throw CameraError.exportFailed(exporter.error?.localizedDescription ?? "export failed")
|
||||
case .cancelled:
|
||||
throw CameraError.exportFailed("export cancelled")
|
||||
default:
|
||||
throw CameraError.exportFailed("export did not complete")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func warmUpCaptureSession() async {
|
||||
// A short delay after `startRunning()` significantly reduces "blank first frame" captures on some devices.
|
||||
try? await Task.sleep(nanoseconds: 150_000_000) // 150ms
|
||||
}
|
||||
|
||||
private nonisolated static func sleepDelayMs(_ delayMs: Int) async {
|
||||
guard delayMs > 0 else { return }
|
||||
let maxDelayMs = 10 * 1000
|
||||
let ns = UInt64(min(delayMs, maxDelayMs)) * UInt64(NSEC_PER_MSEC)
|
||||
try? await Task.sleep(nanoseconds: ns)
|
||||
}
|
||||
}
|
||||
|
||||
private final class PhotoCaptureDelegate: NSObject, AVCapturePhotoCaptureDelegate {
|
||||
private let continuation: CheckedContinuation<Data, Error>
|
||||
private var didResume = false
|
||||
|
||||
init(_ continuation: CheckedContinuation<Data, Error>) {
|
||||
self.continuation = continuation
|
||||
}
|
||||
|
||||
func photoOutput(
|
||||
_ output: AVCapturePhotoOutput,
|
||||
didFinishProcessingPhoto photo: AVCapturePhoto,
|
||||
error: Error?)
|
||||
{
|
||||
guard !self.didResume else { return }
|
||||
self.didResume = true
|
||||
|
||||
if let error {
|
||||
self.continuation.resume(throwing: error)
|
||||
return
|
||||
}
|
||||
guard let data = photo.fileDataRepresentation() else {
|
||||
self.continuation.resume(
|
||||
throwing: NSError(domain: "Camera", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "photo data missing",
|
||||
]))
|
||||
return
|
||||
}
|
||||
if data.isEmpty {
|
||||
self.continuation.resume(
|
||||
throwing: NSError(domain: "Camera", code: 2, userInfo: [
|
||||
NSLocalizedDescriptionKey: "photo data empty",
|
||||
]))
|
||||
return
|
||||
}
|
||||
self.continuation.resume(returning: data)
|
||||
}
|
||||
|
||||
func photoOutput(
|
||||
_ output: AVCapturePhotoOutput,
|
||||
didFinishCaptureFor resolvedSettings: AVCaptureResolvedPhotoSettings,
|
||||
error: Error?)
|
||||
{
|
||||
guard let error else { return }
|
||||
guard !self.didResume else { return }
|
||||
self.didResume = true
|
||||
self.continuation.resume(throwing: error)
|
||||
}
|
||||
}
|
||||
|
||||
private final class MovieFileDelegate: NSObject, AVCaptureFileOutputRecordingDelegate {
|
||||
private let continuation: CheckedContinuation<URL, Error>
|
||||
private var didResume = false
|
||||
|
||||
init(_ continuation: CheckedContinuation<URL, Error>) {
|
||||
self.continuation = continuation
|
||||
}
|
||||
|
||||
func fileOutput(
|
||||
_ output: AVCaptureFileOutput,
|
||||
didFinishRecordingTo outputFileURL: URL,
|
||||
from connections: [AVCaptureConnection],
|
||||
error: Error?)
|
||||
{
|
||||
guard !self.didResume else { return }
|
||||
self.didResume = true
|
||||
|
||||
if let error {
|
||||
let ns = error as NSError
|
||||
if ns.domain == AVFoundationErrorDomain,
|
||||
ns.code == AVError.maximumDurationReached.rawValue
|
||||
{
|
||||
self.continuation.resume(returning: outputFileURL)
|
||||
return
|
||||
}
|
||||
self.continuation.resume(throwing: error)
|
||||
return
|
||||
}
|
||||
self.continuation.resume(returning: outputFileURL)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import MoltbotChatUI
|
||||
import MoltbotKit
|
||||
import SwiftUI
|
||||
|
||||
struct ChatSheet: View {
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
@State private var viewModel: MoltbotChatViewModel
|
||||
private let userAccent: Color?
|
||||
|
||||
init(gateway: GatewayNodeSession, sessionKey: String, userAccent: Color? = nil) {
|
||||
let transport = IOSGatewayChatTransport(gateway: gateway)
|
||||
self._viewModel = State(
|
||||
initialValue: MoltbotChatViewModel(
|
||||
sessionKey: sessionKey,
|
||||
transport: transport))
|
||||
self.userAccent = userAccent
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
MoltbotChatView(
|
||||
viewModel: self.viewModel,
|
||||
showsSessionSwitcher: true,
|
||||
userAccent: self.userAccent)
|
||||
.navigationTitle("Chat")
|
||||
.navigationBarTitleDisplayMode(.inline)
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button {
|
||||
self.dismiss()
|
||||
} label: {
|
||||
Image(systemName: "xmark")
|
||||
}
|
||||
.accessibilityLabel("Close")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import MoltbotChatUI
|
||||
import MoltbotKit
|
||||
import MoltbotProtocol
|
||||
import Foundation
|
||||
|
||||
struct IOSGatewayChatTransport: MoltbotChatTransport, Sendable {
|
||||
private let gateway: GatewayNodeSession
|
||||
|
||||
init(gateway: GatewayNodeSession) {
|
||||
self.gateway = gateway
|
||||
}
|
||||
|
||||
func abortRun(sessionKey: String, runId: String) async throws {
|
||||
struct Params: Codable {
|
||||
var sessionKey: String
|
||||
var runId: String
|
||||
}
|
||||
let data = try JSONEncoder().encode(Params(sessionKey: sessionKey, runId: runId))
|
||||
let json = String(data: data, encoding: .utf8)
|
||||
_ = try await self.gateway.request(method: "chat.abort", paramsJSON: json, timeoutSeconds: 10)
|
||||
}
|
||||
|
||||
func listSessions(limit: Int?) async throws -> MoltbotChatSessionsListResponse {
|
||||
struct Params: Codable {
|
||||
var includeGlobal: Bool
|
||||
var includeUnknown: Bool
|
||||
var limit: Int?
|
||||
}
|
||||
let data = try JSONEncoder().encode(Params(includeGlobal: true, includeUnknown: false, limit: limit))
|
||||
let json = String(data: data, encoding: .utf8)
|
||||
let res = try await self.gateway.request(method: "sessions.list", paramsJSON: json, timeoutSeconds: 15)
|
||||
return try JSONDecoder().decode(MoltbotChatSessionsListResponse.self, from: res)
|
||||
}
|
||||
|
||||
func setActiveSessionKey(_ sessionKey: String) async throws {
|
||||
struct Subscribe: Codable { var sessionKey: String }
|
||||
let data = try JSONEncoder().encode(Subscribe(sessionKey: sessionKey))
|
||||
let json = String(data: data, encoding: .utf8)
|
||||
await self.gateway.sendEvent(event: "chat.subscribe", payloadJSON: json)
|
||||
}
|
||||
|
||||
func requestHistory(sessionKey: String) async throws -> MoltbotChatHistoryPayload {
|
||||
struct Params: Codable { var sessionKey: String }
|
||||
let data = try JSONEncoder().encode(Params(sessionKey: sessionKey))
|
||||
let json = String(data: data, encoding: .utf8)
|
||||
let res = try await self.gateway.request(method: "chat.history", paramsJSON: json, timeoutSeconds: 15)
|
||||
return try JSONDecoder().decode(MoltbotChatHistoryPayload.self, from: res)
|
||||
}
|
||||
|
||||
func sendMessage(
|
||||
sessionKey: String,
|
||||
message: String,
|
||||
thinking: String,
|
||||
idempotencyKey: String,
|
||||
attachments: [MoltbotChatAttachmentPayload]) async throws -> MoltbotChatSendResponse
|
||||
{
|
||||
struct Params: Codable {
|
||||
var sessionKey: String
|
||||
var message: String
|
||||
var thinking: String
|
||||
var attachments: [MoltbotChatAttachmentPayload]?
|
||||
var timeoutMs: Int
|
||||
var idempotencyKey: String
|
||||
}
|
||||
|
||||
let params = Params(
|
||||
sessionKey: sessionKey,
|
||||
message: message,
|
||||
thinking: thinking,
|
||||
attachments: attachments.isEmpty ? nil : attachments,
|
||||
timeoutMs: 30000,
|
||||
idempotencyKey: idempotencyKey)
|
||||
let data = try JSONEncoder().encode(params)
|
||||
let json = String(data: data, encoding: .utf8)
|
||||
let res = try await self.gateway.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 35)
|
||||
return try JSONDecoder().decode(MoltbotChatSendResponse.self, from: res)
|
||||
}
|
||||
|
||||
func requestHealth(timeoutMs: Int) async throws -> Bool {
|
||||
let seconds = max(1, Int(ceil(Double(timeoutMs) / 1000.0)))
|
||||
let res = try await self.gateway.request(method: "health", paramsJSON: nil, timeoutSeconds: seconds)
|
||||
return (try? JSONDecoder().decode(MoltbotGatewayHealthOK.self, from: res))?.ok ?? true
|
||||
}
|
||||
|
||||
func events() -> AsyncStream<MoltbotChatTransportEvent> {
|
||||
AsyncStream { continuation in
|
||||
let task = Task {
|
||||
let stream = await self.gateway.subscribeServerEvents()
|
||||
for await evt in stream {
|
||||
if Task.isCancelled { return }
|
||||
switch evt.event {
|
||||
case "tick":
|
||||
continuation.yield(.tick)
|
||||
case "seqGap":
|
||||
continuation.yield(.seqGap)
|
||||
case "health":
|
||||
guard let payload = evt.payload else { break }
|
||||
let ok = (try? GatewayPayloadDecoding.decode(
|
||||
payload,
|
||||
as: MoltbotGatewayHealthOK.self))?.ok ?? true
|
||||
continuation.yield(.health(ok: ok))
|
||||
case "chat":
|
||||
guard let payload = evt.payload else { break }
|
||||
if let chatPayload = try? GatewayPayloadDecoding.decode(
|
||||
payload,
|
||||
as: MoltbotChatEventPayload.self)
|
||||
{
|
||||
continuation.yield(.chat(chatPayload))
|
||||
}
|
||||
case "agent":
|
||||
guard let payload = evt.payload else { break }
|
||||
if let agentPayload = try? GatewayPayloadDecoding.decode(
|
||||
payload,
|
||||
as: MoltbotAgentEventPayload.self)
|
||||
{
|
||||
continuation.yield(.agent(agentPayload))
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
continuation.onTermination = { @Sendable _ in
|
||||
task.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct MoltbotApp: App {
|
||||
@State private var appModel: NodeAppModel
|
||||
@State private var gatewayController: GatewayConnectionController
|
||||
@Environment(\.scenePhase) private var scenePhase
|
||||
|
||||
init() {
|
||||
GatewaySettingsStore.bootstrapPersistence()
|
||||
let appModel = NodeAppModel()
|
||||
_appModel = State(initialValue: appModel)
|
||||
_gatewayController = State(initialValue: GatewayConnectionController(appModel: appModel))
|
||||
}
|
||||
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
RootCanvas()
|
||||
.environment(self.appModel)
|
||||
.environment(self.appModel.voiceWake)
|
||||
.environment(self.gatewayController)
|
||||
.onOpenURL { url in
|
||||
Task { await self.appModel.handleDeepLink(url: url) }
|
||||
}
|
||||
.onChange(of: self.scenePhase) { _, newValue in
|
||||
self.appModel.setScenePhase(newValue)
|
||||
self.gatewayController.setScenePhase(newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
import MoltbotKit
|
||||
import Darwin
|
||||
import Foundation
|
||||
import Network
|
||||
import Observation
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class GatewayConnectionController {
|
||||
private(set) var gateways: [GatewayDiscoveryModel.DiscoveredGateway] = []
|
||||
private(set) var discoveryStatusText: String = "Idle"
|
||||
private(set) var discoveryDebugLog: [GatewayDiscoveryModel.DebugLogEntry] = []
|
||||
|
||||
private let discovery = GatewayDiscoveryModel()
|
||||
private weak var appModel: NodeAppModel?
|
||||
private var didAutoConnect = false
|
||||
|
||||
init(appModel: NodeAppModel, startDiscovery: Bool = true) {
|
||||
self.appModel = appModel
|
||||
|
||||
GatewaySettingsStore.bootstrapPersistence()
|
||||
let defaults = UserDefaults.standard
|
||||
self.discovery.setDebugLoggingEnabled(defaults.bool(forKey: "gateway.discovery.debugLogs"))
|
||||
|
||||
self.updateFromDiscovery()
|
||||
self.observeDiscovery()
|
||||
|
||||
if startDiscovery {
|
||||
self.discovery.start()
|
||||
}
|
||||
}
|
||||
|
||||
func setDiscoveryDebugLoggingEnabled(_ enabled: Bool) {
|
||||
self.discovery.setDebugLoggingEnabled(enabled)
|
||||
}
|
||||
|
||||
func setScenePhase(_ phase: ScenePhase) {
|
||||
switch phase {
|
||||
case .background:
|
||||
self.discovery.stop()
|
||||
case .active, .inactive:
|
||||
self.discovery.start()
|
||||
@unknown default:
|
||||
self.discovery.start()
|
||||
}
|
||||
}
|
||||
|
||||
func connect(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) async {
|
||||
let instanceId = UserDefaults.standard.string(forKey: "node.instanceId")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId)
|
||||
let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId)
|
||||
guard let host = self.resolveGatewayHost(gateway) else { return }
|
||||
let port = gateway.gatewayPort ?? 18789
|
||||
let tlsParams = self.resolveDiscoveredTLSParams(gateway: gateway)
|
||||
guard let url = self.buildGatewayURL(
|
||||
host: host,
|
||||
port: port,
|
||||
useTLS: tlsParams?.required == true)
|
||||
else { return }
|
||||
self.didAutoConnect = true
|
||||
self.startAutoConnect(
|
||||
url: url,
|
||||
gatewayStableID: gateway.stableID,
|
||||
tls: tlsParams,
|
||||
token: token,
|
||||
password: password)
|
||||
}
|
||||
|
||||
func connectManual(host: String, port: Int, useTLS: Bool) async {
|
||||
let instanceId = UserDefaults.standard.string(forKey: "node.instanceId")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId)
|
||||
let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId)
|
||||
let stableID = self.manualStableID(host: host, port: port)
|
||||
let tlsParams = self.resolveManualTLSParams(stableID: stableID, tlsEnabled: useTLS)
|
||||
guard let url = self.buildGatewayURL(
|
||||
host: host,
|
||||
port: port,
|
||||
useTLS: tlsParams?.required == true)
|
||||
else { return }
|
||||
self.didAutoConnect = true
|
||||
self.startAutoConnect(
|
||||
url: url,
|
||||
gatewayStableID: stableID,
|
||||
tls: tlsParams,
|
||||
token: token,
|
||||
password: password)
|
||||
}
|
||||
|
||||
private func updateFromDiscovery() {
|
||||
let newGateways = self.discovery.gateways
|
||||
self.gateways = newGateways
|
||||
self.discoveryStatusText = self.discovery.statusText
|
||||
self.discoveryDebugLog = self.discovery.debugLog
|
||||
self.updateLastDiscoveredGateway(from: newGateways)
|
||||
self.maybeAutoConnect()
|
||||
}
|
||||
|
||||
private func observeDiscovery() {
|
||||
withObservationTracking {
|
||||
_ = self.discovery.gateways
|
||||
_ = self.discovery.statusText
|
||||
_ = self.discovery.debugLog
|
||||
} onChange: { [weak self] in
|
||||
Task { @MainActor in
|
||||
guard let self else { return }
|
||||
self.updateFromDiscovery()
|
||||
self.observeDiscovery()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func maybeAutoConnect() {
|
||||
guard !self.didAutoConnect else { return }
|
||||
guard let appModel = self.appModel else { return }
|
||||
guard appModel.gatewayServerName == nil else { return }
|
||||
|
||||
let defaults = UserDefaults.standard
|
||||
let manualEnabled = defaults.bool(forKey: "gateway.manual.enabled")
|
||||
|
||||
let instanceId = defaults.string(forKey: "node.instanceId")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
guard !instanceId.isEmpty else { return }
|
||||
|
||||
let token = GatewaySettingsStore.loadGatewayToken(instanceId: instanceId)
|
||||
let password = GatewaySettingsStore.loadGatewayPassword(instanceId: instanceId)
|
||||
|
||||
if manualEnabled {
|
||||
let manualHost = defaults.string(forKey: "gateway.manual.host")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
guard !manualHost.isEmpty else { return }
|
||||
|
||||
let manualPort = defaults.integer(forKey: "gateway.manual.port")
|
||||
let resolvedPort = manualPort > 0 ? manualPort : 18789
|
||||
let manualTLS = defaults.bool(forKey: "gateway.manual.tls")
|
||||
|
||||
let stableID = self.manualStableID(host: manualHost, port: resolvedPort)
|
||||
let tlsParams = self.resolveManualTLSParams(stableID: stableID, tlsEnabled: manualTLS)
|
||||
|
||||
guard let url = self.buildGatewayURL(
|
||||
host: manualHost,
|
||||
port: resolvedPort,
|
||||
useTLS: tlsParams?.required == true)
|
||||
else { return }
|
||||
|
||||
self.didAutoConnect = true
|
||||
self.startAutoConnect(
|
||||
url: url,
|
||||
gatewayStableID: stableID,
|
||||
tls: tlsParams,
|
||||
token: token,
|
||||
password: password)
|
||||
return
|
||||
}
|
||||
|
||||
let preferredStableID = defaults.string(forKey: "gateway.preferredStableID")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let lastDiscoveredStableID = defaults.string(forKey: "gateway.lastDiscoveredStableID")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
|
||||
let candidates = [preferredStableID, lastDiscoveredStableID].filter { !$0.isEmpty }
|
||||
guard let targetStableID = candidates.first(where: { id in
|
||||
self.gateways.contains(where: { $0.stableID == id })
|
||||
}) else { return }
|
||||
|
||||
guard let target = self.gateways.first(where: { $0.stableID == targetStableID }) else { return }
|
||||
guard let host = self.resolveGatewayHost(target) else { return }
|
||||
let port = target.gatewayPort ?? 18789
|
||||
let tlsParams = self.resolveDiscoveredTLSParams(gateway: target)
|
||||
guard let url = self.buildGatewayURL(host: host, port: port, useTLS: tlsParams?.required == true)
|
||||
else { return }
|
||||
|
||||
self.didAutoConnect = true
|
||||
self.startAutoConnect(
|
||||
url: url,
|
||||
gatewayStableID: target.stableID,
|
||||
tls: tlsParams,
|
||||
token: token,
|
||||
password: password)
|
||||
}
|
||||
|
||||
private func updateLastDiscoveredGateway(from gateways: [GatewayDiscoveryModel.DiscoveredGateway]) {
|
||||
let defaults = UserDefaults.standard
|
||||
let preferred = defaults.string(forKey: "gateway.preferredStableID")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let existingLast = defaults.string(forKey: "gateway.lastDiscoveredStableID")?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
|
||||
// Avoid overriding user intent (preferred/lastDiscovered are also set on manual Connect).
|
||||
guard preferred.isEmpty, existingLast.isEmpty else { return }
|
||||
guard let first = gateways.first else { return }
|
||||
|
||||
defaults.set(first.stableID, forKey: "gateway.lastDiscoveredStableID")
|
||||
GatewaySettingsStore.saveLastDiscoveredGatewayStableID(first.stableID)
|
||||
}
|
||||
|
||||
private func startAutoConnect(
|
||||
url: URL,
|
||||
gatewayStableID: String,
|
||||
tls: GatewayTLSParams?,
|
||||
token: String?,
|
||||
password: String?)
|
||||
{
|
||||
guard let appModel else { return }
|
||||
let connectOptions = self.makeConnectOptions()
|
||||
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
await MainActor.run {
|
||||
appModel.gatewayStatusText = "Connecting…"
|
||||
}
|
||||
appModel.connectToGateway(
|
||||
url: url,
|
||||
gatewayStableID: gatewayStableID,
|
||||
tls: tls,
|
||||
token: token,
|
||||
password: password,
|
||||
connectOptions: connectOptions)
|
||||
}
|
||||
}
|
||||
|
||||
private func resolveDiscoveredTLSParams(gateway: GatewayDiscoveryModel.DiscoveredGateway) -> GatewayTLSParams? {
|
||||
let stableID = gateway.stableID
|
||||
let stored = GatewayTLSStore.loadFingerprint(stableID: stableID)
|
||||
|
||||
if gateway.tlsEnabled || gateway.tlsFingerprintSha256 != nil || stored != nil {
|
||||
return GatewayTLSParams(
|
||||
required: true,
|
||||
expectedFingerprint: gateway.tlsFingerprintSha256 ?? stored,
|
||||
allowTOFU: stored == nil,
|
||||
storeKey: stableID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
private func resolveManualTLSParams(stableID: String, tlsEnabled: Bool) -> GatewayTLSParams? {
|
||||
let stored = GatewayTLSStore.loadFingerprint(stableID: stableID)
|
||||
if tlsEnabled || stored != nil {
|
||||
return GatewayTLSParams(
|
||||
required: true,
|
||||
expectedFingerprint: stored,
|
||||
allowTOFU: stored == nil,
|
||||
storeKey: stableID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
private func resolveGatewayHost(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> String? {
|
||||
if let lanHost = gateway.lanHost?.trimmingCharacters(in: .whitespacesAndNewlines), !lanHost.isEmpty {
|
||||
return lanHost
|
||||
}
|
||||
if let tailnet = gateway.tailnetDns?.trimmingCharacters(in: .whitespacesAndNewlines), !tailnet.isEmpty {
|
||||
return tailnet
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private func buildGatewayURL(host: String, port: Int, useTLS: Bool) -> URL? {
|
||||
let scheme = useTLS ? "wss" : "ws"
|
||||
var components = URLComponents()
|
||||
components.scheme = scheme
|
||||
components.host = host
|
||||
components.port = port
|
||||
return components.url
|
||||
}
|
||||
|
||||
private func manualStableID(host: String, port: Int) -> String {
|
||||
"manual|\(host.lowercased())|\(port)"
|
||||
}
|
||||
|
||||
private func makeConnectOptions() -> GatewayConnectOptions {
|
||||
let defaults = UserDefaults.standard
|
||||
let displayName = self.resolvedDisplayName(defaults: defaults)
|
||||
|
||||
return GatewayConnectOptions(
|
||||
role: "node",
|
||||
scopes: [],
|
||||
caps: self.currentCaps(),
|
||||
commands: self.currentCommands(),
|
||||
permissions: [:],
|
||||
clientId: "moltbot-ios",
|
||||
clientMode: "node",
|
||||
clientDisplayName: displayName)
|
||||
}
|
||||
|
||||
private func resolvedDisplayName(defaults: UserDefaults) -> String {
|
||||
let key = "node.displayName"
|
||||
let existing = defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
if !existing.isEmpty, existing != "iOS Node" { return existing }
|
||||
|
||||
let deviceName = UIDevice.current.name.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let candidate = deviceName.isEmpty ? "iOS Node" : deviceName
|
||||
|
||||
if existing.isEmpty || existing == "iOS Node" {
|
||||
defaults.set(candidate, forKey: key)
|
||||
}
|
||||
|
||||
return candidate
|
||||
}
|
||||
|
||||
private func currentCaps() -> [String] {
|
||||
var caps = [MoltbotCapability.canvas.rawValue, MoltbotCapability.screen.rawValue]
|
||||
|
||||
// Default-on: if the key doesn't exist yet, treat it as enabled.
|
||||
let cameraEnabled =
|
||||
UserDefaults.standard.object(forKey: "camera.enabled") == nil
|
||||
? true
|
||||
: UserDefaults.standard.bool(forKey: "camera.enabled")
|
||||
if cameraEnabled { caps.append(MoltbotCapability.camera.rawValue) }
|
||||
|
||||
let voiceWakeEnabled = UserDefaults.standard.bool(forKey: VoiceWakePreferences.enabledKey)
|
||||
if voiceWakeEnabled { caps.append(MoltbotCapability.voiceWake.rawValue) }
|
||||
|
||||
let locationModeRaw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
|
||||
let locationMode = MoltbotLocationMode(rawValue: locationModeRaw) ?? .off
|
||||
if locationMode != .off { caps.append(MoltbotCapability.location.rawValue) }
|
||||
|
||||
return caps
|
||||
}
|
||||
|
||||
private func currentCommands() -> [String] {
|
||||
var commands: [String] = [
|
||||
MoltbotCanvasCommand.present.rawValue,
|
||||
MoltbotCanvasCommand.hide.rawValue,
|
||||
MoltbotCanvasCommand.navigate.rawValue,
|
||||
MoltbotCanvasCommand.evalJS.rawValue,
|
||||
MoltbotCanvasCommand.snapshot.rawValue,
|
||||
MoltbotCanvasA2UICommand.push.rawValue,
|
||||
MoltbotCanvasA2UICommand.pushJSONL.rawValue,
|
||||
MoltbotCanvasA2UICommand.reset.rawValue,
|
||||
MoltbotScreenCommand.record.rawValue,
|
||||
MoltbotSystemCommand.notify.rawValue,
|
||||
MoltbotSystemCommand.which.rawValue,
|
||||
MoltbotSystemCommand.run.rawValue,
|
||||
MoltbotSystemCommand.execApprovalsGet.rawValue,
|
||||
MoltbotSystemCommand.execApprovalsSet.rawValue,
|
||||
]
|
||||
|
||||
let caps = Set(self.currentCaps())
|
||||
if caps.contains(MoltbotCapability.camera.rawValue) {
|
||||
commands.append(MoltbotCameraCommand.list.rawValue)
|
||||
commands.append(MoltbotCameraCommand.snap.rawValue)
|
||||
commands.append(MoltbotCameraCommand.clip.rawValue)
|
||||
}
|
||||
if caps.contains(MoltbotCapability.location.rawValue) {
|
||||
commands.append(MoltbotLocationCommand.get.rawValue)
|
||||
}
|
||||
|
||||
return commands
|
||||
}
|
||||
|
||||
private func platformString() -> String {
|
||||
let v = ProcessInfo.processInfo.operatingSystemVersion
|
||||
let name = switch UIDevice.current.userInterfaceIdiom {
|
||||
case .pad:
|
||||
"iPadOS"
|
||||
case .phone:
|
||||
"iOS"
|
||||
default:
|
||||
"iOS"
|
||||
}
|
||||
return "\(name) \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
|
||||
}
|
||||
|
||||
private func deviceFamily() -> String {
|
||||
switch UIDevice.current.userInterfaceIdiom {
|
||||
case .pad:
|
||||
"iPad"
|
||||
case .phone:
|
||||
"iPhone"
|
||||
default:
|
||||
"iOS"
|
||||
}
|
||||
}
|
||||
|
||||
private func modelIdentifier() -> String {
|
||||
var systemInfo = utsname()
|
||||
uname(&systemInfo)
|
||||
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
|
||||
String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8)
|
||||
}
|
||||
let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return trimmed.isEmpty ? "unknown" : trimmed
|
||||
}
|
||||
|
||||
private func appVersion() -> String {
|
||||
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
|
||||
}
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
extension GatewayConnectionController {
|
||||
func _test_resolvedDisplayName(defaults: UserDefaults) -> String {
|
||||
self.resolvedDisplayName(defaults: defaults)
|
||||
}
|
||||
|
||||
func _test_currentCaps() -> [String] {
|
||||
self.currentCaps()
|
||||
}
|
||||
|
||||
func _test_currentCommands() -> [String] {
|
||||
self.currentCommands()
|
||||
}
|
||||
|
||||
func _test_platformString() -> String {
|
||||
self.platformString()
|
||||
}
|
||||
|
||||
func _test_deviceFamily() -> String {
|
||||
self.deviceFamily()
|
||||
}
|
||||
|
||||
func _test_modelIdentifier() -> String {
|
||||
self.modelIdentifier()
|
||||
}
|
||||
|
||||
func _test_appVersion() -> String {
|
||||
self.appVersion()
|
||||
}
|
||||
|
||||
func _test_setGateways(_ gateways: [GatewayDiscoveryModel.DiscoveredGateway]) {
|
||||
self.gateways = gateways
|
||||
}
|
||||
|
||||
func _test_triggerAutoConnect() {
|
||||
self.maybeAutoConnect()
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,68 @@
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
struct GatewayDiscoveryDebugLogView: View {
|
||||
@Environment(GatewayConnectionController.self) private var gatewayController
|
||||
@AppStorage("gateway.discovery.debugLogs") private var debugLogsEnabled: Bool = false
|
||||
|
||||
var body: some View {
|
||||
List {
|
||||
if !self.debugLogsEnabled {
|
||||
Text("Enable “Discovery Debug Logs” to start collecting events.")
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
|
||||
if self.gatewayController.discoveryDebugLog.isEmpty {
|
||||
Text("No log entries yet.")
|
||||
.foregroundStyle(.secondary)
|
||||
} else {
|
||||
ForEach(self.gatewayController.discoveryDebugLog) { entry in
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text(Self.formatTime(entry.ts))
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
Text(entry.message)
|
||||
.font(.callout)
|
||||
.textSelection(.enabled)
|
||||
}
|
||||
.padding(.vertical, 4)
|
||||
}
|
||||
}
|
||||
}
|
||||
.navigationTitle("Discovery Logs")
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button("Copy") {
|
||||
UIPasteboard.general.string = self.formattedLog()
|
||||
}
|
||||
.disabled(self.gatewayController.discoveryDebugLog.isEmpty)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func formattedLog() -> String {
|
||||
self.gatewayController.discoveryDebugLog
|
||||
.map { "\(Self.formatISO($0.ts)) \($0.message)" }
|
||||
.joined(separator: "\n")
|
||||
}
|
||||
|
||||
private static let timeFormatter: DateFormatter = {
|
||||
let formatter = DateFormatter()
|
||||
formatter.dateFormat = "HH:mm:ss"
|
||||
return formatter
|
||||
}()
|
||||
|
||||
private static let isoFormatter: ISO8601DateFormatter = {
|
||||
let formatter = ISO8601DateFormatter()
|
||||
formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
|
||||
return formatter
|
||||
}()
|
||||
|
||||
private static func formatTime(_ date: Date) -> String {
|
||||
self.timeFormatter.string(from: date)
|
||||
}
|
||||
|
||||
private static func formatISO(_ date: Date) -> String {
|
||||
self.isoFormatter.string(from: date)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
import MoltbotKit
|
||||
import Foundation
|
||||
import Network
|
||||
import Observation
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class GatewayDiscoveryModel {
|
||||
struct DebugLogEntry: Identifiable, Equatable {
|
||||
var id = UUID()
|
||||
var ts: Date
|
||||
var message: String
|
||||
}
|
||||
|
||||
struct DiscoveredGateway: Identifiable, Equatable {
|
||||
var id: String { self.stableID }
|
||||
var name: String
|
||||
var endpoint: NWEndpoint
|
||||
var stableID: String
|
||||
var debugID: String
|
||||
var lanHost: String?
|
||||
var tailnetDns: String?
|
||||
var gatewayPort: Int?
|
||||
var canvasPort: Int?
|
||||
var tlsEnabled: Bool
|
||||
var tlsFingerprintSha256: String?
|
||||
var cliPath: String?
|
||||
}
|
||||
|
||||
var gateways: [DiscoveredGateway] = []
|
||||
var statusText: String = "Idle"
|
||||
private(set) var debugLog: [DebugLogEntry] = []
|
||||
|
||||
private var browsers: [String: NWBrowser] = [:]
|
||||
private var gatewaysByDomain: [String: [DiscoveredGateway]] = [:]
|
||||
private var statesByDomain: [String: NWBrowser.State] = [:]
|
||||
private var debugLoggingEnabled = false
|
||||
private var lastStableIDs = Set<String>()
|
||||
|
||||
func setDebugLoggingEnabled(_ enabled: Bool) {
|
||||
let wasEnabled = self.debugLoggingEnabled
|
||||
self.debugLoggingEnabled = enabled
|
||||
if !enabled {
|
||||
self.debugLog = []
|
||||
} else if !wasEnabled {
|
||||
self.appendDebugLog("debug logging enabled")
|
||||
self.appendDebugLog("snapshot: status=\(self.statusText) gateways=\(self.gateways.count)")
|
||||
}
|
||||
}
|
||||
|
||||
func start() {
|
||||
if !self.browsers.isEmpty { return }
|
||||
self.appendDebugLog("start()")
|
||||
|
||||
for domain in MoltbotBonjour.gatewayServiceDomains {
|
||||
let params = NWParameters.tcp
|
||||
params.includePeerToPeer = true
|
||||
let browser = NWBrowser(
|
||||
for: .bonjour(type: MoltbotBonjour.gatewayServiceType, domain: domain),
|
||||
using: params)
|
||||
|
||||
browser.stateUpdateHandler = { [weak self] state in
|
||||
Task { @MainActor in
|
||||
guard let self else { return }
|
||||
self.statesByDomain[domain] = state
|
||||
self.updateStatusText()
|
||||
self.appendDebugLog("state[\(domain)]: \(Self.prettyState(state))")
|
||||
}
|
||||
}
|
||||
|
||||
browser.browseResultsChangedHandler = { [weak self] results, _ in
|
||||
Task { @MainActor in
|
||||
guard let self else { return }
|
||||
self.gatewaysByDomain[domain] = results.compactMap { result -> DiscoveredGateway? in
|
||||
switch result.endpoint {
|
||||
case let .service(name, _, _, _):
|
||||
let decodedName = BonjourEscapes.decode(name)
|
||||
let txt = result.endpoint.txtRecord?.dictionary ?? [:]
|
||||
let advertisedName = txt["displayName"]
|
||||
let prettyAdvertised = advertisedName
|
||||
.map(Self.prettifyInstanceName)
|
||||
.flatMap { $0.isEmpty ? nil : $0 }
|
||||
let prettyName = prettyAdvertised ?? Self.prettifyInstanceName(decodedName)
|
||||
return DiscoveredGateway(
|
||||
name: prettyName,
|
||||
endpoint: result.endpoint,
|
||||
stableID: GatewayEndpointID.stableID(result.endpoint),
|
||||
debugID: GatewayEndpointID.prettyDescription(result.endpoint),
|
||||
lanHost: Self.txtValue(txt, key: "lanHost"),
|
||||
tailnetDns: Self.txtValue(txt, key: "tailnetDns"),
|
||||
gatewayPort: Self.txtIntValue(txt, key: "gatewayPort"),
|
||||
canvasPort: Self.txtIntValue(txt, key: "canvasPort"),
|
||||
tlsEnabled: Self.txtBoolValue(txt, key: "gatewayTls"),
|
||||
tlsFingerprintSha256: Self.txtValue(txt, key: "gatewayTlsSha256"),
|
||||
cliPath: Self.txtValue(txt, key: "cliPath"))
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
|
||||
|
||||
self.recomputeGateways()
|
||||
}
|
||||
}
|
||||
|
||||
self.browsers[domain] = browser
|
||||
browser.start(queue: DispatchQueue(label: "bot.molt.ios.gateway-discovery.\(domain)"))
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
self.appendDebugLog("stop()")
|
||||
for browser in self.browsers.values {
|
||||
browser.cancel()
|
||||
}
|
||||
self.browsers = [:]
|
||||
self.gatewaysByDomain = [:]
|
||||
self.statesByDomain = [:]
|
||||
self.gateways = []
|
||||
self.statusText = "Stopped"
|
||||
}
|
||||
|
||||
private func recomputeGateways() {
|
||||
let next = self.gatewaysByDomain.values
|
||||
.flatMap(\.self)
|
||||
.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending }
|
||||
|
||||
let nextIDs = Set(next.map(\.stableID))
|
||||
let added = nextIDs.subtracting(self.lastStableIDs)
|
||||
let removed = self.lastStableIDs.subtracting(nextIDs)
|
||||
if !added.isEmpty || !removed.isEmpty {
|
||||
self.appendDebugLog("results: total=\(next.count) added=\(added.count) removed=\(removed.count)")
|
||||
}
|
||||
self.lastStableIDs = nextIDs
|
||||
self.gateways = next
|
||||
}
|
||||
|
||||
private func updateStatusText() {
|
||||
let states = Array(self.statesByDomain.values)
|
||||
if states.isEmpty {
|
||||
self.statusText = self.browsers.isEmpty ? "Idle" : "Setup"
|
||||
return
|
||||
}
|
||||
|
||||
if let failed = states.first(where: { state in
|
||||
if case .failed = state { return true }
|
||||
return false
|
||||
}) {
|
||||
if case let .failed(err) = failed {
|
||||
self.statusText = "Failed: \(err)"
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if let waiting = states.first(where: { state in
|
||||
if case .waiting = state { return true }
|
||||
return false
|
||||
}) {
|
||||
if case let .waiting(err) = waiting {
|
||||
self.statusText = "Waiting: \(err)"
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if states.contains(where: { if case .ready = $0 { true } else { false } }) {
|
||||
self.statusText = "Searching…"
|
||||
return
|
||||
}
|
||||
|
||||
if states.contains(where: { if case .setup = $0 { true } else { false } }) {
|
||||
self.statusText = "Setup"
|
||||
return
|
||||
}
|
||||
|
||||
self.statusText = "Searching…"
|
||||
}
|
||||
|
||||
private static func prettyState(_ state: NWBrowser.State) -> String {
|
||||
switch state {
|
||||
case .setup:
|
||||
"setup"
|
||||
case .ready:
|
||||
"ready"
|
||||
case let .failed(err):
|
||||
"failed (\(err))"
|
||||
case .cancelled:
|
||||
"cancelled"
|
||||
case let .waiting(err):
|
||||
"waiting (\(err))"
|
||||
@unknown default:
|
||||
"unknown"
|
||||
}
|
||||
}
|
||||
|
||||
private func appendDebugLog(_ message: String) {
|
||||
guard self.debugLoggingEnabled else { return }
|
||||
self.debugLog.append(DebugLogEntry(ts: Date(), message: message))
|
||||
if self.debugLog.count > 200 {
|
||||
self.debugLog.removeFirst(self.debugLog.count - 200)
|
||||
}
|
||||
}
|
||||
|
||||
private static func prettifyInstanceName(_ decodedName: String) -> String {
|
||||
let normalized = decodedName.split(whereSeparator: \.isWhitespace).joined(separator: " ")
|
||||
let stripped = normalized.replacingOccurrences(of: " (Moltbot)", with: "")
|
||||
.replacingOccurrences(of: #"\s+\(\d+\)$"#, with: "", options: .regularExpression)
|
||||
return stripped.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
private static func txtValue(_ dict: [String: String], key: String) -> String? {
|
||||
let raw = dict[key]?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return raw.isEmpty ? nil : raw
|
||||
}
|
||||
|
||||
private static func txtIntValue(_ dict: [String: String], key: String) -> Int? {
|
||||
guard let raw = self.txtValue(dict, key: key) else { return nil }
|
||||
return Int(raw)
|
||||
}
|
||||
|
||||
private static func txtBoolValue(_ dict: [String: String], key: String) -> Bool {
|
||||
guard let raw = self.txtValue(dict, key: key)?.lowercased() else { return false }
|
||||
return raw == "1" || raw == "true" || raw == "yes"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
import Foundation
|
||||
|
||||
enum GatewaySettingsStore {
|
||||
private static let gatewayService = "bot.molt.gateway"
|
||||
private static let legacyGatewayService = "com.clawdbot.gateway"
|
||||
private static let legacyBridgeService = "com.clawdbot.bridge"
|
||||
private static let nodeService = "bot.molt.node"
|
||||
private static let legacyNodeService = "com.clawdbot.node"
|
||||
|
||||
private static let instanceIdDefaultsKey = "node.instanceId"
|
||||
private static let preferredGatewayStableIDDefaultsKey = "gateway.preferredStableID"
|
||||
private static let lastDiscoveredGatewayStableIDDefaultsKey = "gateway.lastDiscoveredStableID"
|
||||
private static let manualEnabledDefaultsKey = "gateway.manual.enabled"
|
||||
private static let manualHostDefaultsKey = "gateway.manual.host"
|
||||
private static let manualPortDefaultsKey = "gateway.manual.port"
|
||||
private static let manualTlsDefaultsKey = "gateway.manual.tls"
|
||||
private static let discoveryDebugLogsDefaultsKey = "gateway.discovery.debugLogs"
|
||||
|
||||
private static let legacyPreferredBridgeStableIDDefaultsKey = "bridge.preferredStableID"
|
||||
private static let legacyLastDiscoveredBridgeStableIDDefaultsKey = "bridge.lastDiscoveredStableID"
|
||||
private static let legacyManualEnabledDefaultsKey = "bridge.manual.enabled"
|
||||
private static let legacyManualHostDefaultsKey = "bridge.manual.host"
|
||||
private static let legacyManualPortDefaultsKey = "bridge.manual.port"
|
||||
private static let legacyDiscoveryDebugLogsDefaultsKey = "bridge.discovery.debugLogs"
|
||||
|
||||
private static let instanceIdAccount = "instanceId"
|
||||
private static let preferredGatewayStableIDAccount = "preferredStableID"
|
||||
private static let lastDiscoveredGatewayStableIDAccount = "lastDiscoveredStableID"
|
||||
|
||||
static func bootstrapPersistence() {
|
||||
self.ensureStableInstanceID()
|
||||
self.ensurePreferredGatewayStableID()
|
||||
self.ensureLastDiscoveredGatewayStableID()
|
||||
self.migrateLegacyDefaults()
|
||||
}
|
||||
|
||||
static func loadStableInstanceID() -> String? {
|
||||
if let value = KeychainStore.loadString(service: self.nodeService, account: self.instanceIdAccount)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!value.isEmpty
|
||||
{
|
||||
return value
|
||||
}
|
||||
|
||||
if let legacy = KeychainStore.loadString(service: self.legacyNodeService, account: self.instanceIdAccount)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!legacy.isEmpty
|
||||
{
|
||||
_ = KeychainStore.saveString(legacy, service: self.nodeService, account: self.instanceIdAccount)
|
||||
return legacy
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
static func saveStableInstanceID(_ instanceId: String) {
|
||||
_ = KeychainStore.saveString(instanceId, service: self.nodeService, account: self.instanceIdAccount)
|
||||
}
|
||||
|
||||
static func loadPreferredGatewayStableID() -> String? {
|
||||
if let value = KeychainStore.loadString(
|
||||
service: self.gatewayService,
|
||||
account: self.preferredGatewayStableIDAccount
|
||||
)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!value.isEmpty
|
||||
{
|
||||
return value
|
||||
}
|
||||
|
||||
if let legacy = KeychainStore.loadString(
|
||||
service: self.legacyGatewayService,
|
||||
account: self.preferredGatewayStableIDAccount
|
||||
)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!legacy.isEmpty
|
||||
{
|
||||
_ = KeychainStore.saveString(
|
||||
legacy,
|
||||
service: self.gatewayService,
|
||||
account: self.preferredGatewayStableIDAccount)
|
||||
return legacy
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
static func savePreferredGatewayStableID(_ stableID: String) {
|
||||
_ = KeychainStore.saveString(
|
||||
stableID,
|
||||
service: self.gatewayService,
|
||||
account: self.preferredGatewayStableIDAccount)
|
||||
}
|
||||
|
||||
static func loadLastDiscoveredGatewayStableID() -> String? {
|
||||
if let value = KeychainStore.loadString(
|
||||
service: self.gatewayService,
|
||||
account: self.lastDiscoveredGatewayStableIDAccount
|
||||
)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!value.isEmpty
|
||||
{
|
||||
return value
|
||||
}
|
||||
|
||||
if let legacy = KeychainStore.loadString(
|
||||
service: self.legacyGatewayService,
|
||||
account: self.lastDiscoveredGatewayStableIDAccount
|
||||
)?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!legacy.isEmpty
|
||||
{
|
||||
_ = KeychainStore.saveString(
|
||||
legacy,
|
||||
service: self.gatewayService,
|
||||
account: self.lastDiscoveredGatewayStableIDAccount)
|
||||
return legacy
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
static func saveLastDiscoveredGatewayStableID(_ stableID: String) {
|
||||
_ = KeychainStore.saveString(
|
||||
stableID,
|
||||
service: self.gatewayService,
|
||||
account: self.lastDiscoveredGatewayStableIDAccount)
|
||||
}
|
||||
|
||||
static func loadGatewayToken(instanceId: String) -> String? {
|
||||
let account = self.gatewayTokenAccount(instanceId: instanceId)
|
||||
let token = KeychainStore.loadString(service: self.gatewayService, account: account)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if token?.isEmpty == false { return token }
|
||||
|
||||
let legacyAccount = self.legacyBridgeTokenAccount(instanceId: instanceId)
|
||||
let legacy = KeychainStore.loadString(service: self.legacyBridgeService, account: legacyAccount)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if let legacy, !legacy.isEmpty {
|
||||
_ = KeychainStore.saveString(legacy, service: self.gatewayService, account: account)
|
||||
return legacy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
static func saveGatewayToken(_ token: String, instanceId: String) {
|
||||
_ = KeychainStore.saveString(
|
||||
token,
|
||||
service: self.gatewayService,
|
||||
account: self.gatewayTokenAccount(instanceId: instanceId))
|
||||
}
|
||||
|
||||
static func loadGatewayPassword(instanceId: String) -> String? {
|
||||
KeychainStore.loadString(
|
||||
service: self.gatewayService,
|
||||
account: self.gatewayPasswordAccount(instanceId: instanceId))?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
static func saveGatewayPassword(_ password: String, instanceId: String) {
|
||||
_ = KeychainStore.saveString(
|
||||
password,
|
||||
service: self.gatewayService,
|
||||
account: self.gatewayPasswordAccount(instanceId: instanceId))
|
||||
}
|
||||
|
||||
private static func gatewayTokenAccount(instanceId: String) -> String {
|
||||
"gateway-token.\(instanceId)"
|
||||
}
|
||||
|
||||
private static func legacyBridgeTokenAccount(instanceId: String) -> String {
|
||||
"bridge-token.\(instanceId)"
|
||||
}
|
||||
|
||||
private static func gatewayPasswordAccount(instanceId: String) -> String {
|
||||
"gateway-password.\(instanceId)"
|
||||
}
|
||||
|
||||
private static func ensureStableInstanceID() {
|
||||
let defaults = UserDefaults.standard
|
||||
|
||||
if let existing = defaults.string(forKey: self.instanceIdDefaultsKey)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!existing.isEmpty
|
||||
{
|
||||
if self.loadStableInstanceID() == nil {
|
||||
self.saveStableInstanceID(existing)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if let stored = self.loadStableInstanceID(), !stored.isEmpty {
|
||||
defaults.set(stored, forKey: self.instanceIdDefaultsKey)
|
||||
return
|
||||
}
|
||||
|
||||
let fresh = UUID().uuidString
|
||||
self.saveStableInstanceID(fresh)
|
||||
defaults.set(fresh, forKey: self.instanceIdDefaultsKey)
|
||||
}
|
||||
|
||||
private static func ensurePreferredGatewayStableID() {
|
||||
let defaults = UserDefaults.standard
|
||||
|
||||
if let existing = defaults.string(forKey: self.preferredGatewayStableIDDefaultsKey)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!existing.isEmpty
|
||||
{
|
||||
if self.loadPreferredGatewayStableID() == nil {
|
||||
self.savePreferredGatewayStableID(existing)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if let stored = self.loadPreferredGatewayStableID(), !stored.isEmpty {
|
||||
defaults.set(stored, forKey: self.preferredGatewayStableIDDefaultsKey)
|
||||
}
|
||||
}
|
||||
|
||||
private static func ensureLastDiscoveredGatewayStableID() {
|
||||
let defaults = UserDefaults.standard
|
||||
|
||||
if let existing = defaults.string(forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!existing.isEmpty
|
||||
{
|
||||
if self.loadLastDiscoveredGatewayStableID() == nil {
|
||||
self.saveLastDiscoveredGatewayStableID(existing)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if let stored = self.loadLastDiscoveredGatewayStableID(), !stored.isEmpty {
|
||||
defaults.set(stored, forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)
|
||||
}
|
||||
}
|
||||
|
||||
private static func migrateLegacyDefaults() {
|
||||
let defaults = UserDefaults.standard
|
||||
|
||||
if defaults.string(forKey: self.preferredGatewayStableIDDefaultsKey)?.isEmpty != false,
|
||||
let legacy = defaults.string(forKey: self.legacyPreferredBridgeStableIDDefaultsKey),
|
||||
!legacy.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
{
|
||||
defaults.set(legacy, forKey: self.preferredGatewayStableIDDefaultsKey)
|
||||
self.savePreferredGatewayStableID(legacy)
|
||||
}
|
||||
|
||||
if defaults.string(forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)?.isEmpty != false,
|
||||
let legacy = defaults.string(forKey: self.legacyLastDiscoveredBridgeStableIDDefaultsKey),
|
||||
!legacy.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
{
|
||||
defaults.set(legacy, forKey: self.lastDiscoveredGatewayStableIDDefaultsKey)
|
||||
self.saveLastDiscoveredGatewayStableID(legacy)
|
||||
}
|
||||
|
||||
if defaults.object(forKey: self.manualEnabledDefaultsKey) == nil,
|
||||
defaults.object(forKey: self.legacyManualEnabledDefaultsKey) != nil
|
||||
{
|
||||
defaults.set(
|
||||
defaults.bool(forKey: self.legacyManualEnabledDefaultsKey),
|
||||
forKey: self.manualEnabledDefaultsKey)
|
||||
}
|
||||
|
||||
if defaults.string(forKey: self.manualHostDefaultsKey)?.isEmpty != false,
|
||||
let legacy = defaults.string(forKey: self.legacyManualHostDefaultsKey),
|
||||
!legacy.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
||||
{
|
||||
defaults.set(legacy, forKey: self.manualHostDefaultsKey)
|
||||
}
|
||||
|
||||
if defaults.integer(forKey: self.manualPortDefaultsKey) == 0,
|
||||
defaults.integer(forKey: self.legacyManualPortDefaultsKey) > 0
|
||||
{
|
||||
defaults.set(
|
||||
defaults.integer(forKey: self.legacyManualPortDefaultsKey),
|
||||
forKey: self.manualPortDefaultsKey)
|
||||
}
|
||||
|
||||
if defaults.object(forKey: self.discoveryDebugLogsDefaultsKey) == nil,
|
||||
defaults.object(forKey: self.legacyDiscoveryDebugLogsDefaultsKey) != nil
|
||||
{
|
||||
defaults.set(
|
||||
defaults.bool(forKey: self.legacyDiscoveryDebugLogsDefaultsKey),
|
||||
forKey: self.discoveryDebugLogsDefaultsKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import Foundation
|
||||
import Security
|
||||
|
||||
enum KeychainStore {
|
||||
static func loadString(service: String, account: String) -> String? {
|
||||
let query: [String: Any] = [
|
||||
kSecClass as String: kSecClassGenericPassword,
|
||||
kSecAttrService as String: service,
|
||||
kSecAttrAccount as String: account,
|
||||
kSecReturnData as String: true,
|
||||
kSecMatchLimit as String: kSecMatchLimitOne,
|
||||
]
|
||||
|
||||
var item: CFTypeRef?
|
||||
let status = SecItemCopyMatching(query as CFDictionary, &item)
|
||||
guard status == errSecSuccess, let data = item as? Data else { return nil }
|
||||
return String(data: data, encoding: .utf8)
|
||||
}
|
||||
|
||||
static func saveString(_ value: String, service: String, account: String) -> Bool {
|
||||
let data = Data(value.utf8)
|
||||
let query: [String: Any] = [
|
||||
kSecClass as String: kSecClassGenericPassword,
|
||||
kSecAttrService as String: service,
|
||||
kSecAttrAccount as String: account,
|
||||
]
|
||||
|
||||
let update: [String: Any] = [kSecValueData as String: data]
|
||||
let status = SecItemUpdate(query as CFDictionary, update as CFDictionary)
|
||||
if status == errSecSuccess { return true }
|
||||
if status != errSecItemNotFound { return false }
|
||||
|
||||
var insert = query
|
||||
insert[kSecValueData as String] = data
|
||||
insert[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly
|
||||
return SecItemAdd(insert as CFDictionary, nil) == errSecSuccess
|
||||
}
|
||||
|
||||
static func delete(service: String, account: String) -> Bool {
|
||||
let query: [String: Any] = [
|
||||
kSecClass as String: kSecClassGenericPassword,
|
||||
kSecAttrService as String: service,
|
||||
kSecAttrAccount as String: account,
|
||||
]
|
||||
let status = SecItemDelete(query as CFDictionary)
|
||||
return status == errSecSuccess || status == errSecItemNotFound
|
||||
}
|
||||
}
|
||||
72
docker-compose/ez-assistant/apps/ios/Sources/Info.plist
Normal file
@@ -0,0 +1,72 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDevelopmentRegion</key>
|
||||
<string>$(DEVELOPMENT_LANGUAGE)</string>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Moltbot</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIconName</key>
|
||||
<string>AppIcon</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>2026.1.26</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>20260126</string>
|
||||
<key>NSAppTransportSecurity</key>
|
||||
<dict>
|
||||
<key>NSAllowsArbitraryLoadsInWebContent</key>
|
||||
<true/>
|
||||
</dict>
|
||||
<key>NSBonjourServices</key>
|
||||
<array>
|
||||
<string>_moltbot-gw._tcp</string>
|
||||
</array>
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>Moltbot can capture photos or short video clips when requested via the gateway.</string>
|
||||
<key>NSLocalNetworkUsageDescription</key>
|
||||
<string>Moltbot discovers and connects to your Moltbot gateway on the local network.</string>
|
||||
<key>NSLocationAlwaysAndWhenInUseUsageDescription</key>
|
||||
<string>Moltbot can share your location in the background when you enable Always.</string>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Moltbot uses your location when you allow location sharing.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Moltbot needs microphone access for voice wake.</string>
|
||||
<key>NSSpeechRecognitionUsageDescription</key>
|
||||
<string>Moltbot uses on-device speech recognition for voice wake.</string>
|
||||
<key>UIApplicationSceneManifest</key>
|
||||
<dict>
|
||||
<key>UIApplicationSupportsMultipleScenes</key>
|
||||
<false/>
|
||||
</dict>
|
||||
<key>UIBackgroundModes</key>
|
||||
<array>
|
||||
<string>audio</string>
|
||||
</array>
|
||||
<key>UILaunchScreen</key>
|
||||
<dict/>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations~ipad</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,138 @@
|
||||
import MoltbotKit
|
||||
import CoreLocation
|
||||
import Foundation
|
||||
|
||||
@MainActor
|
||||
final class LocationService: NSObject, CLLocationManagerDelegate {
|
||||
enum Error: Swift.Error {
|
||||
case timeout
|
||||
case unavailable
|
||||
}
|
||||
|
||||
private let manager = CLLocationManager()
|
||||
private var authContinuation: CheckedContinuation<CLAuthorizationStatus, Never>?
|
||||
private var locationContinuation: CheckedContinuation<CLLocation, Swift.Error>?
|
||||
|
||||
override init() {
|
||||
super.init()
|
||||
self.manager.delegate = self
|
||||
self.manager.desiredAccuracy = kCLLocationAccuracyBest
|
||||
}
|
||||
|
||||
func authorizationStatus() -> CLAuthorizationStatus {
|
||||
self.manager.authorizationStatus
|
||||
}
|
||||
|
||||
func accuracyAuthorization() -> CLAccuracyAuthorization {
|
||||
if #available(iOS 14.0, *) {
|
||||
return self.manager.accuracyAuthorization
|
||||
}
|
||||
return .fullAccuracy
|
||||
}
|
||||
|
||||
func ensureAuthorization(mode: MoltbotLocationMode) async -> CLAuthorizationStatus {
|
||||
guard CLLocationManager.locationServicesEnabled() else { return .denied }
|
||||
|
||||
let status = self.manager.authorizationStatus
|
||||
if status == .notDetermined {
|
||||
self.manager.requestWhenInUseAuthorization()
|
||||
let updated = await self.awaitAuthorizationChange()
|
||||
if mode != .always { return updated }
|
||||
}
|
||||
|
||||
if mode == .always {
|
||||
let current = self.manager.authorizationStatus
|
||||
if current == .authorizedWhenInUse {
|
||||
self.manager.requestAlwaysAuthorization()
|
||||
return await self.awaitAuthorizationChange()
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
return self.manager.authorizationStatus
|
||||
}
|
||||
|
||||
func currentLocation(
|
||||
params: MoltbotLocationGetParams,
|
||||
desiredAccuracy: MoltbotLocationAccuracy,
|
||||
maxAgeMs: Int?,
|
||||
timeoutMs: Int?) async throws -> CLLocation
|
||||
{
|
||||
let now = Date()
|
||||
if let maxAgeMs,
|
||||
let cached = self.manager.location,
|
||||
now.timeIntervalSince(cached.timestamp) * 1000 <= Double(maxAgeMs)
|
||||
{
|
||||
return cached
|
||||
}
|
||||
|
||||
self.manager.desiredAccuracy = Self.accuracyValue(desiredAccuracy)
|
||||
let timeout = max(0, timeoutMs ?? 10000)
|
||||
return try await self.withTimeout(timeoutMs: timeout) {
|
||||
try await self.requestLocation()
|
||||
}
|
||||
}
|
||||
|
||||
private func requestLocation() async throws -> CLLocation {
|
||||
try await withCheckedThrowingContinuation { cont in
|
||||
self.locationContinuation = cont
|
||||
self.manager.requestLocation()
|
||||
}
|
||||
}
|
||||
|
||||
private func awaitAuthorizationChange() async -> CLAuthorizationStatus {
|
||||
await withCheckedContinuation { cont in
|
||||
self.authContinuation = cont
|
||||
}
|
||||
}
|
||||
|
||||
private func withTimeout<T: Sendable>(
|
||||
timeoutMs: Int,
|
||||
operation: @escaping @Sendable () async throws -> T) async throws -> T
|
||||
{
|
||||
try await AsyncTimeout.withTimeoutMs(timeoutMs: timeoutMs, onTimeout: { Error.timeout }, operation: operation)
|
||||
}
|
||||
|
||||
private static func accuracyValue(_ accuracy: MoltbotLocationAccuracy) -> CLLocationAccuracy {
|
||||
switch accuracy {
|
||||
case .coarse:
|
||||
kCLLocationAccuracyKilometer
|
||||
case .balanced:
|
||||
kCLLocationAccuracyHundredMeters
|
||||
case .precise:
|
||||
kCLLocationAccuracyBest
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) {
|
||||
let status = manager.authorizationStatus
|
||||
Task { @MainActor in
|
||||
if let cont = self.authContinuation {
|
||||
self.authContinuation = nil
|
||||
cont.resume(returning: status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func locationManager(_ manager: CLLocationManager, didUpdateLocations locations: [CLLocation]) {
|
||||
let locs = locations
|
||||
Task { @MainActor in
|
||||
guard let cont = self.locationContinuation else { return }
|
||||
self.locationContinuation = nil
|
||||
if let latest = locs.last {
|
||||
cont.resume(returning: latest)
|
||||
} else {
|
||||
cont.resume(throwing: Error.unavailable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nonisolated func locationManager(_ manager: CLLocationManager, didFailWithError error: Swift.Error) {
|
||||
let err = error
|
||||
Task { @MainActor in
|
||||
guard let cont = self.locationContinuation else { return }
|
||||
self.locationContinuation = nil
|
||||
cont.resume(throwing: err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,959 @@
|
||||
import MoltbotKit
|
||||
import Network
|
||||
import Observation
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class NodeAppModel {
|
||||
enum CameraHUDKind {
|
||||
case photo
|
||||
case recording
|
||||
case success
|
||||
case error
|
||||
}
|
||||
|
||||
var isBackgrounded: Bool = false
|
||||
let screen = ScreenController()
|
||||
let camera = CameraController()
|
||||
private let screenRecorder = ScreenRecordService()
|
||||
var gatewayStatusText: String = "Offline"
|
||||
var gatewayServerName: String?
|
||||
var gatewayRemoteAddress: String?
|
||||
var connectedGatewayID: String?
|
||||
var seamColorHex: String?
|
||||
var mainSessionKey: String = "main"
|
||||
|
||||
private let gateway = GatewayNodeSession()
|
||||
private var gatewayTask: Task<Void, Never>?
|
||||
private var voiceWakeSyncTask: Task<Void, Never>?
|
||||
@ObservationIgnored private var cameraHUDDismissTask: Task<Void, Never>?
|
||||
let voiceWake = VoiceWakeManager()
|
||||
let talkMode = TalkModeManager()
|
||||
private let locationService = LocationService()
|
||||
private var lastAutoA2uiURL: String?
|
||||
|
||||
private var gatewayConnected = false
|
||||
var gatewaySession: GatewayNodeSession { self.gateway }
|
||||
|
||||
var cameraHUDText: String?
|
||||
var cameraHUDKind: CameraHUDKind?
|
||||
var cameraFlashNonce: Int = 0
|
||||
var screenRecordActive: Bool = false
|
||||
|
||||
init() {
|
||||
self.voiceWake.configure { [weak self] cmd in
|
||||
guard let self else { return }
|
||||
let sessionKey = await MainActor.run { self.mainSessionKey }
|
||||
do {
|
||||
try await self.sendVoiceTranscript(text: cmd, sessionKey: sessionKey)
|
||||
} catch {
|
||||
// Best-effort only.
|
||||
}
|
||||
}
|
||||
|
||||
let enabled = UserDefaults.standard.bool(forKey: "voiceWake.enabled")
|
||||
self.voiceWake.setEnabled(enabled)
|
||||
self.talkMode.attachGateway(self.gateway)
|
||||
let talkEnabled = UserDefaults.standard.bool(forKey: "talk.enabled")
|
||||
self.talkMode.setEnabled(talkEnabled)
|
||||
|
||||
// Wire up deep links from canvas taps
|
||||
self.screen.onDeepLink = { [weak self] url in
|
||||
guard let self else { return }
|
||||
Task { @MainActor in
|
||||
await self.handleDeepLink(url: url)
|
||||
}
|
||||
}
|
||||
|
||||
// Wire up A2UI action clicks (buttons, etc.)
|
||||
self.screen.onA2UIAction = { [weak self] body in
|
||||
guard let self else { return }
|
||||
Task { @MainActor in
|
||||
await self.handleCanvasA2UIAction(body: body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCanvasA2UIAction(body: [String: Any]) async {
|
||||
let userActionAny = body["userAction"] ?? body
|
||||
let userAction: [String: Any] = {
|
||||
if let dict = userActionAny as? [String: Any] { return dict }
|
||||
if let dict = userActionAny as? [AnyHashable: Any] {
|
||||
return dict.reduce(into: [String: Any]()) { acc, pair in
|
||||
guard let key = pair.key as? String else { return }
|
||||
acc[key] = pair.value
|
||||
}
|
||||
}
|
||||
return [:]
|
||||
}()
|
||||
guard !userAction.isEmpty else { return }
|
||||
|
||||
guard let name = MoltbotCanvasA2UIAction.extractActionName(userAction) else { return }
|
||||
let actionId: String = {
|
||||
let id = (userAction["id"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return id.isEmpty ? UUID().uuidString : id
|
||||
}()
|
||||
|
||||
let surfaceId: String = {
|
||||
let raw = (userAction["surfaceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return raw.isEmpty ? "main" : raw
|
||||
}()
|
||||
let sourceComponentId: String = {
|
||||
let raw = (userAction[
|
||||
"sourceComponentId",
|
||||
] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return raw.isEmpty ? "-" : raw
|
||||
}()
|
||||
|
||||
let host = UserDefaults.standard.string(forKey: "node.displayName") ?? UIDevice.current.name
|
||||
let instanceId = (UserDefaults.standard.string(forKey: "node.instanceId") ?? "ios-node").lowercased()
|
||||
let contextJSON = MoltbotCanvasA2UIAction.compactJSON(userAction["context"])
|
||||
let sessionKey = self.mainSessionKey
|
||||
|
||||
let messageContext = MoltbotCanvasA2UIAction.AgentMessageContext(
|
||||
actionName: name,
|
||||
session: .init(key: sessionKey, surfaceId: surfaceId),
|
||||
component: .init(id: sourceComponentId, host: host, instanceId: instanceId),
|
||||
contextJSON: contextJSON)
|
||||
let message = MoltbotCanvasA2UIAction.formatAgentMessage(messageContext)
|
||||
|
||||
let ok: Bool
|
||||
var errorText: String?
|
||||
if await !self.isGatewayConnected() {
|
||||
ok = false
|
||||
errorText = "gateway not connected"
|
||||
} else {
|
||||
do {
|
||||
try await self.sendAgentRequest(link: AgentDeepLink(
|
||||
message: message,
|
||||
sessionKey: sessionKey,
|
||||
thinking: "low",
|
||||
deliver: false,
|
||||
to: nil,
|
||||
channel: nil,
|
||||
timeoutSeconds: nil,
|
||||
key: actionId))
|
||||
ok = true
|
||||
} catch {
|
||||
ok = false
|
||||
errorText = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
let js = MoltbotCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId: actionId, ok: ok, error: errorText)
|
||||
do {
|
||||
_ = try await self.screen.eval(javaScript: js)
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private func resolveA2UIHostURL() async -> String? {
|
||||
guard let raw = await self.gateway.currentCanvasHostUrl() else { return nil }
|
||||
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil }
|
||||
return base.appendingPathComponent("__moltbot__/a2ui/").absoluteString + "?platform=ios"
|
||||
}
|
||||
|
||||
private func showA2UIOnConnectIfNeeded() async {
|
||||
guard let a2uiUrl = await self.resolveA2UIHostURL() else { return }
|
||||
let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if current.isEmpty || current == self.lastAutoA2uiURL {
|
||||
self.screen.navigate(to: a2uiUrl)
|
||||
self.lastAutoA2uiURL = a2uiUrl
|
||||
}
|
||||
}
|
||||
|
||||
private func showLocalCanvasOnDisconnect() {
|
||||
self.lastAutoA2uiURL = nil
|
||||
self.screen.showDefaultCanvas()
|
||||
}
|
||||
|
||||
func setScenePhase(_ phase: ScenePhase) {
|
||||
switch phase {
|
||||
case .background:
|
||||
self.isBackgrounded = true
|
||||
case .active, .inactive:
|
||||
self.isBackgrounded = false
|
||||
@unknown default:
|
||||
self.isBackgrounded = false
|
||||
}
|
||||
}
|
||||
|
||||
func setVoiceWakeEnabled(_ enabled: Bool) {
|
||||
self.voiceWake.setEnabled(enabled)
|
||||
}
|
||||
|
||||
func setTalkEnabled(_ enabled: Bool) {
|
||||
self.talkMode.setEnabled(enabled)
|
||||
}
|
||||
|
||||
func requestLocationPermissions(mode: MoltbotLocationMode) async -> Bool {
|
||||
guard mode != .off else { return true }
|
||||
let status = await self.locationService.ensureAuthorization(mode: mode)
|
||||
switch status {
|
||||
case .authorizedAlways:
|
||||
return true
|
||||
case .authorizedWhenInUse:
|
||||
return mode != .always
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func connectToGateway(
|
||||
url: URL,
|
||||
gatewayStableID: String,
|
||||
tls: GatewayTLSParams?,
|
||||
token: String?,
|
||||
password: String?,
|
||||
connectOptions: GatewayConnectOptions)
|
||||
{
|
||||
self.gatewayTask?.cancel()
|
||||
self.gatewayServerName = nil
|
||||
self.gatewayRemoteAddress = nil
|
||||
let id = gatewayStableID.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.connectedGatewayID = id.isEmpty ? url.absoluteString : id
|
||||
self.gatewayConnected = false
|
||||
self.voiceWakeSyncTask?.cancel()
|
||||
self.voiceWakeSyncTask = nil
|
||||
let sessionBox = tls.map { WebSocketSessionBox(session: GatewayTLSPinningSession(params: $0)) }
|
||||
|
||||
self.gatewayTask = Task {
|
||||
var attempt = 0
|
||||
while !Task.isCancelled {
|
||||
await MainActor.run {
|
||||
if attempt == 0 {
|
||||
self.gatewayStatusText = "Connecting…"
|
||||
} else {
|
||||
self.gatewayStatusText = "Reconnecting…"
|
||||
}
|
||||
self.gatewayServerName = nil
|
||||
self.gatewayRemoteAddress = nil
|
||||
}
|
||||
|
||||
do {
|
||||
try await self.gateway.connect(
|
||||
url: url,
|
||||
token: token,
|
||||
password: password,
|
||||
connectOptions: connectOptions,
|
||||
sessionBox: sessionBox,
|
||||
onConnected: { [weak self] in
|
||||
guard let self else { return }
|
||||
await MainActor.run {
|
||||
self.gatewayStatusText = "Connected"
|
||||
self.gatewayServerName = url.host ?? "gateway"
|
||||
self.gatewayConnected = true
|
||||
}
|
||||
if let addr = await self.gateway.currentRemoteAddress() {
|
||||
await MainActor.run {
|
||||
self.gatewayRemoteAddress = addr
|
||||
}
|
||||
}
|
||||
await self.refreshBrandingFromGateway()
|
||||
await self.startVoiceWakeSync()
|
||||
await self.showA2UIOnConnectIfNeeded()
|
||||
},
|
||||
onDisconnected: { [weak self] reason in
|
||||
guard let self else { return }
|
||||
await MainActor.run {
|
||||
self.gatewayStatusText = "Disconnected"
|
||||
self.gatewayRemoteAddress = nil
|
||||
self.gatewayConnected = false
|
||||
self.showLocalCanvasOnDisconnect()
|
||||
self.gatewayStatusText = "Disconnected: \(reason)"
|
||||
}
|
||||
},
|
||||
onInvoke: { [weak self] req in
|
||||
guard let self else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "UNAVAILABLE: node not ready"))
|
||||
}
|
||||
return await self.handleInvoke(req)
|
||||
})
|
||||
|
||||
if Task.isCancelled { break }
|
||||
attempt = 0
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
} catch {
|
||||
if Task.isCancelled { break }
|
||||
attempt += 1
|
||||
await MainActor.run {
|
||||
self.gatewayStatusText = "Gateway error: \(error.localizedDescription)"
|
||||
self.gatewayServerName = nil
|
||||
self.gatewayRemoteAddress = nil
|
||||
self.gatewayConnected = false
|
||||
self.showLocalCanvasOnDisconnect()
|
||||
}
|
||||
let sleepSeconds = min(8.0, 0.5 * pow(1.7, Double(attempt)))
|
||||
try? await Task.sleep(nanoseconds: UInt64(sleepSeconds * 1_000_000_000))
|
||||
}
|
||||
}
|
||||
|
||||
await MainActor.run {
|
||||
self.gatewayStatusText = "Offline"
|
||||
self.gatewayServerName = nil
|
||||
self.gatewayRemoteAddress = nil
|
||||
self.connectedGatewayID = nil
|
||||
self.gatewayConnected = false
|
||||
self.seamColorHex = nil
|
||||
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
|
||||
self.mainSessionKey = "main"
|
||||
self.talkMode.updateMainSessionKey(self.mainSessionKey)
|
||||
}
|
||||
self.showLocalCanvasOnDisconnect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func disconnectGateway() {
|
||||
self.gatewayTask?.cancel()
|
||||
self.gatewayTask = nil
|
||||
self.voiceWakeSyncTask?.cancel()
|
||||
self.voiceWakeSyncTask = nil
|
||||
Task { await self.gateway.disconnect() }
|
||||
self.gatewayStatusText = "Offline"
|
||||
self.gatewayServerName = nil
|
||||
self.gatewayRemoteAddress = nil
|
||||
self.connectedGatewayID = nil
|
||||
self.gatewayConnected = false
|
||||
self.seamColorHex = nil
|
||||
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
|
||||
self.mainSessionKey = "main"
|
||||
self.talkMode.updateMainSessionKey(self.mainSessionKey)
|
||||
}
|
||||
self.showLocalCanvasOnDisconnect()
|
||||
}
|
||||
|
||||
private func applyMainSessionKey(_ key: String?) {
|
||||
let trimmed = (key ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
let current = self.mainSessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if SessionKey.isCanonicalMainSessionKey(current) { return }
|
||||
if trimmed == current { return }
|
||||
self.mainSessionKey = trimmed
|
||||
self.talkMode.updateMainSessionKey(trimmed)
|
||||
}
|
||||
|
||||
var seamColor: Color {
|
||||
Self.color(fromHex: self.seamColorHex) ?? Self.defaultSeamColor
|
||||
}
|
||||
|
||||
private static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0)
|
||||
|
||||
private static func color(fromHex raw: String?) -> Color? {
|
||||
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed
|
||||
guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil }
|
||||
let r = Double((value >> 16) & 0xFF) / 255.0
|
||||
let g = Double((value >> 8) & 0xFF) / 255.0
|
||||
let b = Double(value & 0xFF) / 255.0
|
||||
return Color(red: r, green: g, blue: b)
|
||||
}
|
||||
|
||||
private func refreshBrandingFromGateway() async {
|
||||
do {
|
||||
let res = try await self.gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
|
||||
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
|
||||
guard let config = json["config"] as? [String: Any] else { return }
|
||||
let ui = config["ui"] as? [String: Any]
|
||||
let raw = (ui?["seamColor"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
let session = config["session"] as? [String: Any]
|
||||
let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String)
|
||||
await MainActor.run {
|
||||
self.seamColorHex = raw.isEmpty ? nil : raw
|
||||
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
|
||||
self.mainSessionKey = mainKey
|
||||
self.talkMode.updateMainSessionKey(mainKey)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
func setGlobalWakeWords(_ words: [String]) async {
|
||||
let sanitized = VoiceWakePreferences.sanitizeTriggerWords(words)
|
||||
|
||||
struct Payload: Codable {
|
||||
var triggers: [String]
|
||||
}
|
||||
let payload = Payload(triggers: sanitized)
|
||||
guard let data = try? JSONEncoder().encode(payload),
|
||||
let json = String(data: data, encoding: .utf8)
|
||||
else { return }
|
||||
|
||||
do {
|
||||
_ = try await self.gateway.request(method: "voicewake.set", paramsJSON: json, timeoutSeconds: 12)
|
||||
} catch {
|
||||
// Best-effort only.
|
||||
}
|
||||
}
|
||||
|
||||
private func startVoiceWakeSync() async {
|
||||
self.voiceWakeSyncTask?.cancel()
|
||||
self.voiceWakeSyncTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
|
||||
await self.refreshWakeWordsFromGateway()
|
||||
|
||||
let stream = await self.gateway.subscribeServerEvents(bufferingNewest: 200)
|
||||
for await evt in stream {
|
||||
if Task.isCancelled { return }
|
||||
guard evt.event == "voicewake.changed" else { continue }
|
||||
guard let payload = evt.payload else { continue }
|
||||
struct Payload: Decodable { var triggers: [String] }
|
||||
guard let decoded = try? GatewayPayloadDecoding.decode(payload, as: Payload.self) else { continue }
|
||||
let triggers = VoiceWakePreferences.sanitizeTriggerWords(decoded.triggers)
|
||||
VoiceWakePreferences.saveTriggerWords(triggers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func refreshWakeWordsFromGateway() async {
|
||||
do {
|
||||
let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8)
|
||||
guard let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: data) else { return }
|
||||
VoiceWakePreferences.saveTriggerWords(triggers)
|
||||
} catch {
|
||||
// Best-effort only.
|
||||
}
|
||||
}
|
||||
|
||||
func sendVoiceTranscript(text: String, sessionKey: String?) async throws {
|
||||
if await !self.isGatewayConnected() {
|
||||
throw NSError(domain: "Gateway", code: 10, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Gateway not connected",
|
||||
])
|
||||
}
|
||||
struct Payload: Codable {
|
||||
var text: String
|
||||
var sessionKey: String?
|
||||
}
|
||||
let payload = Payload(text: text, sessionKey: sessionKey)
|
||||
let data = try JSONEncoder().encode(payload)
|
||||
guard let json = String(bytes: data, encoding: .utf8) else {
|
||||
throw NSError(domain: "NodeAppModel", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Failed to encode voice transcript payload as UTF-8",
|
||||
])
|
||||
}
|
||||
await self.gateway.sendEvent(event: "voice.transcript", payloadJSON: json)
|
||||
}
|
||||
|
||||
func handleDeepLink(url: URL) async {
|
||||
guard let route = DeepLinkParser.parse(url) else { return }
|
||||
|
||||
switch route {
|
||||
case let .agent(link):
|
||||
await self.handleAgentDeepLink(link, originalURL: url)
|
||||
}
|
||||
}
|
||||
|
||||
private func handleAgentDeepLink(_ link: AgentDeepLink, originalURL: URL) async {
|
||||
let message = link.message.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !message.isEmpty else { return }
|
||||
|
||||
if message.count > 20000 {
|
||||
self.screen.errorText = "Deep link too large (message exceeds 20,000 characters)."
|
||||
return
|
||||
}
|
||||
|
||||
guard await self.isGatewayConnected() else {
|
||||
self.screen.errorText = "Gateway not connected (cannot forward deep link)."
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
try await self.sendAgentRequest(link: link)
|
||||
self.screen.errorText = nil
|
||||
} catch {
|
||||
self.screen.errorText = "Agent request failed: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
|
||||
private func sendAgentRequest(link: AgentDeepLink) async throws {
|
||||
if link.message.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
||||
throw NSError(domain: "DeepLink", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "invalid agent message",
|
||||
])
|
||||
}
|
||||
|
||||
// iOS gateway forwards to the gateway; no local auth prompts here.
|
||||
// (Key-based unattended auth is handled on macOS for moltbot:// links.)
|
||||
let data = try JSONEncoder().encode(link)
|
||||
guard let json = String(bytes: data, encoding: .utf8) else {
|
||||
throw NSError(domain: "NodeAppModel", code: 2, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Failed to encode agent request payload as UTF-8",
|
||||
])
|
||||
}
|
||||
await self.gateway.sendEvent(event: "agent.request", payloadJSON: json)
|
||||
}
|
||||
|
||||
private func isGatewayConnected() async -> Bool {
|
||||
self.gatewayConnected
|
||||
}
|
||||
|
||||
private func handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
|
||||
let command = req.command
|
||||
|
||||
if self.isBackgrounded, self.isBackgroundRestricted(command) {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .backgroundUnavailable,
|
||||
message: "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground"))
|
||||
}
|
||||
|
||||
if command.hasPrefix("camera."), !self.isCameraEnabled() {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "CAMERA_DISABLED: enable Camera in iOS Settings → Camera → Allow Camera"))
|
||||
}
|
||||
|
||||
do {
|
||||
switch command {
|
||||
case MoltbotLocationCommand.get.rawValue:
|
||||
return try await self.handleLocationInvoke(req)
|
||||
case MoltbotCanvasCommand.present.rawValue,
|
||||
MoltbotCanvasCommand.hide.rawValue,
|
||||
MoltbotCanvasCommand.navigate.rawValue,
|
||||
MoltbotCanvasCommand.evalJS.rawValue,
|
||||
MoltbotCanvasCommand.snapshot.rawValue:
|
||||
return try await self.handleCanvasInvoke(req)
|
||||
case MoltbotCanvasA2UICommand.reset.rawValue,
|
||||
MoltbotCanvasA2UICommand.push.rawValue,
|
||||
MoltbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
return try await self.handleCanvasA2UIInvoke(req)
|
||||
case MoltbotCameraCommand.list.rawValue,
|
||||
MoltbotCameraCommand.snap.rawValue,
|
||||
MoltbotCameraCommand.clip.rawValue:
|
||||
return try await self.handleCameraInvoke(req)
|
||||
case MoltbotScreenCommand.record.rawValue:
|
||||
return try await self.handleScreenRecordInvoke(req)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
} catch {
|
||||
if command.hasPrefix("camera.") {
|
||||
let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
|
||||
self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2)
|
||||
}
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(code: .unavailable, message: error.localizedDescription))
|
||||
}
|
||||
}
|
||||
|
||||
private func isBackgroundRestricted(_ command: String) -> Bool {
|
||||
command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen.")
|
||||
}
|
||||
|
||||
private func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let mode = self.locationMode()
|
||||
guard mode != .off else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "LOCATION_DISABLED: enable Location in Settings"))
|
||||
}
|
||||
if self.isBackgrounded, mode != .always {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .backgroundUnavailable,
|
||||
message: "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always"))
|
||||
}
|
||||
let params = (try? Self.decodeParams(MoltbotLocationGetParams.self, from: req.paramsJSON)) ??
|
||||
MoltbotLocationGetParams()
|
||||
let desired = params.desiredAccuracy ??
|
||||
(self.isLocationPreciseEnabled() ? .precise : .balanced)
|
||||
let status = self.locationService.authorizationStatus()
|
||||
if status != .authorizedAlways, status != .authorizedWhenInUse {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "LOCATION_PERMISSION_REQUIRED: grant Location permission"))
|
||||
}
|
||||
if self.isBackgrounded, status != .authorizedAlways {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "LOCATION_PERMISSION_REQUIRED: enable Always for background access"))
|
||||
}
|
||||
let location = try await self.locationService.currentLocation(
|
||||
params: params,
|
||||
desiredAccuracy: desired,
|
||||
maxAgeMs: params.maxAgeMs,
|
||||
timeoutMs: params.timeoutMs)
|
||||
let isPrecise = self.locationService.accuracyAuthorization() == .fullAccuracy
|
||||
let payload = MoltbotLocationPayload(
|
||||
lat: location.coordinate.latitude,
|
||||
lon: location.coordinate.longitude,
|
||||
accuracyMeters: location.horizontalAccuracy,
|
||||
altitudeMeters: location.verticalAccuracy >= 0 ? location.altitude : nil,
|
||||
speedMps: location.speed >= 0 ? location.speed : nil,
|
||||
headingDeg: location.course >= 0 ? location.course : nil,
|
||||
timestamp: ISO8601DateFormatter().string(from: location.timestamp),
|
||||
isPrecise: isPrecise,
|
||||
source: nil)
|
||||
let json = try Self.encodePayload(payload)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
|
||||
}
|
||||
|
||||
private func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
switch req.command {
|
||||
case MoltbotCanvasCommand.present.rawValue:
|
||||
let params = (try? Self.decodeParams(MoltbotCanvasPresentParams.self, from: req.paramsJSON)) ??
|
||||
MoltbotCanvasPresentParams()
|
||||
let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
if url.isEmpty {
|
||||
self.screen.showDefaultCanvas()
|
||||
} else {
|
||||
self.screen.navigate(to: url)
|
||||
}
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
case MoltbotCanvasCommand.hide.rawValue:
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
case MoltbotCanvasCommand.navigate.rawValue:
|
||||
let params = try Self.decodeParams(MoltbotCanvasNavigateParams.self, from: req.paramsJSON)
|
||||
self.screen.navigate(to: params.url)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true)
|
||||
case MoltbotCanvasCommand.evalJS.rawValue:
|
||||
let params = try Self.decodeParams(MoltbotCanvasEvalParams.self, from: req.paramsJSON)
|
||||
let result = try await self.screen.eval(javaScript: params.javaScript)
|
||||
let payload = try Self.encodePayload(["result": result])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
case MoltbotCanvasCommand.snapshot.rawValue:
|
||||
let params = try? Self.decodeParams(MoltbotCanvasSnapshotParams.self, from: req.paramsJSON)
|
||||
let format = params?.format ?? .jpeg
|
||||
let maxWidth: CGFloat? = {
|
||||
if let raw = params?.maxWidth, raw > 0 { return CGFloat(raw) }
|
||||
// Keep default snapshots comfortably below the gateway client's maxPayload.
|
||||
// For full-res, clients should explicitly request a larger maxWidth.
|
||||
return switch format {
|
||||
case .png: 900
|
||||
case .jpeg: 1600
|
||||
}
|
||||
}()
|
||||
let base64 = try await self.screen.snapshotBase64(
|
||||
maxWidth: maxWidth,
|
||||
format: format,
|
||||
quality: params?.quality)
|
||||
let payload = try Self.encodePayload([
|
||||
"format": format == .jpeg ? "jpeg" : "png",
|
||||
"base64": base64,
|
||||
])
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCanvasA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let command = req.command
|
||||
switch command {
|
||||
case MoltbotCanvasA2UICommand.reset.rawValue:
|
||||
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
|
||||
}
|
||||
self.screen.navigate(to: a2uiUrl)
|
||||
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
|
||||
}
|
||||
|
||||
let json = try await self.screen.eval(javaScript: """
|
||||
(() => {
|
||||
if (!globalThis.clawdbotA2UI) return JSON.stringify({ ok: false, error: "missing moltbotA2UI" });
|
||||
return JSON.stringify(globalThis.clawdbotA2UI.reset());
|
||||
})()
|
||||
""")
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json)
|
||||
case MoltbotCanvasA2UICommand.push.rawValue, MoltbotCanvasA2UICommand.pushJSONL.rawValue:
|
||||
let messages: [AnyCodable]
|
||||
if command == MoltbotCanvasA2UICommand.pushJSONL.rawValue {
|
||||
let params = try Self.decodeParams(MoltbotCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
|
||||
messages = try MoltbotCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
|
||||
} else {
|
||||
do {
|
||||
let params = try Self.decodeParams(MoltbotCanvasA2UIPushParams.self, from: req.paramsJSON)
|
||||
messages = params.messages
|
||||
} catch {
|
||||
// Be forgiving: some clients still send JSONL payloads to `canvas.a2ui.push`.
|
||||
let params = try Self.decodeParams(MoltbotCanvasA2UIPushJSONLParams.self, from: req.paramsJSON)
|
||||
messages = try MoltbotCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl)
|
||||
}
|
||||
}
|
||||
|
||||
guard let a2uiUrl = await self.resolveA2UIHostURL() else {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host"))
|
||||
}
|
||||
self.screen.navigate(to: a2uiUrl)
|
||||
if await !self.screen.waitForA2UIReady(timeoutMs: 5000) {
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(
|
||||
code: .unavailable,
|
||||
message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable"))
|
||||
}
|
||||
|
||||
let messagesJSON = try MoltbotCanvasA2UIJSONL.encodeMessagesJSONArray(messages)
|
||||
let js = """
|
||||
(() => {
|
||||
try {
|
||||
if (!globalThis.clawdbotA2UI) return JSON.stringify({ ok: false, error: "missing moltbotA2UI" });
|
||||
const messages = \(messagesJSON);
|
||||
return JSON.stringify(globalThis.clawdbotA2UI.applyMessages(messages));
|
||||
} catch (e) {
|
||||
return JSON.stringify({ ok: false, error: String(e?.message ?? e) });
|
||||
}
|
||||
})()
|
||||
"""
|
||||
let resultJSON = try await self.screen.eval(javaScript: js)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
}
|
||||
|
||||
private func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
switch req.command {
|
||||
case MoltbotCameraCommand.list.rawValue:
|
||||
let devices = await self.camera.listDevices()
|
||||
struct Payload: Codable {
|
||||
var devices: [CameraController.CameraDeviceInfo]
|
||||
}
|
||||
let payload = try Self.encodePayload(Payload(devices: devices))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
case MoltbotCameraCommand.snap.rawValue:
|
||||
self.showCameraHUD(text: "Taking photo…", kind: .photo)
|
||||
self.triggerCameraFlash()
|
||||
let params = (try? Self.decodeParams(MoltbotCameraSnapParams.self, from: req.paramsJSON)) ??
|
||||
MoltbotCameraSnapParams()
|
||||
let res = try await self.camera.snap(params: params)
|
||||
|
||||
struct Payload: Codable {
|
||||
var format: String
|
||||
var base64: String
|
||||
var width: Int
|
||||
var height: Int
|
||||
}
|
||||
let payload = try Self.encodePayload(Payload(
|
||||
format: res.format,
|
||||
base64: res.base64,
|
||||
width: res.width,
|
||||
height: res.height))
|
||||
self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
case MoltbotCameraCommand.clip.rawValue:
|
||||
let params = (try? Self.decodeParams(MoltbotCameraClipParams.self, from: req.paramsJSON)) ??
|
||||
MoltbotCameraClipParams()
|
||||
|
||||
let suspended = (params.includeAudio ?? true) ? self.voiceWake.suspendForExternalAudioCapture() : false
|
||||
defer { self.voiceWake.resumeAfterExternalAudioCapture(wasSuspended: suspended) }
|
||||
|
||||
self.showCameraHUD(text: "Recording…", kind: .recording)
|
||||
let res = try await self.camera.clip(params: params)
|
||||
|
||||
struct Payload: Codable {
|
||||
var format: String
|
||||
var base64: String
|
||||
var durationMs: Int
|
||||
var hasAudio: Bool
|
||||
}
|
||||
let payload = try Self.encodePayload(Payload(
|
||||
format: res.format,
|
||||
base64: res.base64,
|
||||
durationMs: res.durationMs,
|
||||
hasAudio: res.hasAudio))
|
||||
self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8)
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
default:
|
||||
return BridgeInvokeResponse(
|
||||
id: req.id,
|
||||
ok: false,
|
||||
error: MoltbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command"))
|
||||
}
|
||||
}
|
||||
|
||||
private func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse {
|
||||
let params = (try? Self.decodeParams(MoltbotScreenRecordParams.self, from: req.paramsJSON)) ??
|
||||
MoltbotScreenRecordParams()
|
||||
if let format = params.format, format.lowercased() != "mp4" {
|
||||
throw NSError(domain: "Screen", code: 30, userInfo: [
|
||||
NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4",
|
||||
])
|
||||
}
|
||||
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
|
||||
self.screenRecordActive = true
|
||||
defer { self.screenRecordActive = false }
|
||||
let path = try await self.screenRecorder.record(
|
||||
screenIndex: params.screenIndex,
|
||||
durationMs: params.durationMs,
|
||||
fps: params.fps,
|
||||
includeAudio: params.includeAudio,
|
||||
outPath: nil)
|
||||
defer { try? FileManager().removeItem(atPath: path) }
|
||||
let data = try Data(contentsOf: URL(fileURLWithPath: path))
|
||||
struct Payload: Codable {
|
||||
var format: String
|
||||
var base64: String
|
||||
var durationMs: Int?
|
||||
var fps: Double?
|
||||
var screenIndex: Int?
|
||||
var hasAudio: Bool
|
||||
}
|
||||
let payload = try Self.encodePayload(Payload(
|
||||
format: "mp4",
|
||||
base64: data.base64EncodedString(),
|
||||
durationMs: params.durationMs,
|
||||
fps: params.fps,
|
||||
screenIndex: params.screenIndex,
|
||||
hasAudio: params.includeAudio ?? true))
|
||||
return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private extension NodeAppModel {
|
||||
func locationMode() -> MoltbotLocationMode {
|
||||
let raw = UserDefaults.standard.string(forKey: "location.enabledMode") ?? "off"
|
||||
return MoltbotLocationMode(rawValue: raw) ?? .off
|
||||
}
|
||||
|
||||
func isLocationPreciseEnabled() -> Bool {
|
||||
if UserDefaults.standard.object(forKey: "location.preciseEnabled") == nil { return true }
|
||||
return UserDefaults.standard.bool(forKey: "location.preciseEnabled")
|
||||
}
|
||||
|
||||
static func decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
|
||||
guard let json, let data = json.data(using: .utf8) else {
|
||||
throw NSError(domain: "Gateway", code: 20, userInfo: [
|
||||
NSLocalizedDescriptionKey: "INVALID_REQUEST: paramsJSON required",
|
||||
])
|
||||
}
|
||||
return try JSONDecoder().decode(type, from: data)
|
||||
}
|
||||
|
||||
static func encodePayload(_ obj: some Encodable) throws -> String {
|
||||
let data = try JSONEncoder().encode(obj)
|
||||
guard let json = String(bytes: data, encoding: .utf8) else {
|
||||
throw NSError(domain: "NodeAppModel", code: 21, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Failed to encode payload as UTF-8",
|
||||
])
|
||||
}
|
||||
return json
|
||||
}
|
||||
|
||||
func isCameraEnabled() -> Bool {
|
||||
// Default-on: if the key doesn't exist yet, treat it as enabled.
|
||||
if UserDefaults.standard.object(forKey: "camera.enabled") == nil { return true }
|
||||
return UserDefaults.standard.bool(forKey: "camera.enabled")
|
||||
}
|
||||
|
||||
func triggerCameraFlash() {
|
||||
self.cameraFlashNonce &+= 1
|
||||
}
|
||||
|
||||
func showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
|
||||
self.cameraHUDDismissTask?.cancel()
|
||||
|
||||
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
|
||||
self.cameraHUDText = text
|
||||
self.cameraHUDKind = kind
|
||||
}
|
||||
|
||||
guard let autoHideSeconds else { return }
|
||||
self.cameraHUDDismissTask = Task { @MainActor in
|
||||
try? await Task.sleep(nanoseconds: UInt64(autoHideSeconds * 1_000_000_000))
|
||||
withAnimation(.easeOut(duration: 0.25)) {
|
||||
self.cameraHUDText = nil
|
||||
self.cameraHUDKind = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
extension NodeAppModel {
|
||||
func _test_handleInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse {
|
||||
await self.handleInvoke(req)
|
||||
}
|
||||
|
||||
static func _test_decodeParams<T: Decodable>(_ type: T.Type, from json: String?) throws -> T {
|
||||
try self.decodeParams(type, from: json)
|
||||
}
|
||||
|
||||
static func _test_encodePayload(_ obj: some Encodable) throws -> String {
|
||||
try self.encodePayload(obj)
|
||||
}
|
||||
|
||||
func _test_isCameraEnabled() -> Bool {
|
||||
self.isCameraEnabled()
|
||||
}
|
||||
|
||||
func _test_triggerCameraFlash() {
|
||||
self.triggerCameraFlash()
|
||||
}
|
||||
|
||||
func _test_showCameraHUD(text: String, kind: CameraHUDKind, autoHideSeconds: Double? = nil) {
|
||||
self.showCameraHUD(text: text, kind: kind, autoHideSeconds: autoHideSeconds)
|
||||
}
|
||||
|
||||
func _test_handleCanvasA2UIAction(body: [String: Any]) async {
|
||||
await self.handleCanvasA2UIAction(body: body)
|
||||
}
|
||||
|
||||
func _test_resolveA2UIHostURL() async -> String? {
|
||||
await self.resolveA2UIHostURL()
|
||||
}
|
||||
|
||||
func _test_showLocalCanvasOnDisconnect() {
|
||||
self.showLocalCanvasOnDisconnect()
|
||||
}
|
||||
}
|
||||
#endif
|
||||
341
docker-compose/ez-assistant/apps/ios/Sources/RootCanvas.swift
Normal file
@@ -0,0 +1,341 @@
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
struct RootCanvas: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
@Environment(VoiceWakeManager.self) private var voiceWake
|
||||
@Environment(\.colorScheme) private var systemColorScheme
|
||||
@Environment(\.scenePhase) private var scenePhase
|
||||
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
|
||||
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
|
||||
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
|
||||
@State private var presentedSheet: PresentedSheet?
|
||||
@State private var voiceWakeToastText: String?
|
||||
@State private var toastDismissTask: Task<Void, Never>?
|
||||
|
||||
private enum PresentedSheet: Identifiable {
|
||||
case settings
|
||||
case chat
|
||||
|
||||
var id: Int {
|
||||
switch self {
|
||||
case .settings: 0
|
||||
case .chat: 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
ZStack {
|
||||
CanvasContent(
|
||||
systemColorScheme: self.systemColorScheme,
|
||||
gatewayStatus: self.gatewayStatus,
|
||||
voiceWakeEnabled: self.voiceWakeEnabled,
|
||||
voiceWakeToastText: self.voiceWakeToastText,
|
||||
cameraHUDText: self.appModel.cameraHUDText,
|
||||
cameraHUDKind: self.appModel.cameraHUDKind,
|
||||
openChat: {
|
||||
self.presentedSheet = .chat
|
||||
},
|
||||
openSettings: {
|
||||
self.presentedSheet = .settings
|
||||
})
|
||||
.preferredColorScheme(.dark)
|
||||
|
||||
if self.appModel.cameraFlashNonce != 0 {
|
||||
CameraFlashOverlay(nonce: self.appModel.cameraFlashNonce)
|
||||
}
|
||||
}
|
||||
.sheet(item: self.$presentedSheet) { sheet in
|
||||
switch sheet {
|
||||
case .settings:
|
||||
SettingsTab()
|
||||
case .chat:
|
||||
ChatSheet(
|
||||
gateway: self.appModel.gatewaySession,
|
||||
sessionKey: self.appModel.mainSessionKey,
|
||||
userAccent: self.appModel.seamColor)
|
||||
}
|
||||
}
|
||||
.onAppear { self.updateIdleTimer() }
|
||||
.onChange(of: self.preventSleep) { _, _ in self.updateIdleTimer() }
|
||||
.onChange(of: self.scenePhase) { _, _ in self.updateIdleTimer() }
|
||||
.onAppear { self.updateCanvasDebugStatus() }
|
||||
.onChange(of: self.canvasDebugStatusEnabled) { _, _ in self.updateCanvasDebugStatus() }
|
||||
.onChange(of: self.appModel.gatewayStatusText) { _, _ in self.updateCanvasDebugStatus() }
|
||||
.onChange(of: self.appModel.gatewayServerName) { _, _ in self.updateCanvasDebugStatus() }
|
||||
.onChange(of: self.appModel.gatewayRemoteAddress) { _, _ in self.updateCanvasDebugStatus() }
|
||||
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
|
||||
guard let newValue else { return }
|
||||
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
|
||||
self.toastDismissTask?.cancel()
|
||||
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
|
||||
self.voiceWakeToastText = trimmed
|
||||
}
|
||||
|
||||
self.toastDismissTask = Task {
|
||||
try? await Task.sleep(nanoseconds: 2_300_000_000)
|
||||
await MainActor.run {
|
||||
withAnimation(.easeOut(duration: 0.25)) {
|
||||
self.voiceWakeToastText = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.onDisappear {
|
||||
UIApplication.shared.isIdleTimerDisabled = false
|
||||
self.toastDismissTask?.cancel()
|
||||
self.toastDismissTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
private var gatewayStatus: StatusPill.GatewayState {
|
||||
if self.appModel.gatewayServerName != nil { return .connected }
|
||||
|
||||
let text = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if text.localizedCaseInsensitiveContains("connecting") ||
|
||||
text.localizedCaseInsensitiveContains("reconnecting")
|
||||
{
|
||||
return .connecting
|
||||
}
|
||||
|
||||
if text.localizedCaseInsensitiveContains("error") {
|
||||
return .error
|
||||
}
|
||||
|
||||
return .disconnected
|
||||
}
|
||||
|
||||
private func updateIdleTimer() {
|
||||
UIApplication.shared.isIdleTimerDisabled = (self.scenePhase == .active && self.preventSleep)
|
||||
}
|
||||
|
||||
private func updateCanvasDebugStatus() {
|
||||
self.appModel.screen.setDebugStatusEnabled(self.canvasDebugStatusEnabled)
|
||||
guard self.canvasDebugStatusEnabled else { return }
|
||||
let title = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let subtitle = self.appModel.gatewayServerName ?? self.appModel.gatewayRemoteAddress
|
||||
self.appModel.screen.updateDebugStatus(title: title, subtitle: subtitle)
|
||||
}
|
||||
}
|
||||
|
||||
private struct CanvasContent: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
|
||||
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
|
||||
var systemColorScheme: ColorScheme
|
||||
var gatewayStatus: StatusPill.GatewayState
|
||||
var voiceWakeEnabled: Bool
|
||||
var voiceWakeToastText: String?
|
||||
var cameraHUDText: String?
|
||||
var cameraHUDKind: NodeAppModel.CameraHUDKind?
|
||||
var openChat: () -> Void
|
||||
var openSettings: () -> Void
|
||||
|
||||
private var brightenButtons: Bool { self.systemColorScheme == .light }
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .topTrailing) {
|
||||
ScreenTab()
|
||||
|
||||
VStack(spacing: 10) {
|
||||
OverlayButton(systemImage: "text.bubble.fill", brighten: self.brightenButtons) {
|
||||
self.openChat()
|
||||
}
|
||||
.accessibilityLabel("Chat")
|
||||
|
||||
if self.talkButtonEnabled {
|
||||
// Talk mode lives on a side bubble so it doesn't get buried in settings.
|
||||
OverlayButton(
|
||||
systemImage: self.appModel.talkMode.isEnabled ? "waveform.circle.fill" : "waveform.circle",
|
||||
brighten: self.brightenButtons,
|
||||
tint: self.appModel.seamColor,
|
||||
isActive: self.appModel.talkMode.isEnabled)
|
||||
{
|
||||
let next = !self.appModel.talkMode.isEnabled
|
||||
self.talkEnabled = next
|
||||
self.appModel.setTalkEnabled(next)
|
||||
}
|
||||
.accessibilityLabel("Talk Mode")
|
||||
}
|
||||
|
||||
OverlayButton(systemImage: "gearshape.fill", brighten: self.brightenButtons) {
|
||||
self.openSettings()
|
||||
}
|
||||
.accessibilityLabel("Settings")
|
||||
}
|
||||
.padding(.top, 10)
|
||||
.padding(.trailing, 10)
|
||||
}
|
||||
.overlay(alignment: .center) {
|
||||
if self.appModel.talkMode.isEnabled {
|
||||
TalkOrbOverlay()
|
||||
.transition(.opacity)
|
||||
}
|
||||
}
|
||||
.overlay(alignment: .topLeading) {
|
||||
StatusPill(
|
||||
gateway: self.gatewayStatus,
|
||||
voiceWakeEnabled: self.voiceWakeEnabled,
|
||||
activity: self.statusActivity,
|
||||
brighten: self.brightenButtons,
|
||||
onTap: {
|
||||
self.openSettings()
|
||||
})
|
||||
.padding(.leading, 10)
|
||||
.safeAreaPadding(.top, 10)
|
||||
}
|
||||
.overlay(alignment: .topLeading) {
|
||||
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
|
||||
VoiceWakeToast(
|
||||
command: voiceWakeToastText,
|
||||
brighten: self.brightenButtons)
|
||||
.padding(.leading, 10)
|
||||
.safeAreaPadding(.top, 58)
|
||||
.transition(.move(edge: .top).combined(with: .opacity))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var statusActivity: StatusPill.Activity? {
|
||||
// Status pill owns transient activity state so it doesn't overlap the connection indicator.
|
||||
if self.appModel.isBackgrounded {
|
||||
return StatusPill.Activity(
|
||||
title: "Foreground required",
|
||||
systemImage: "exclamationmark.triangle.fill",
|
||||
tint: .orange)
|
||||
}
|
||||
|
||||
let gatewayStatus = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let gatewayLower = gatewayStatus.lowercased()
|
||||
if gatewayLower.contains("repair") {
|
||||
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
|
||||
}
|
||||
if gatewayLower.contains("approval") || gatewayLower.contains("pairing") {
|
||||
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
|
||||
}
|
||||
// Avoid duplicating the primary gateway status ("Connecting…") in the activity slot.
|
||||
|
||||
if self.appModel.screenRecordActive {
|
||||
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
|
||||
}
|
||||
|
||||
if let cameraHUDText, !cameraHUDText.isEmpty, let cameraHUDKind {
|
||||
let systemImage: String
|
||||
let tint: Color?
|
||||
switch cameraHUDKind {
|
||||
case .photo:
|
||||
systemImage = "camera.fill"
|
||||
tint = nil
|
||||
case .recording:
|
||||
systemImage = "video.fill"
|
||||
tint = .red
|
||||
case .success:
|
||||
systemImage = "checkmark.circle.fill"
|
||||
tint = .green
|
||||
case .error:
|
||||
systemImage = "exclamationmark.triangle.fill"
|
||||
tint = .red
|
||||
}
|
||||
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
|
||||
}
|
||||
|
||||
if self.voiceWakeEnabled {
|
||||
let voiceStatus = self.appModel.voiceWake.statusText
|
||||
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
|
||||
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
|
||||
}
|
||||
if voiceStatus == "Paused" {
|
||||
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
|
||||
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
private struct OverlayButton: View {
|
||||
let systemImage: String
|
||||
let brighten: Bool
|
||||
var tint: Color?
|
||||
var isActive: Bool = false
|
||||
let action: () -> Void
|
||||
|
||||
var body: some View {
|
||||
Button(action: self.action) {
|
||||
Image(systemName: self.systemImage)
|
||||
.font(.system(size: 16, weight: .semibold))
|
||||
.foregroundStyle(self.isActive ? (self.tint ?? .primary) : .primary)
|
||||
.padding(10)
|
||||
.background {
|
||||
RoundedRectangle(cornerRadius: 12, style: .continuous)
|
||||
.fill(.ultraThinMaterial)
|
||||
.overlay {
|
||||
RoundedRectangle(cornerRadius: 12, style: .continuous)
|
||||
.fill(
|
||||
LinearGradient(
|
||||
colors: [
|
||||
.white.opacity(self.brighten ? 0.26 : 0.18),
|
||||
.white.opacity(self.brighten ? 0.08 : 0.04),
|
||||
.clear,
|
||||
],
|
||||
startPoint: .topLeading,
|
||||
endPoint: .bottomTrailing))
|
||||
.blendMode(.overlay)
|
||||
}
|
||||
.overlay {
|
||||
if let tint {
|
||||
RoundedRectangle(cornerRadius: 12, style: .continuous)
|
||||
.fill(
|
||||
LinearGradient(
|
||||
colors: [
|
||||
tint.opacity(self.isActive ? 0.22 : 0.14),
|
||||
tint.opacity(self.isActive ? 0.10 : 0.06),
|
||||
.clear,
|
||||
],
|
||||
startPoint: .topLeading,
|
||||
endPoint: .bottomTrailing))
|
||||
.blendMode(.overlay)
|
||||
}
|
||||
}
|
||||
.overlay {
|
||||
RoundedRectangle(cornerRadius: 12, style: .continuous)
|
||||
.strokeBorder(
|
||||
(self.tint ?? .white).opacity(self.isActive ? 0.34 : (self.brighten ? 0.24 : 0.18)),
|
||||
lineWidth: self.isActive ? 0.7 : 0.5)
|
||||
}
|
||||
.shadow(color: .black.opacity(0.35), radius: 12, y: 6)
|
||||
}
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
}
|
||||
|
||||
private struct CameraFlashOverlay: View {
|
||||
var nonce: Int
|
||||
|
||||
@State private var opacity: CGFloat = 0
|
||||
@State private var task: Task<Void, Never>?
|
||||
|
||||
var body: some View {
|
||||
Color.white
|
||||
.opacity(self.opacity)
|
||||
.ignoresSafeArea()
|
||||
.allowsHitTesting(false)
|
||||
.onChange(of: self.nonce) { _, _ in
|
||||
self.task?.cancel()
|
||||
self.task = Task { @MainActor in
|
||||
withAnimation(.easeOut(duration: 0.08)) {
|
||||
self.opacity = 0.85
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 110_000_000)
|
||||
withAnimation(.easeOut(duration: 0.32)) {
|
||||
self.opacity = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
143
docker-compose/ez-assistant/apps/ios/Sources/RootTabs.swift
Normal file
@@ -0,0 +1,143 @@
|
||||
import SwiftUI
|
||||
|
||||
struct RootTabs: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
@Environment(VoiceWakeManager.self) private var voiceWake
|
||||
@AppStorage(VoiceWakePreferences.enabledKey) private var voiceWakeEnabled: Bool = false
|
||||
@State private var selectedTab: Int = 0
|
||||
@State private var voiceWakeToastText: String?
|
||||
@State private var toastDismissTask: Task<Void, Never>?
|
||||
|
||||
var body: some View {
|
||||
TabView(selection: self.$selectedTab) {
|
||||
ScreenTab()
|
||||
.tabItem { Label("Screen", systemImage: "rectangle.and.hand.point.up.left") }
|
||||
.tag(0)
|
||||
|
||||
VoiceTab()
|
||||
.tabItem { Label("Voice", systemImage: "mic") }
|
||||
.tag(1)
|
||||
|
||||
SettingsTab()
|
||||
.tabItem { Label("Settings", systemImage: "gearshape") }
|
||||
.tag(2)
|
||||
}
|
||||
.overlay(alignment: .topLeading) {
|
||||
StatusPill(
|
||||
gateway: self.gatewayStatus,
|
||||
voiceWakeEnabled: self.voiceWakeEnabled,
|
||||
activity: self.statusActivity,
|
||||
onTap: { self.selectedTab = 2 })
|
||||
.padding(.leading, 10)
|
||||
.safeAreaPadding(.top, 10)
|
||||
}
|
||||
.overlay(alignment: .topLeading) {
|
||||
if let voiceWakeToastText, !voiceWakeToastText.isEmpty {
|
||||
VoiceWakeToast(command: voiceWakeToastText)
|
||||
.padding(.leading, 10)
|
||||
.safeAreaPadding(.top, 58)
|
||||
.transition(.move(edge: .top).combined(with: .opacity))
|
||||
}
|
||||
}
|
||||
.onChange(of: self.voiceWake.lastTriggeredCommand) { _, newValue in
|
||||
guard let newValue else { return }
|
||||
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
|
||||
self.toastDismissTask?.cancel()
|
||||
withAnimation(.spring(response: 0.25, dampingFraction: 0.85)) {
|
||||
self.voiceWakeToastText = trimmed
|
||||
}
|
||||
|
||||
self.toastDismissTask = Task {
|
||||
try? await Task.sleep(nanoseconds: 2_300_000_000)
|
||||
await MainActor.run {
|
||||
withAnimation(.easeOut(duration: 0.25)) {
|
||||
self.voiceWakeToastText = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.onDisappear {
|
||||
self.toastDismissTask?.cancel()
|
||||
self.toastDismissTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
private var gatewayStatus: StatusPill.GatewayState {
|
||||
if self.appModel.gatewayServerName != nil { return .connected }
|
||||
|
||||
let text = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if text.localizedCaseInsensitiveContains("connecting") ||
|
||||
text.localizedCaseInsensitiveContains("reconnecting")
|
||||
{
|
||||
return .connecting
|
||||
}
|
||||
|
||||
if text.localizedCaseInsensitiveContains("error") {
|
||||
return .error
|
||||
}
|
||||
|
||||
return .disconnected
|
||||
}
|
||||
|
||||
private var statusActivity: StatusPill.Activity? {
|
||||
// Keep the top pill consistent across tabs (camera + voice wake + pairing states).
|
||||
if self.appModel.isBackgrounded {
|
||||
return StatusPill.Activity(
|
||||
title: "Foreground required",
|
||||
systemImage: "exclamationmark.triangle.fill",
|
||||
tint: .orange)
|
||||
}
|
||||
|
||||
let gatewayStatus = self.appModel.gatewayStatusText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let gatewayLower = gatewayStatus.lowercased()
|
||||
if gatewayLower.contains("repair") {
|
||||
return StatusPill.Activity(title: "Repairing…", systemImage: "wrench.and.screwdriver", tint: .orange)
|
||||
}
|
||||
if gatewayLower.contains("approval") || gatewayLower.contains("pairing") {
|
||||
return StatusPill.Activity(title: "Approval pending", systemImage: "person.crop.circle.badge.clock")
|
||||
}
|
||||
// Avoid duplicating the primary gateway status ("Connecting…") in the activity slot.
|
||||
|
||||
if self.appModel.screenRecordActive {
|
||||
return StatusPill.Activity(title: "Recording screen…", systemImage: "record.circle.fill", tint: .red)
|
||||
}
|
||||
|
||||
if let cameraHUDText = self.appModel.cameraHUDText,
|
||||
let cameraHUDKind = self.appModel.cameraHUDKind,
|
||||
!cameraHUDText.isEmpty
|
||||
{
|
||||
let systemImage: String
|
||||
let tint: Color?
|
||||
switch cameraHUDKind {
|
||||
case .photo:
|
||||
systemImage = "camera.fill"
|
||||
tint = nil
|
||||
case .recording:
|
||||
systemImage = "video.fill"
|
||||
tint = .red
|
||||
case .success:
|
||||
systemImage = "checkmark.circle.fill"
|
||||
tint = .green
|
||||
case .error:
|
||||
systemImage = "exclamationmark.triangle.fill"
|
||||
tint = .red
|
||||
}
|
||||
return StatusPill.Activity(title: cameraHUDText, systemImage: systemImage, tint: tint)
|
||||
}
|
||||
|
||||
if self.voiceWakeEnabled {
|
||||
let voiceStatus = self.appModel.voiceWake.statusText
|
||||
if voiceStatus.localizedCaseInsensitiveContains("microphone permission") {
|
||||
return StatusPill.Activity(title: "Mic permission", systemImage: "mic.slash", tint: .orange)
|
||||
}
|
||||
if voiceStatus == "Paused" {
|
||||
let suffix = self.appModel.isBackgrounded ? " (background)" : ""
|
||||
return StatusPill.Activity(title: "Voice Wake paused\(suffix)", systemImage: "pause.circle.fill")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
import MoltbotKit
|
||||
import Observation
|
||||
import SwiftUI
|
||||
import WebKit
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class ScreenController {
|
||||
let webView: WKWebView
|
||||
private let navigationDelegate: ScreenNavigationDelegate
|
||||
private let a2uiActionHandler: CanvasA2UIActionMessageHandler
|
||||
|
||||
var urlString: String = ""
|
||||
var errorText: String?
|
||||
|
||||
/// Callback invoked when a moltbot:// deep link is tapped in the canvas
|
||||
var onDeepLink: ((URL) -> Void)?
|
||||
|
||||
/// Callback invoked when the user clicks an A2UI action (e.g. button) inside the canvas web UI.
|
||||
var onA2UIAction: (([String: Any]) -> Void)?
|
||||
|
||||
private var debugStatusEnabled: Bool = false
|
||||
private var debugStatusTitle: String?
|
||||
private var debugStatusSubtitle: String?
|
||||
|
||||
init() {
|
||||
let config = WKWebViewConfiguration()
|
||||
config.websiteDataStore = .nonPersistent()
|
||||
let a2uiActionHandler = CanvasA2UIActionMessageHandler()
|
||||
let userContentController = WKUserContentController()
|
||||
for name in CanvasA2UIActionMessageHandler.handlerNames {
|
||||
userContentController.add(a2uiActionHandler, name: name)
|
||||
}
|
||||
config.userContentController = userContentController
|
||||
self.navigationDelegate = ScreenNavigationDelegate()
|
||||
self.a2uiActionHandler = a2uiActionHandler
|
||||
self.webView = WKWebView(frame: .zero, configuration: config)
|
||||
// Canvas scaffold is a fully self-contained HTML page; avoid relying on transparency underlays.
|
||||
self.webView.isOpaque = true
|
||||
self.webView.backgroundColor = .black
|
||||
self.webView.scrollView.backgroundColor = .black
|
||||
self.webView.scrollView.contentInsetAdjustmentBehavior = .never
|
||||
self.webView.scrollView.contentInset = .zero
|
||||
self.webView.scrollView.scrollIndicatorInsets = .zero
|
||||
self.webView.scrollView.automaticallyAdjustsScrollIndicatorInsets = false
|
||||
self.applyScrollBehavior()
|
||||
self.webView.navigationDelegate = self.navigationDelegate
|
||||
self.navigationDelegate.controller = self
|
||||
a2uiActionHandler.controller = self
|
||||
self.reload()
|
||||
}
|
||||
|
||||
func navigate(to urlString: String) {
|
||||
let trimmed = urlString.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.urlString = (trimmed == "/" ? "" : trimmed)
|
||||
self.reload()
|
||||
}
|
||||
|
||||
func reload() {
|
||||
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.applyScrollBehavior()
|
||||
if trimmed.isEmpty {
|
||||
guard let url = Self.canvasScaffoldURL else { return }
|
||||
self.errorText = nil
|
||||
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
|
||||
return
|
||||
} else {
|
||||
guard let url = URL(string: trimmed) else {
|
||||
self.errorText = "Invalid URL: \(trimmed)"
|
||||
return
|
||||
}
|
||||
self.errorText = nil
|
||||
if url.isFileURL {
|
||||
self.webView.loadFileURL(url, allowingReadAccessTo: url.deletingLastPathComponent())
|
||||
} else {
|
||||
self.webView.load(URLRequest(url: url))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func showDefaultCanvas() {
|
||||
self.urlString = ""
|
||||
self.reload()
|
||||
}
|
||||
|
||||
func setDebugStatusEnabled(_ enabled: Bool) {
|
||||
self.debugStatusEnabled = enabled
|
||||
self.applyDebugStatusIfNeeded()
|
||||
}
|
||||
|
||||
func updateDebugStatus(title: String?, subtitle: String?) {
|
||||
self.debugStatusTitle = title
|
||||
self.debugStatusSubtitle = subtitle
|
||||
self.applyDebugStatusIfNeeded()
|
||||
}
|
||||
|
||||
fileprivate func applyDebugStatusIfNeeded() {
|
||||
let enabled = self.debugStatusEnabled
|
||||
let title = self.debugStatusTitle
|
||||
let subtitle = self.debugStatusSubtitle
|
||||
let js = """
|
||||
(() => {
|
||||
try {
|
||||
const api = globalThis.__moltbot;
|
||||
if (!api) return;
|
||||
if (typeof api.setDebugStatusEnabled === 'function') {
|
||||
api.setDebugStatusEnabled(\(enabled ? "true" : "false"));
|
||||
}
|
||||
if (!\(enabled ? "true" : "false")) return;
|
||||
if (typeof api.setStatus === 'function') {
|
||||
api.setStatus(\(Self.jsValue(title)), \(Self.jsValue(subtitle)));
|
||||
}
|
||||
} catch (_) {}
|
||||
})()
|
||||
"""
|
||||
self.webView.evaluateJavaScript(js) { _, _ in }
|
||||
}
|
||||
|
||||
func waitForA2UIReady(timeoutMs: Int) async -> Bool {
|
||||
let clock = ContinuousClock()
|
||||
let deadline = clock.now.advanced(by: .milliseconds(timeoutMs))
|
||||
while clock.now < deadline {
|
||||
do {
|
||||
let res = try await self.eval(javaScript: """
|
||||
(() => {
|
||||
try {
|
||||
return !!globalThis.clawdbotA2UI && typeof globalThis.clawdbotA2UI.applyMessages === 'function';
|
||||
} catch (_) { return false; }
|
||||
})()
|
||||
""")
|
||||
let trimmed = res.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
if trimmed == "true" || trimmed == "1" { return true }
|
||||
} catch {
|
||||
// ignore; page likely still loading
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 120_000_000)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func eval(javaScript: String) async throws -> String {
|
||||
try await withCheckedThrowingContinuation { cont in
|
||||
self.webView.evaluateJavaScript(javaScript) { result, error in
|
||||
if let error {
|
||||
cont.resume(throwing: error)
|
||||
return
|
||||
}
|
||||
if let result {
|
||||
cont.resume(returning: String(describing: result))
|
||||
} else {
|
||||
cont.resume(returning: "")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func snapshotPNGBase64(maxWidth: CGFloat? = nil) async throws -> String {
|
||||
let config = WKSnapshotConfiguration()
|
||||
if let maxWidth {
|
||||
config.snapshotWidth = NSNumber(value: Double(maxWidth))
|
||||
}
|
||||
let image: UIImage = try await withCheckedThrowingContinuation { cont in
|
||||
self.webView.takeSnapshot(with: config) { image, error in
|
||||
if let error {
|
||||
cont.resume(throwing: error)
|
||||
return
|
||||
}
|
||||
guard let image else {
|
||||
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
|
||||
NSLocalizedDescriptionKey: "snapshot failed",
|
||||
]))
|
||||
return
|
||||
}
|
||||
cont.resume(returning: image)
|
||||
}
|
||||
}
|
||||
guard let data = image.pngData() else {
|
||||
throw NSError(domain: "Screen", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "snapshot encode failed",
|
||||
])
|
||||
}
|
||||
return data.base64EncodedString()
|
||||
}
|
||||
|
||||
func snapshotBase64(
|
||||
maxWidth: CGFloat? = nil,
|
||||
format: MoltbotCanvasSnapshotFormat,
|
||||
quality: Double? = nil) async throws -> String
|
||||
{
|
||||
let config = WKSnapshotConfiguration()
|
||||
if let maxWidth {
|
||||
config.snapshotWidth = NSNumber(value: Double(maxWidth))
|
||||
}
|
||||
let image: UIImage = try await withCheckedThrowingContinuation { cont in
|
||||
self.webView.takeSnapshot(with: config) { image, error in
|
||||
if let error {
|
||||
cont.resume(throwing: error)
|
||||
return
|
||||
}
|
||||
guard let image else {
|
||||
cont.resume(throwing: NSError(domain: "Screen", code: 2, userInfo: [
|
||||
NSLocalizedDescriptionKey: "snapshot failed",
|
||||
]))
|
||||
return
|
||||
}
|
||||
cont.resume(returning: image)
|
||||
}
|
||||
}
|
||||
|
||||
let data: Data?
|
||||
switch format {
|
||||
case .png:
|
||||
data = image.pngData()
|
||||
case .jpeg:
|
||||
let q = (quality ?? 0.82).clamped(to: 0.1...1.0)
|
||||
data = image.jpegData(compressionQuality: q)
|
||||
}
|
||||
guard let data else {
|
||||
throw NSError(domain: "Screen", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "snapshot encode failed",
|
||||
])
|
||||
}
|
||||
return data.base64EncodedString()
|
||||
}
|
||||
|
||||
private static func bundledResourceURL(
|
||||
name: String,
|
||||
ext: String,
|
||||
subdirectory: String)
|
||||
-> URL?
|
||||
{
|
||||
let bundle = MoltbotKitResources.bundle
|
||||
return bundle.url(forResource: name, withExtension: ext, subdirectory: subdirectory)
|
||||
?? bundle.url(forResource: name, withExtension: ext)
|
||||
}
|
||||
|
||||
private static let canvasScaffoldURL: URL? = ScreenController.bundledResourceURL(
|
||||
name: "scaffold",
|
||||
ext: "html",
|
||||
subdirectory: "CanvasScaffold")
|
||||
func isTrustedCanvasUIURL(_ url: URL) -> Bool {
|
||||
guard url.isFileURL else { return false }
|
||||
let std = url.standardizedFileURL
|
||||
if let expected = Self.canvasScaffoldURL,
|
||||
std == expected.standardizedFileURL
|
||||
{
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private func applyScrollBehavior() {
|
||||
let trimmed = self.urlString.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let allowScroll = !trimmed.isEmpty
|
||||
let scrollView = self.webView.scrollView
|
||||
// Default canvas needs raw touch events; external pages should scroll.
|
||||
scrollView.isScrollEnabled = allowScroll
|
||||
scrollView.bounces = allowScroll
|
||||
}
|
||||
|
||||
private static func jsValue(_ value: String?) -> String {
|
||||
guard let value else { return "null" }
|
||||
if let data = try? JSONSerialization.data(withJSONObject: [value]),
|
||||
let encoded = String(data: data, encoding: .utf8),
|
||||
encoded.count >= 2
|
||||
{
|
||||
return String(encoded.dropFirst().dropLast())
|
||||
}
|
||||
return "null"
|
||||
}
|
||||
|
||||
func isLocalNetworkCanvasURL(_ url: URL) -> Bool {
|
||||
guard let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https" else {
|
||||
return false
|
||||
}
|
||||
guard let host = url.host?.trimmingCharacters(in: .whitespacesAndNewlines), !host.isEmpty else {
|
||||
return false
|
||||
}
|
||||
if host == "localhost" { return true }
|
||||
if host.hasSuffix(".local") { return true }
|
||||
if host.hasSuffix(".ts.net") { return true }
|
||||
if host.hasSuffix(".tailscale.net") { return true }
|
||||
// Allow MagicDNS / LAN hostnames like "peters-mac-studio-1".
|
||||
if !host.contains("."), !host.contains(":") { return true }
|
||||
if let ipv4 = Self.parseIPv4(host) {
|
||||
return Self.isLocalNetworkIPv4(ipv4)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private static func parseIPv4(_ host: String) -> (UInt8, UInt8, UInt8, UInt8)? {
|
||||
let parts = host.split(separator: ".", omittingEmptySubsequences: false)
|
||||
guard parts.count == 4 else { return nil }
|
||||
let bytes: [UInt8] = parts.compactMap { UInt8($0) }
|
||||
guard bytes.count == 4 else { return nil }
|
||||
return (bytes[0], bytes[1], bytes[2], bytes[3])
|
||||
}
|
||||
|
||||
private static func isLocalNetworkIPv4(_ ip: (UInt8, UInt8, UInt8, UInt8)) -> Bool {
|
||||
let (a, b, _, _) = ip
|
||||
// 10.0.0.0/8
|
||||
if a == 10 { return true }
|
||||
// 172.16.0.0/12
|
||||
if a == 172, (16...31).contains(Int(b)) { return true }
|
||||
// 192.168.0.0/16
|
||||
if a == 192, b == 168 { return true }
|
||||
// 127.0.0.0/8
|
||||
if a == 127 { return true }
|
||||
// 169.254.0.0/16 (link-local)
|
||||
if a == 169, b == 254 { return true }
|
||||
// Tailscale: 100.64.0.0/10
|
||||
if a == 100, (64...127).contains(Int(b)) { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
nonisolated static func parseA2UIActionBody(_ body: Any) -> [String: Any]? {
|
||||
if let dict = body as? [String: Any] { return dict.isEmpty ? nil : dict }
|
||||
if let str = body as? String,
|
||||
let data = str.data(using: .utf8),
|
||||
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
|
||||
{
|
||||
return json.isEmpty ? nil : json
|
||||
}
|
||||
if let dict = body as? [AnyHashable: Any] {
|
||||
let mapped = dict.reduce(into: [String: Any]()) { acc, pair in
|
||||
guard let key = pair.key as? String else { return }
|
||||
acc[key] = pair.value
|
||||
}
|
||||
return mapped.isEmpty ? nil : mapped
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
extension Double {
|
||||
fileprivate func clamped(to range: ClosedRange<Double>) -> Double {
|
||||
if self < range.lowerBound { return range.lowerBound }
|
||||
if self > range.upperBound { return range.upperBound }
|
||||
return self
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Navigation Delegate
|
||||
|
||||
/// Handles navigation policy to intercept moltbot:// deep links from canvas
|
||||
@MainActor
|
||||
private final class ScreenNavigationDelegate: NSObject, WKNavigationDelegate {
|
||||
weak var controller: ScreenController?
|
||||
|
||||
func webView(
|
||||
_ webView: WKWebView,
|
||||
decidePolicyFor navigationAction: WKNavigationAction,
|
||||
decisionHandler: @escaping @MainActor @Sendable (WKNavigationActionPolicy) -> Void)
|
||||
{
|
||||
guard let url = navigationAction.request.url else {
|
||||
decisionHandler(.allow)
|
||||
return
|
||||
}
|
||||
|
||||
// Intercept moltbot:// deep links
|
||||
if url.scheme == "moltbot" {
|
||||
decisionHandler(.cancel)
|
||||
self.controller?.onDeepLink?(url)
|
||||
return
|
||||
}
|
||||
|
||||
decisionHandler(.allow)
|
||||
}
|
||||
|
||||
func webView(
|
||||
_: WKWebView,
|
||||
didFailProvisionalNavigation _: WKNavigation?,
|
||||
withError error: any Error)
|
||||
{
|
||||
self.controller?.errorText = error.localizedDescription
|
||||
}
|
||||
|
||||
func webView(_: WKWebView, didFinish _: WKNavigation?) {
|
||||
self.controller?.errorText = nil
|
||||
self.controller?.applyDebugStatusIfNeeded()
|
||||
}
|
||||
|
||||
func webView(_: WKWebView, didFail _: WKNavigation?, withError error: any Error) {
|
||||
self.controller?.errorText = error.localizedDescription
|
||||
}
|
||||
}
|
||||
|
||||
private final class CanvasA2UIActionMessageHandler: NSObject, WKScriptMessageHandler {
|
||||
static let messageName = "moltbotCanvasA2UIAction"
|
||||
static let legacyMessageNames = ["canvas", "a2ui", "userAction", "action"]
|
||||
static let handlerNames = [messageName] + legacyMessageNames
|
||||
|
||||
weak var controller: ScreenController?
|
||||
|
||||
func userContentController(_: WKUserContentController, didReceive message: WKScriptMessage) {
|
||||
guard message.name == Self.messageName else { return }
|
||||
guard let controller else { return }
|
||||
|
||||
guard let url = message.webView?.url else { return }
|
||||
if url.isFileURL {
|
||||
guard controller.isTrustedCanvasUIURL(url) else { return }
|
||||
} else {
|
||||
// For security, only accept actions from local-network pages (e.g. the canvas host).
|
||||
guard controller.isLocalNetworkCanvasURL(url) else { return }
|
||||
}
|
||||
|
||||
guard let body = ScreenController.parseA2UIActionBody(message.body) else { return }
|
||||
|
||||
controller.onA2UIAction?(body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
import AVFoundation
|
||||
import ReplayKit
|
||||
|
||||
final class ScreenRecordService: @unchecked Sendable {
|
||||
private struct UncheckedSendableBox<T>: @unchecked Sendable {
|
||||
let value: T
|
||||
}
|
||||
|
||||
private final class CaptureState: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
var writer: AVAssetWriter?
|
||||
var videoInput: AVAssetWriterInput?
|
||||
var audioInput: AVAssetWriterInput?
|
||||
var started = false
|
||||
var sawVideo = false
|
||||
var lastVideoTime: CMTime?
|
||||
var handlerError: Error?
|
||||
|
||||
func withLock<T>(_ body: (CaptureState) -> T) -> T {
|
||||
self.lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return body(self)
|
||||
}
|
||||
}
|
||||
|
||||
enum ScreenRecordError: LocalizedError {
|
||||
case invalidScreenIndex(Int)
|
||||
case captureFailed(String)
|
||||
case writeFailed(String)
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case let .invalidScreenIndex(idx):
|
||||
"Invalid screen index \(idx)"
|
||||
case let .captureFailed(msg):
|
||||
msg
|
||||
case let .writeFailed(msg):
|
||||
msg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func record(
|
||||
screenIndex: Int?,
|
||||
durationMs: Int?,
|
||||
fps: Double?,
|
||||
includeAudio: Bool?,
|
||||
outPath: String?) async throws -> String
|
||||
{
|
||||
let config = try self.makeRecordConfig(
|
||||
screenIndex: screenIndex,
|
||||
durationMs: durationMs,
|
||||
fps: fps,
|
||||
includeAudio: includeAudio,
|
||||
outPath: outPath)
|
||||
|
||||
let state = CaptureState()
|
||||
let recordQueue = DispatchQueue(label: "bot.molt.screenrecord")
|
||||
|
||||
try await self.startCapture(state: state, config: config, recordQueue: recordQueue)
|
||||
try await Task.sleep(nanoseconds: UInt64(config.durationMs) * 1_000_000)
|
||||
try await self.stopCapture()
|
||||
try self.finalizeCapture(state: state)
|
||||
try await self.finishWriting(state: state)
|
||||
|
||||
return config.outURL.path
|
||||
}
|
||||
|
||||
private struct RecordConfig {
|
||||
let durationMs: Int
|
||||
let fpsValue: Double
|
||||
let includeAudio: Bool
|
||||
let outURL: URL
|
||||
}
|
||||
|
||||
private func makeRecordConfig(
|
||||
screenIndex: Int?,
|
||||
durationMs: Int?,
|
||||
fps: Double?,
|
||||
includeAudio: Bool?,
|
||||
outPath: String?) throws -> RecordConfig
|
||||
{
|
||||
if let idx = screenIndex, idx != 0 {
|
||||
throw ScreenRecordError.invalidScreenIndex(idx)
|
||||
}
|
||||
|
||||
let durationMs = Self.clampDurationMs(durationMs)
|
||||
let fps = Self.clampFps(fps)
|
||||
let fpsInt = Int32(fps.rounded())
|
||||
let fpsValue = Double(fpsInt)
|
||||
let includeAudio = includeAudio ?? true
|
||||
|
||||
let outURL = self.makeOutputURL(outPath: outPath)
|
||||
try? FileManager().removeItem(at: outURL)
|
||||
|
||||
return RecordConfig(
|
||||
durationMs: durationMs,
|
||||
fpsValue: fpsValue,
|
||||
includeAudio: includeAudio,
|
||||
outURL: outURL)
|
||||
}
|
||||
|
||||
private func makeOutputURL(outPath: String?) -> URL {
|
||||
if let outPath, !outPath.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
|
||||
return URL(fileURLWithPath: outPath)
|
||||
}
|
||||
return FileManager().temporaryDirectory
|
||||
.appendingPathComponent("moltbot-screen-record-\(UUID().uuidString).mp4")
|
||||
}
|
||||
|
||||
private func startCapture(
|
||||
state: CaptureState,
|
||||
config: RecordConfig,
|
||||
recordQueue: DispatchQueue) async throws
|
||||
{
|
||||
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||
let handler = self.makeCaptureHandler(
|
||||
state: state,
|
||||
config: config,
|
||||
recordQueue: recordQueue)
|
||||
let completion: @Sendable (Error?) -> Void = { error in
|
||||
if let error { cont.resume(throwing: error) } else { cont.resume() }
|
||||
}
|
||||
|
||||
Task { @MainActor in
|
||||
startReplayKitCapture(
|
||||
includeAudio: config.includeAudio,
|
||||
handler: handler,
|
||||
completion: completion)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func makeCaptureHandler(
|
||||
state: CaptureState,
|
||||
config: RecordConfig,
|
||||
recordQueue: DispatchQueue) -> @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void
|
||||
{
|
||||
{ sample, type, error in
|
||||
let sampleBox = UncheckedSendableBox(value: sample)
|
||||
// ReplayKit can call the capture handler on a background queue.
|
||||
// Serialize writes to avoid queue asserts.
|
||||
recordQueue.async {
|
||||
let sample = sampleBox.value
|
||||
if let error {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil { state.handlerError = error }
|
||||
}
|
||||
return
|
||||
}
|
||||
guard CMSampleBufferDataIsReady(sample) else { return }
|
||||
|
||||
switch type {
|
||||
case .video:
|
||||
self.handleVideoSample(sample, state: state, config: config)
|
||||
case .audioApp, .audioMic:
|
||||
self.handleAudioSample(sample, state: state, includeAudio: config.includeAudio)
|
||||
@unknown default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleVideoSample(
|
||||
_ sample: CMSampleBuffer,
|
||||
state: CaptureState,
|
||||
config: RecordConfig)
|
||||
{
|
||||
let pts = CMSampleBufferGetPresentationTimeStamp(sample)
|
||||
let shouldSkip = state.withLock { state in
|
||||
if let lastVideoTime = state.lastVideoTime {
|
||||
let delta = CMTimeSubtract(pts, lastVideoTime)
|
||||
return delta.seconds < (1.0 / config.fpsValue)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if shouldSkip { return }
|
||||
|
||||
if state.withLock({ $0.writer == nil }) {
|
||||
self.prepareWriter(sample: sample, state: state, config: config, pts: pts)
|
||||
}
|
||||
|
||||
let vInput = state.withLock { $0.videoInput }
|
||||
let isStarted = state.withLock { $0.started }
|
||||
guard let vInput, isStarted else { return }
|
||||
if vInput.isReadyForMoreMediaData {
|
||||
if vInput.append(sample) {
|
||||
state.withLock { state in
|
||||
state.sawVideo = true
|
||||
state.lastVideoTime = pts
|
||||
}
|
||||
} else {
|
||||
let err = state.withLock { $0.writer?.error }
|
||||
if let err {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil {
|
||||
state.handlerError = ScreenRecordError.writeFailed(err.localizedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func prepareWriter(
|
||||
sample: CMSampleBuffer,
|
||||
state: CaptureState,
|
||||
config: RecordConfig,
|
||||
pts: CMTime)
|
||||
{
|
||||
guard let imageBuffer = CMSampleBufferGetImageBuffer(sample) else {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil {
|
||||
state.handlerError = ScreenRecordError.captureFailed("Missing image buffer")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
let width = CVPixelBufferGetWidth(imageBuffer)
|
||||
let height = CVPixelBufferGetHeight(imageBuffer)
|
||||
do {
|
||||
let writer = try AVAssetWriter(outputURL: config.outURL, fileType: .mp4)
|
||||
let settings: [String: Any] = [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: width,
|
||||
AVVideoHeightKey: height,
|
||||
]
|
||||
let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
|
||||
vInput.expectsMediaDataInRealTime = true
|
||||
guard writer.canAdd(vInput) else {
|
||||
throw ScreenRecordError.writeFailed("Cannot add video input")
|
||||
}
|
||||
writer.add(vInput)
|
||||
|
||||
if config.includeAudio {
|
||||
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: nil)
|
||||
aInput.expectsMediaDataInRealTime = true
|
||||
if writer.canAdd(aInput) {
|
||||
writer.add(aInput)
|
||||
state.withLock { state in
|
||||
state.audioInput = aInput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
guard writer.startWriting() else {
|
||||
throw ScreenRecordError.writeFailed(
|
||||
writer.error?.localizedDescription ?? "Failed to start writer")
|
||||
}
|
||||
writer.startSession(atSourceTime: pts)
|
||||
state.withLock { state in
|
||||
state.writer = writer
|
||||
state.videoInput = vInput
|
||||
state.started = true
|
||||
}
|
||||
} catch {
|
||||
state.withLock { state in
|
||||
if state.handlerError == nil { state.handlerError = error }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleAudioSample(
|
||||
_ sample: CMSampleBuffer,
|
||||
state: CaptureState,
|
||||
includeAudio: Bool)
|
||||
{
|
||||
let aInput = state.withLock { $0.audioInput }
|
||||
let isStarted = state.withLock { $0.started }
|
||||
guard includeAudio, let aInput, isStarted else { return }
|
||||
if aInput.isReadyForMoreMediaData {
|
||||
_ = aInput.append(sample)
|
||||
}
|
||||
}
|
||||
|
||||
private func stopCapture() async throws {
|
||||
let stopError = await withCheckedContinuation { cont in
|
||||
Task { @MainActor in
|
||||
stopReplayKitCapture { error in cont.resume(returning: error) }
|
||||
}
|
||||
}
|
||||
if let stopError { throw stopError }
|
||||
}
|
||||
|
||||
private func finalizeCapture(state: CaptureState) throws {
|
||||
if let handlerErrorSnapshot = state.withLock({ $0.handlerError }) {
|
||||
throw handlerErrorSnapshot
|
||||
}
|
||||
let writerSnapshot = state.withLock { $0.writer }
|
||||
let videoInputSnapshot = state.withLock { $0.videoInput }
|
||||
let audioInputSnapshot = state.withLock { $0.audioInput }
|
||||
let sawVideoSnapshot = state.withLock { $0.sawVideo }
|
||||
guard let writerSnapshot, let videoInputSnapshot, sawVideoSnapshot else {
|
||||
throw ScreenRecordError.captureFailed("No frames captured")
|
||||
}
|
||||
|
||||
videoInputSnapshot.markAsFinished()
|
||||
audioInputSnapshot?.markAsFinished()
|
||||
_ = writerSnapshot
|
||||
}
|
||||
|
||||
private func finishWriting(state: CaptureState) async throws {
|
||||
guard let writerSnapshot = state.withLock({ $0.writer }) else {
|
||||
throw ScreenRecordError.captureFailed("Missing writer")
|
||||
}
|
||||
let writerBox = UncheckedSendableBox(value: writerSnapshot)
|
||||
try await withCheckedThrowingContinuation { (cont: CheckedContinuation<Void, Error>) in
|
||||
writerBox.value.finishWriting {
|
||||
let writer = writerBox.value
|
||||
if let err = writer.error {
|
||||
cont.resume(throwing: ScreenRecordError.writeFailed(err.localizedDescription))
|
||||
} else if writer.status != .completed {
|
||||
cont.resume(throwing: ScreenRecordError.writeFailed("Failed to finalize video"))
|
||||
} else {
|
||||
cont.resume()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func clampDurationMs(_ ms: Int?) -> Int {
|
||||
let v = ms ?? 10000
|
||||
return min(60000, max(250, v))
|
||||
}
|
||||
|
||||
private nonisolated static func clampFps(_ fps: Double?) -> Double {
|
||||
let v = fps ?? 10
|
||||
if !v.isFinite { return 10 }
|
||||
return min(30, max(1, v))
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func startReplayKitCapture(
|
||||
includeAudio: Bool,
|
||||
handler: @escaping @Sendable (CMSampleBuffer, RPSampleBufferType, Error?) -> Void,
|
||||
completion: @escaping @Sendable (Error?) -> Void)
|
||||
{
|
||||
let recorder = RPScreenRecorder.shared()
|
||||
recorder.isMicrophoneEnabled = includeAudio
|
||||
recorder.startCapture(handler: handler, completionHandler: completion)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func stopReplayKitCapture(_ completion: @escaping @Sendable (Error?) -> Void) {
|
||||
RPScreenRecorder.shared().stopCapture { error in completion(error) }
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
extension ScreenRecordService {
|
||||
nonisolated static func _test_clampDurationMs(_ ms: Int?) -> Int {
|
||||
self.clampDurationMs(ms)
|
||||
}
|
||||
|
||||
nonisolated static func _test_clampFps(_ fps: Double?) -> Double {
|
||||
self.clampFps(fps)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,25 @@
|
||||
import MoltbotKit
|
||||
import SwiftUI
|
||||
|
||||
struct ScreenTab: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .top) {
|
||||
ScreenWebView(controller: self.appModel.screen)
|
||||
.ignoresSafeArea()
|
||||
.overlay(alignment: .top) {
|
||||
if let errorText = self.appModel.screen.errorText {
|
||||
Text(errorText)
|
||||
.font(.footnote)
|
||||
.padding(10)
|
||||
.background(.thinMaterial)
|
||||
.clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous))
|
||||
.padding()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Navigation is agent-driven; no local URL bar here.
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import MoltbotKit
|
||||
import SwiftUI
|
||||
import WebKit
|
||||
|
||||
struct ScreenWebView: UIViewRepresentable {
|
||||
var controller: ScreenController
|
||||
|
||||
func makeUIView(context: Context) -> WKWebView {
|
||||
self.controller.webView
|
||||
}
|
||||
|
||||
func updateUIView(_ webView: WKWebView, context: Context) {
|
||||
// State changes are driven by ScreenController.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import Foundation
|
||||
|
||||
enum SessionKey {
|
||||
static func normalizeMainKey(_ raw: String?) -> String {
|
||||
let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return trimmed.isEmpty ? "main" : trimmed
|
||||
}
|
||||
|
||||
static func isCanonicalMainSessionKey(_ value: String?) -> Bool {
|
||||
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty { return false }
|
||||
if trimmed == "global" { return true }
|
||||
return trimmed.hasPrefix("agent:")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import Foundation
|
||||
|
||||
struct SettingsHostPort: Equatable {
|
||||
var host: String
|
||||
var port: Int
|
||||
}
|
||||
|
||||
enum SettingsNetworkingHelpers {
|
||||
static func parseHostPort(from address: String) -> SettingsHostPort? {
|
||||
let trimmed = address.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
|
||||
if trimmed.hasPrefix("["),
|
||||
let close = trimmed.firstIndex(of: "]"),
|
||||
close < trimmed.endIndex
|
||||
{
|
||||
let host = String(trimmed[trimmed.index(after: trimmed.startIndex)..<close])
|
||||
let portStart = trimmed.index(after: close)
|
||||
guard portStart < trimmed.endIndex, trimmed[portStart] == ":" else { return nil }
|
||||
let portString = String(trimmed[trimmed.index(after: portStart)...])
|
||||
guard let port = Int(portString) else { return nil }
|
||||
return SettingsHostPort(host: host, port: port)
|
||||
}
|
||||
|
||||
guard let colon = trimmed.lastIndex(of: ":") else { return nil }
|
||||
let host = String(trimmed[..<colon])
|
||||
let portString = String(trimmed[trimmed.index(after: colon)...])
|
||||
guard !host.isEmpty, let port = Int(portString) else { return nil }
|
||||
return SettingsHostPort(host: host, port: port)
|
||||
}
|
||||
|
||||
static func httpURLString(host: String?, port: Int?, fallback: String) -> String {
|
||||
if let host, let port {
|
||||
let needsBrackets = host.contains(":") && !host.hasPrefix("[") && !host.hasSuffix("]")
|
||||
let hostPart = needsBrackets ? "[\(host)]" : host
|
||||
return "http://\(hostPart):\(port)"
|
||||
}
|
||||
return "http://\(fallback)"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
import MoltbotKit
|
||||
import Network
|
||||
import Observation
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
private final class ConnectStatusStore {
|
||||
var text: String?
|
||||
}
|
||||
|
||||
extension ConnectStatusStore: @unchecked Sendable {}
|
||||
|
||||
struct SettingsTab: View {
|
||||
@Environment(NodeAppModel.self) private var appModel: NodeAppModel
|
||||
@Environment(VoiceWakeManager.self) private var voiceWake: VoiceWakeManager
|
||||
@Environment(GatewayConnectionController.self) private var gatewayController: GatewayConnectionController
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
@AppStorage("node.displayName") private var displayName: String = "iOS Node"
|
||||
@AppStorage("node.instanceId") private var instanceId: String = UUID().uuidString
|
||||
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
|
||||
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
|
||||
@AppStorage("talk.button.enabled") private var talkButtonEnabled: Bool = true
|
||||
@AppStorage("camera.enabled") private var cameraEnabled: Bool = true
|
||||
@AppStorage("location.enabledMode") private var locationEnabledModeRaw: String = MoltbotLocationMode.off.rawValue
|
||||
@AppStorage("location.preciseEnabled") private var locationPreciseEnabled: Bool = true
|
||||
@AppStorage("screen.preventSleep") private var preventSleep: Bool = true
|
||||
@AppStorage("gateway.preferredStableID") private var preferredGatewayStableID: String = ""
|
||||
@AppStorage("gateway.lastDiscoveredStableID") private var lastDiscoveredGatewayStableID: String = ""
|
||||
@AppStorage("gateway.manual.enabled") private var manualGatewayEnabled: Bool = false
|
||||
@AppStorage("gateway.manual.host") private var manualGatewayHost: String = ""
|
||||
@AppStorage("gateway.manual.port") private var manualGatewayPort: Int = 18789
|
||||
@AppStorage("gateway.manual.tls") private var manualGatewayTLS: Bool = true
|
||||
@AppStorage("gateway.discovery.debugLogs") private var discoveryDebugLogsEnabled: Bool = false
|
||||
@AppStorage("canvas.debugStatusEnabled") private var canvasDebugStatusEnabled: Bool = false
|
||||
@State private var connectStatus = ConnectStatusStore()
|
||||
@State private var connectingGatewayID: String?
|
||||
@State private var localIPAddress: String?
|
||||
@State private var lastLocationModeRaw: String = MoltbotLocationMode.off.rawValue
|
||||
@State private var gatewayToken: String = ""
|
||||
@State private var gatewayPassword: String = ""
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
Form {
|
||||
Section("Node") {
|
||||
TextField("Name", text: self.$displayName)
|
||||
Text(self.instanceId)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
LabeledContent("IP", value: self.localIPAddress ?? "—")
|
||||
.contextMenu {
|
||||
if let ip = self.localIPAddress {
|
||||
Button {
|
||||
UIPasteboard.general.string = ip
|
||||
} label: {
|
||||
Label("Copy", systemImage: "doc.on.doc")
|
||||
}
|
||||
}
|
||||
}
|
||||
LabeledContent("Platform", value: self.platformString())
|
||||
LabeledContent("Version", value: self.appVersion())
|
||||
LabeledContent("Model", value: self.modelIdentifier())
|
||||
}
|
||||
|
||||
Section("Gateway") {
|
||||
LabeledContent("Discovery", value: self.gatewayController.discoveryStatusText)
|
||||
LabeledContent("Status", value: self.appModel.gatewayStatusText)
|
||||
if let serverName = self.appModel.gatewayServerName {
|
||||
LabeledContent("Server", value: serverName)
|
||||
if let addr = self.appModel.gatewayRemoteAddress {
|
||||
let parts = Self.parseHostPort(from: addr)
|
||||
let urlString = Self.httpURLString(host: parts?.host, port: parts?.port, fallback: addr)
|
||||
LabeledContent("Address") {
|
||||
Text(urlString)
|
||||
}
|
||||
.contextMenu {
|
||||
Button {
|
||||
UIPasteboard.general.string = urlString
|
||||
} label: {
|
||||
Label("Copy URL", systemImage: "doc.on.doc")
|
||||
}
|
||||
|
||||
if let parts {
|
||||
Button {
|
||||
UIPasteboard.general.string = parts.host
|
||||
} label: {
|
||||
Label("Copy Host", systemImage: "doc.on.doc")
|
||||
}
|
||||
|
||||
Button {
|
||||
UIPasteboard.general.string = "\(parts.port)"
|
||||
} label: {
|
||||
Label("Copy Port", systemImage: "doc.on.doc")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Button("Disconnect", role: .destructive) {
|
||||
self.appModel.disconnectGateway()
|
||||
}
|
||||
|
||||
self.gatewayList(showing: .availableOnly)
|
||||
} else {
|
||||
self.gatewayList(showing: .all)
|
||||
}
|
||||
|
||||
if let text = self.connectStatus.text {
|
||||
Text(text)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
|
||||
DisclosureGroup("Advanced") {
|
||||
Toggle("Use Manual Gateway", isOn: self.$manualGatewayEnabled)
|
||||
|
||||
TextField("Host", text: self.$manualGatewayHost)
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
|
||||
TextField("Port", value: self.$manualGatewayPort, format: .number)
|
||||
.keyboardType(.numberPad)
|
||||
|
||||
Toggle("Use TLS", isOn: self.$manualGatewayTLS)
|
||||
|
||||
Button {
|
||||
Task { await self.connectManual() }
|
||||
} label: {
|
||||
if self.connectingGatewayID == "manual" {
|
||||
HStack(spacing: 8) {
|
||||
ProgressView()
|
||||
.progressViewStyle(.circular)
|
||||
Text("Connecting…")
|
||||
}
|
||||
} else {
|
||||
Text("Connect (Manual)")
|
||||
}
|
||||
}
|
||||
.disabled(self.connectingGatewayID != nil || self.manualGatewayHost
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
.isEmpty || self.manualGatewayPort <= 0 || self.manualGatewayPort > 65535)
|
||||
|
||||
Text(
|
||||
"Use this when mDNS/Bonjour discovery is blocked. "
|
||||
+ "The gateway WebSocket listens on port 18789 by default.")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
|
||||
Toggle("Discovery Debug Logs", isOn: self.$discoveryDebugLogsEnabled)
|
||||
.onChange(of: self.discoveryDebugLogsEnabled) { _, newValue in
|
||||
self.gatewayController.setDiscoveryDebugLoggingEnabled(newValue)
|
||||
}
|
||||
|
||||
NavigationLink("Discovery Logs") {
|
||||
GatewayDiscoveryDebugLogView()
|
||||
}
|
||||
|
||||
Toggle("Debug Canvas Status", isOn: self.$canvasDebugStatusEnabled)
|
||||
|
||||
TextField("Gateway Token", text: self.$gatewayToken)
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
|
||||
SecureField("Gateway Password", text: self.$gatewayPassword)
|
||||
}
|
||||
}
|
||||
|
||||
Section("Voice") {
|
||||
Toggle("Voice Wake", isOn: self.$voiceWakeEnabled)
|
||||
.onChange(of: self.voiceWakeEnabled) { _, newValue in
|
||||
self.appModel.setVoiceWakeEnabled(newValue)
|
||||
}
|
||||
Toggle("Talk Mode", isOn: self.$talkEnabled)
|
||||
.onChange(of: self.talkEnabled) { _, newValue in
|
||||
self.appModel.setTalkEnabled(newValue)
|
||||
}
|
||||
// Keep this separate so users can hide the side bubble without disabling Talk Mode.
|
||||
Toggle("Show Talk Button", isOn: self.$talkButtonEnabled)
|
||||
|
||||
NavigationLink {
|
||||
VoiceWakeWordsSettingsView()
|
||||
} label: {
|
||||
LabeledContent(
|
||||
"Wake Words",
|
||||
value: VoiceWakePreferences.displayString(for: self.voiceWake.triggerWords))
|
||||
}
|
||||
}
|
||||
|
||||
Section("Camera") {
|
||||
Toggle("Allow Camera", isOn: self.$cameraEnabled)
|
||||
Text("Allows the gateway to request photos or short video clips (foreground only).")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
|
||||
Section("Location") {
|
||||
Picker("Location Access", selection: self.$locationEnabledModeRaw) {
|
||||
Text("Off").tag(MoltbotLocationMode.off.rawValue)
|
||||
Text("While Using").tag(MoltbotLocationMode.whileUsing.rawValue)
|
||||
Text("Always").tag(MoltbotLocationMode.always.rawValue)
|
||||
}
|
||||
.pickerStyle(.segmented)
|
||||
|
||||
Toggle("Precise Location", isOn: self.$locationPreciseEnabled)
|
||||
.disabled(self.locationMode == .off)
|
||||
|
||||
Text("Always requires system permission and may prompt to open Settings.")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
|
||||
Section("Screen") {
|
||||
Toggle("Prevent Sleep", isOn: self.$preventSleep)
|
||||
Text("Keeps the screen awake while Moltbot is open.")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
.navigationTitle("Settings")
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .topBarTrailing) {
|
||||
Button {
|
||||
self.dismiss()
|
||||
} label: {
|
||||
Image(systemName: "xmark")
|
||||
}
|
||||
.accessibilityLabel("Close")
|
||||
}
|
||||
}
|
||||
.onAppear {
|
||||
self.localIPAddress = Self.primaryIPv4Address()
|
||||
self.lastLocationModeRaw = self.locationEnabledModeRaw
|
||||
let trimmedInstanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmedInstanceId.isEmpty {
|
||||
self.gatewayToken = GatewaySettingsStore.loadGatewayToken(instanceId: trimmedInstanceId) ?? ""
|
||||
self.gatewayPassword = GatewaySettingsStore.loadGatewayPassword(instanceId: trimmedInstanceId) ?? ""
|
||||
}
|
||||
}
|
||||
.onChange(of: self.preferredGatewayStableID) { _, newValue in
|
||||
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
GatewaySettingsStore.savePreferredGatewayStableID(trimmed)
|
||||
}
|
||||
.onChange(of: self.gatewayToken) { _, newValue in
|
||||
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let instanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !instanceId.isEmpty else { return }
|
||||
GatewaySettingsStore.saveGatewayToken(trimmed, instanceId: instanceId)
|
||||
}
|
||||
.onChange(of: self.gatewayPassword) { _, newValue in
|
||||
let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let instanceId = self.instanceId.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !instanceId.isEmpty else { return }
|
||||
GatewaySettingsStore.saveGatewayPassword(trimmed, instanceId: instanceId)
|
||||
}
|
||||
.onChange(of: self.appModel.gatewayServerName) { _, _ in
|
||||
self.connectStatus.text = nil
|
||||
}
|
||||
.onChange(of: self.locationEnabledModeRaw) { _, newValue in
|
||||
let previous = self.lastLocationModeRaw
|
||||
self.lastLocationModeRaw = newValue
|
||||
guard let mode = MoltbotLocationMode(rawValue: newValue) else { return }
|
||||
Task {
|
||||
let granted = await self.appModel.requestLocationPermissions(mode: mode)
|
||||
if !granted {
|
||||
await MainActor.run {
|
||||
self.locationEnabledModeRaw = previous
|
||||
self.lastLocationModeRaw = previous
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private func gatewayList(showing: GatewayListMode) -> some View {
|
||||
if self.gatewayController.gateways.isEmpty {
|
||||
Text("No gateways found yet.")
|
||||
.foregroundStyle(.secondary)
|
||||
} else {
|
||||
let connectedID = self.appModel.connectedGatewayID
|
||||
let rows = self.gatewayController.gateways.filter { gateway in
|
||||
let isConnected = gateway.stableID == connectedID
|
||||
switch showing {
|
||||
case .all:
|
||||
return true
|
||||
case .availableOnly:
|
||||
return !isConnected
|
||||
}
|
||||
}
|
||||
|
||||
if rows.isEmpty, showing == .availableOnly {
|
||||
Text("No other gateways found.")
|
||||
.foregroundStyle(.secondary)
|
||||
} else {
|
||||
ForEach(rows) { gateway in
|
||||
HStack {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text(gateway.name)
|
||||
let detailLines = self.gatewayDetailLines(gateway)
|
||||
ForEach(detailLines, id: \.self) { line in
|
||||
Text(line)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
Spacer()
|
||||
|
||||
Button {
|
||||
Task { await self.connect(gateway) }
|
||||
} label: {
|
||||
if self.connectingGatewayID == gateway.id {
|
||||
ProgressView()
|
||||
.progressViewStyle(.circular)
|
||||
} else {
|
||||
Text("Connect")
|
||||
}
|
||||
}
|
||||
.disabled(self.connectingGatewayID != nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private enum GatewayListMode: Equatable {
|
||||
case all
|
||||
case availableOnly
|
||||
}
|
||||
|
||||
private func platformString() -> String {
|
||||
let v = ProcessInfo.processInfo.operatingSystemVersion
|
||||
return "iOS \(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
|
||||
}
|
||||
|
||||
private var locationMode: MoltbotLocationMode {
|
||||
MoltbotLocationMode(rawValue: self.locationEnabledModeRaw) ?? .off
|
||||
}
|
||||
|
||||
private func appVersion() -> String {
|
||||
Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "dev"
|
||||
}
|
||||
|
||||
private func deviceFamily() -> String {
|
||||
switch UIDevice.current.userInterfaceIdiom {
|
||||
case .pad:
|
||||
"iPad"
|
||||
case .phone:
|
||||
"iPhone"
|
||||
default:
|
||||
"iOS"
|
||||
}
|
||||
}
|
||||
|
||||
private func modelIdentifier() -> String {
|
||||
var systemInfo = utsname()
|
||||
uname(&systemInfo)
|
||||
let machine = withUnsafeBytes(of: &systemInfo.machine) { ptr in
|
||||
String(bytes: ptr.prefix { $0 != 0 }, encoding: .utf8)
|
||||
}
|
||||
let trimmed = machine?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return trimmed.isEmpty ? "unknown" : trimmed
|
||||
}
|
||||
|
||||
private func connect(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) async {
|
||||
self.connectingGatewayID = gateway.id
|
||||
self.manualGatewayEnabled = false
|
||||
self.preferredGatewayStableID = gateway.stableID
|
||||
GatewaySettingsStore.savePreferredGatewayStableID(gateway.stableID)
|
||||
self.lastDiscoveredGatewayStableID = gateway.stableID
|
||||
GatewaySettingsStore.saveLastDiscoveredGatewayStableID(gateway.stableID)
|
||||
defer { self.connectingGatewayID = nil }
|
||||
|
||||
await self.gatewayController.connect(gateway)
|
||||
}
|
||||
|
||||
private func connectManual() async {
|
||||
let host = self.manualGatewayHost.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !host.isEmpty else {
|
||||
self.connectStatus.text = "Failed: host required"
|
||||
return
|
||||
}
|
||||
guard self.manualGatewayPort > 0, self.manualGatewayPort <= 65535 else {
|
||||
self.connectStatus.text = "Failed: invalid port"
|
||||
return
|
||||
}
|
||||
|
||||
self.connectingGatewayID = "manual"
|
||||
self.manualGatewayEnabled = true
|
||||
defer { self.connectingGatewayID = nil }
|
||||
|
||||
await self.gatewayController.connectManual(
|
||||
host: host,
|
||||
port: self.manualGatewayPort,
|
||||
useTLS: self.manualGatewayTLS)
|
||||
}
|
||||
|
||||
private static func primaryIPv4Address() -> String? {
|
||||
var addrList: UnsafeMutablePointer<ifaddrs>?
|
||||
guard getifaddrs(&addrList) == 0, let first = addrList else { return nil }
|
||||
defer { freeifaddrs(addrList) }
|
||||
|
||||
var fallback: String?
|
||||
var en0: String?
|
||||
|
||||
for ptr in sequence(first: first, next: { $0.pointee.ifa_next }) {
|
||||
let flags = Int32(ptr.pointee.ifa_flags)
|
||||
let isUp = (flags & IFF_UP) != 0
|
||||
let isLoopback = (flags & IFF_LOOPBACK) != 0
|
||||
let name = String(cString: ptr.pointee.ifa_name)
|
||||
let family = ptr.pointee.ifa_addr.pointee.sa_family
|
||||
if !isUp || isLoopback || family != UInt8(AF_INET) { continue }
|
||||
|
||||
var addr = ptr.pointee.ifa_addr.pointee
|
||||
var buffer = [CChar](repeating: 0, count: Int(NI_MAXHOST))
|
||||
let result = getnameinfo(
|
||||
&addr,
|
||||
socklen_t(ptr.pointee.ifa_addr.pointee.sa_len),
|
||||
&buffer,
|
||||
socklen_t(buffer.count),
|
||||
nil,
|
||||
0,
|
||||
NI_NUMERICHOST)
|
||||
guard result == 0 else { continue }
|
||||
let len = buffer.prefix { $0 != 0 }
|
||||
let bytes = len.map { UInt8(bitPattern: $0) }
|
||||
guard let ip = String(bytes: bytes, encoding: .utf8) else { continue }
|
||||
|
||||
if name == "en0" { en0 = ip; break }
|
||||
if fallback == nil { fallback = ip }
|
||||
}
|
||||
|
||||
return en0 ?? fallback
|
||||
}
|
||||
|
||||
private static func parseHostPort(from address: String) -> SettingsHostPort? {
|
||||
SettingsNetworkingHelpers.parseHostPort(from: address)
|
||||
}
|
||||
|
||||
private static func httpURLString(host: String?, port: Int?, fallback: String) -> String {
|
||||
SettingsNetworkingHelpers.httpURLString(host: host, port: port, fallback: fallback)
|
||||
}
|
||||
|
||||
private func gatewayDetailLines(_ gateway: GatewayDiscoveryModel.DiscoveredGateway) -> [String] {
|
||||
var lines: [String] = []
|
||||
if let lanHost = gateway.lanHost { lines.append("LAN: \(lanHost)") }
|
||||
if let tailnet = gateway.tailnetDns { lines.append("Tailnet: \(tailnet)") }
|
||||
|
||||
let gatewayPort = gateway.gatewayPort
|
||||
let canvasPort = gateway.canvasPort
|
||||
if gatewayPort != nil || canvasPort != nil {
|
||||
let gw = gatewayPort.map(String.init) ?? "—"
|
||||
let canvas = canvasPort.map(String.init) ?? "—"
|
||||
lines.append("Ports: gateway \(gw) · canvas \(canvas)")
|
||||
}
|
||||
|
||||
if lines.isEmpty {
|
||||
lines.append(gateway.debugID)
|
||||
}
|
||||
|
||||
return lines
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import SwiftUI
|
||||
import Combine
|
||||
|
||||
struct VoiceWakeWordsSettingsView: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
@State private var triggerWords: [String] = VoiceWakePreferences.loadTriggerWords()
|
||||
@FocusState private var focusedTriggerIndex: Int?
|
||||
@State private var syncTask: Task<Void, Never>?
|
||||
|
||||
var body: some View {
|
||||
Form {
|
||||
Section {
|
||||
ForEach(self.triggerWords.indices, id: \.self) { index in
|
||||
TextField("Wake word", text: self.binding(for: index))
|
||||
.textInputAutocapitalization(.never)
|
||||
.autocorrectionDisabled()
|
||||
.focused(self.$focusedTriggerIndex, equals: index)
|
||||
.onSubmit {
|
||||
self.commitTriggerWords()
|
||||
}
|
||||
}
|
||||
.onDelete(perform: self.removeWords)
|
||||
|
||||
Button {
|
||||
self.addWord()
|
||||
} label: {
|
||||
Label("Add word", systemImage: "plus")
|
||||
}
|
||||
.disabled(self.triggerWords
|
||||
.contains(where: { $0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }))
|
||||
|
||||
Button("Reset defaults") {
|
||||
self.triggerWords = VoiceWakePreferences.defaultTriggerWords
|
||||
}
|
||||
} header: {
|
||||
Text("Wake Words")
|
||||
} footer: {
|
||||
Text(
|
||||
"Moltbot reacts when any trigger appears in a transcription. "
|
||||
+ "Keep them short to avoid false positives.")
|
||||
}
|
||||
}
|
||||
.navigationTitle("Wake Words")
|
||||
.toolbar { EditButton() }
|
||||
.onAppear {
|
||||
if self.triggerWords.isEmpty {
|
||||
self.triggerWords = VoiceWakePreferences.defaultTriggerWords
|
||||
self.commitTriggerWords()
|
||||
}
|
||||
}
|
||||
.onChange(of: self.focusedTriggerIndex) { oldValue, newValue in
|
||||
guard oldValue != nil, oldValue != newValue else { return }
|
||||
self.commitTriggerWords()
|
||||
}
|
||||
.onReceive(NotificationCenter.default.publisher(for: UserDefaults.didChangeNotification)) { _ in
|
||||
guard self.focusedTriggerIndex == nil else { return }
|
||||
let updated = VoiceWakePreferences.loadTriggerWords()
|
||||
if updated != self.triggerWords {
|
||||
self.triggerWords = updated
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func addWord() {
|
||||
self.triggerWords.append("")
|
||||
}
|
||||
|
||||
private func removeWords(at offsets: IndexSet) {
|
||||
self.triggerWords.remove(atOffsets: offsets)
|
||||
if self.triggerWords.isEmpty {
|
||||
self.triggerWords = VoiceWakePreferences.defaultTriggerWords
|
||||
}
|
||||
self.commitTriggerWords()
|
||||
}
|
||||
|
||||
private func binding(for index: Int) -> Binding<String> {
|
||||
Binding(
|
||||
get: {
|
||||
guard self.triggerWords.indices.contains(index) else { return "" }
|
||||
return self.triggerWords[index]
|
||||
},
|
||||
set: { newValue in
|
||||
guard self.triggerWords.indices.contains(index) else { return }
|
||||
self.triggerWords[index] = newValue
|
||||
})
|
||||
}
|
||||
|
||||
private func commitTriggerWords() {
|
||||
VoiceWakePreferences.saveTriggerWords(self.triggerWords)
|
||||
|
||||
let snapshot = VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords)
|
||||
self.syncTask?.cancel()
|
||||
self.syncTask = Task { [snapshot, weak appModel = self.appModel] in
|
||||
try? await Task.sleep(nanoseconds: 650_000_000)
|
||||
await appModel?.setGlobalWakeWords(snapshot)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import SwiftUI
|
||||
|
||||
struct StatusPill: View {
|
||||
@Environment(\.scenePhase) private var scenePhase
|
||||
|
||||
enum GatewayState: Equatable {
|
||||
case connected
|
||||
case connecting
|
||||
case error
|
||||
case disconnected
|
||||
|
||||
var title: String {
|
||||
switch self {
|
||||
case .connected: "Connected"
|
||||
case .connecting: "Connecting…"
|
||||
case .error: "Error"
|
||||
case .disconnected: "Offline"
|
||||
}
|
||||
}
|
||||
|
||||
var color: Color {
|
||||
switch self {
|
||||
case .connected: .green
|
||||
case .connecting: .yellow
|
||||
case .error: .red
|
||||
case .disconnected: .gray
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Activity: Equatable {
|
||||
var title: String
|
||||
var systemImage: String
|
||||
var tint: Color?
|
||||
}
|
||||
|
||||
var gateway: GatewayState
|
||||
var voiceWakeEnabled: Bool
|
||||
var activity: Activity?
|
||||
var brighten: Bool = false
|
||||
var onTap: () -> Void
|
||||
|
||||
@State private var pulse: Bool = false
|
||||
|
||||
var body: some View {
|
||||
Button(action: self.onTap) {
|
||||
HStack(spacing: 10) {
|
||||
HStack(spacing: 8) {
|
||||
Circle()
|
||||
.fill(self.gateway.color)
|
||||
.frame(width: 9, height: 9)
|
||||
.scaleEffect(self.gateway == .connecting ? (self.pulse ? 1.15 : 0.85) : 1.0)
|
||||
.opacity(self.gateway == .connecting ? (self.pulse ? 1.0 : 0.6) : 1.0)
|
||||
|
||||
Text(self.gateway.title)
|
||||
.font(.system(size: 13, weight: .semibold))
|
||||
.foregroundStyle(.primary)
|
||||
}
|
||||
|
||||
Divider()
|
||||
.frame(height: 14)
|
||||
.opacity(0.35)
|
||||
|
||||
if let activity {
|
||||
HStack(spacing: 6) {
|
||||
Image(systemName: activity.systemImage)
|
||||
.font(.system(size: 13, weight: .semibold))
|
||||
.foregroundStyle(activity.tint ?? .primary)
|
||||
Text(activity.title)
|
||||
.font(.system(size: 13, weight: .semibold))
|
||||
.foregroundStyle(.primary)
|
||||
.lineLimit(1)
|
||||
}
|
||||
.transition(.opacity.combined(with: .move(edge: .top)))
|
||||
} else {
|
||||
Image(systemName: self.voiceWakeEnabled ? "mic.fill" : "mic.slash")
|
||||
.font(.system(size: 13, weight: .semibold))
|
||||
.foregroundStyle(self.voiceWakeEnabled ? .primary : .secondary)
|
||||
.accessibilityLabel(self.voiceWakeEnabled ? "Voice Wake enabled" : "Voice Wake disabled")
|
||||
.transition(.opacity.combined(with: .move(edge: .top)))
|
||||
}
|
||||
}
|
||||
.padding(.vertical, 8)
|
||||
.padding(.horizontal, 12)
|
||||
.background {
|
||||
RoundedRectangle(cornerRadius: 14, style: .continuous)
|
||||
.fill(.ultraThinMaterial)
|
||||
.overlay {
|
||||
RoundedRectangle(cornerRadius: 14, style: .continuous)
|
||||
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
|
||||
}
|
||||
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
|
||||
}
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
.accessibilityLabel("Status")
|
||||
.accessibilityValue(self.accessibilityValue)
|
||||
.onAppear { self.updatePulse(for: self.gateway, scenePhase: self.scenePhase) }
|
||||
.onDisappear { self.pulse = false }
|
||||
.onChange(of: self.gateway) { _, newValue in
|
||||
self.updatePulse(for: newValue, scenePhase: self.scenePhase)
|
||||
}
|
||||
.onChange(of: self.scenePhase) { _, newValue in
|
||||
self.updatePulse(for: self.gateway, scenePhase: newValue)
|
||||
}
|
||||
.animation(.easeInOut(duration: 0.18), value: self.activity?.title)
|
||||
}
|
||||
|
||||
private var accessibilityValue: String {
|
||||
if let activity {
|
||||
return "\(self.gateway.title), \(activity.title)"
|
||||
}
|
||||
return "\(self.gateway.title), Voice Wake \(self.voiceWakeEnabled ? "enabled" : "disabled")"
|
||||
}
|
||||
|
||||
private func updatePulse(for gateway: GatewayState, scenePhase: ScenePhase) {
|
||||
guard gateway == .connecting, scenePhase == .active else {
|
||||
withAnimation(.easeOut(duration: 0.2)) { self.pulse = false }
|
||||
return
|
||||
}
|
||||
|
||||
guard !self.pulse else { return }
|
||||
withAnimation(.easeInOut(duration: 0.9).repeatForever(autoreverses: true)) {
|
||||
self.pulse = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import SwiftUI
|
||||
|
||||
struct VoiceWakeToast: View {
|
||||
var command: String
|
||||
var brighten: Bool = false
|
||||
|
||||
var body: some View {
|
||||
HStack(spacing: 10) {
|
||||
Image(systemName: "mic.fill")
|
||||
.font(.system(size: 14, weight: .semibold))
|
||||
.foregroundStyle(.primary)
|
||||
|
||||
Text(self.command)
|
||||
.font(.system(size: 14, weight: .semibold))
|
||||
.foregroundStyle(.primary)
|
||||
.lineLimit(1)
|
||||
.truncationMode(.tail)
|
||||
}
|
||||
.padding(.vertical, 10)
|
||||
.padding(.horizontal, 12)
|
||||
.background {
|
||||
RoundedRectangle(cornerRadius: 14, style: .continuous)
|
||||
.fill(.ultraThinMaterial)
|
||||
.overlay {
|
||||
RoundedRectangle(cornerRadius: 14, style: .continuous)
|
||||
.strokeBorder(.white.opacity(self.brighten ? 0.24 : 0.18), lineWidth: 0.5)
|
||||
}
|
||||
.shadow(color: .black.opacity(0.25), radius: 12, y: 6)
|
||||
}
|
||||
.accessibilityLabel("Voice Wake")
|
||||
.accessibilityValue(self.command)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,729 @@
|
||||
import AVFAudio
|
||||
import MoltbotKit
|
||||
import MoltbotProtocol
|
||||
import Foundation
|
||||
import Observation
|
||||
import OSLog
|
||||
import Speech
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class TalkModeManager: NSObject {
|
||||
private typealias SpeechRequest = SFSpeechAudioBufferRecognitionRequest
|
||||
private static let defaultModelIdFallback = "eleven_v3"
|
||||
var isEnabled: Bool = false
|
||||
var isListening: Bool = false
|
||||
var isSpeaking: Bool = false
|
||||
var statusText: String = "Off"
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private var speechRecognizer: SFSpeechRecognizer?
|
||||
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
||||
private var recognitionTask: SFSpeechRecognitionTask?
|
||||
private var silenceTask: Task<Void, Never>?
|
||||
|
||||
private var lastHeard: Date?
|
||||
private var lastTranscript: String = ""
|
||||
private var lastSpokenText: String?
|
||||
private var lastInterruptedAtSeconds: Double?
|
||||
|
||||
private var defaultVoiceId: String?
|
||||
private var currentVoiceId: String?
|
||||
private var defaultModelId: String?
|
||||
private var currentModelId: String?
|
||||
private var voiceOverrideActive = false
|
||||
private var modelOverrideActive = false
|
||||
private var defaultOutputFormat: String?
|
||||
private var apiKey: String?
|
||||
private var voiceAliases: [String: String] = [:]
|
||||
private var interruptOnSpeech: Bool = true
|
||||
private var mainSessionKey: String = "main"
|
||||
private var fallbackVoiceId: String?
|
||||
private var lastPlaybackWasPCM: Bool = false
|
||||
var pcmPlayer: PCMStreamingAudioPlaying = PCMStreamingAudioPlayer.shared
|
||||
var mp3Player: StreamingAudioPlaying = StreamingAudioPlayer.shared
|
||||
|
||||
private var gateway: GatewayNodeSession?
|
||||
private let silenceWindow: TimeInterval = 0.7
|
||||
|
||||
private var chatSubscribedSessionKeys = Set<String>()
|
||||
|
||||
private let logger = Logger(subsystem: "bot.molt", category: "TalkMode")
|
||||
|
||||
func attachGateway(_ gateway: GatewayNodeSession) {
|
||||
self.gateway = gateway
|
||||
}
|
||||
|
||||
func updateMainSessionKey(_ sessionKey: String?) {
|
||||
let trimmed = (sessionKey ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
if SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { return }
|
||||
self.mainSessionKey = trimmed
|
||||
}
|
||||
|
||||
func setEnabled(_ enabled: Bool) {
|
||||
self.isEnabled = enabled
|
||||
if enabled {
|
||||
self.logger.info("enabled")
|
||||
Task { await self.start() }
|
||||
} else {
|
||||
self.logger.info("disabled")
|
||||
self.stop()
|
||||
}
|
||||
}
|
||||
|
||||
func start() async {
|
||||
guard self.isEnabled else { return }
|
||||
if self.isListening { return }
|
||||
|
||||
self.logger.info("start")
|
||||
self.statusText = "Requesting permissions…"
|
||||
let micOk = await Self.requestMicrophonePermission()
|
||||
guard micOk else {
|
||||
self.logger.warning("start blocked: microphone permission denied")
|
||||
self.statusText = "Microphone permission denied"
|
||||
return
|
||||
}
|
||||
let speechOk = await Self.requestSpeechPermission()
|
||||
guard speechOk else {
|
||||
self.logger.warning("start blocked: speech permission denied")
|
||||
self.statusText = "Speech recognition permission denied"
|
||||
return
|
||||
}
|
||||
|
||||
await self.reloadConfig()
|
||||
do {
|
||||
try Self.configureAudioSession()
|
||||
try self.startRecognition()
|
||||
self.isListening = true
|
||||
self.statusText = "Listening"
|
||||
self.startSilenceMonitor()
|
||||
await self.subscribeChatIfNeeded(sessionKey: self.mainSessionKey)
|
||||
self.logger.info("listening")
|
||||
} catch {
|
||||
self.isListening = false
|
||||
self.statusText = "Start failed: \(error.localizedDescription)"
|
||||
self.logger.error("start failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
self.isEnabled = false
|
||||
self.isListening = false
|
||||
self.statusText = "Off"
|
||||
self.lastTranscript = ""
|
||||
self.lastHeard = nil
|
||||
self.silenceTask?.cancel()
|
||||
self.silenceTask = nil
|
||||
self.stopRecognition()
|
||||
self.stopSpeaking()
|
||||
self.lastInterruptedAtSeconds = nil
|
||||
TalkSystemSpeechSynthesizer.shared.stop()
|
||||
do {
|
||||
try AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation])
|
||||
} catch {
|
||||
self.logger.warning("audio session deactivate failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
Task { await self.unsubscribeAllChats() }
|
||||
}
|
||||
|
||||
func userTappedOrb() {
|
||||
self.stopSpeaking()
|
||||
}
|
||||
|
||||
private func startRecognition() throws {
|
||||
#if targetEnvironment(simulator)
|
||||
throw NSError(domain: "TalkMode", code: 2, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Talk mode is not supported on the iOS simulator",
|
||||
])
|
||||
#endif
|
||||
|
||||
self.stopRecognition()
|
||||
self.speechRecognizer = SFSpeechRecognizer()
|
||||
guard let recognizer = self.speechRecognizer else {
|
||||
throw NSError(domain: "TalkMode", code: 1, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Speech recognizer unavailable",
|
||||
])
|
||||
}
|
||||
|
||||
self.recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
|
||||
self.recognitionRequest?.shouldReportPartialResults = true
|
||||
guard let request = self.recognitionRequest else { return }
|
||||
|
||||
let input = self.audioEngine.inputNode
|
||||
let format = input.outputFormat(forBus: 0)
|
||||
guard format.sampleRate > 0, format.channelCount > 0 else {
|
||||
throw NSError(domain: "TalkMode", code: 3, userInfo: [
|
||||
NSLocalizedDescriptionKey: "Invalid audio input format",
|
||||
])
|
||||
}
|
||||
input.removeTap(onBus: 0)
|
||||
let tapBlock = Self.makeAudioTapAppendCallback(request: request)
|
||||
input.installTap(onBus: 0, bufferSize: 2048, format: format, block: tapBlock)
|
||||
|
||||
self.audioEngine.prepare()
|
||||
try self.audioEngine.start()
|
||||
|
||||
self.recognitionTask = recognizer.recognitionTask(with: request) { [weak self] result, error in
|
||||
guard let self else { return }
|
||||
if let error {
|
||||
if !self.isSpeaking {
|
||||
self.statusText = "Speech error: \(error.localizedDescription)"
|
||||
}
|
||||
self.logger.debug("speech recognition error: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
guard let result else { return }
|
||||
let transcript = result.bestTranscription.formattedString
|
||||
Task { @MainActor in
|
||||
await self.handleTranscript(transcript: transcript, isFinal: result.isFinal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func stopRecognition() {
|
||||
self.recognitionTask?.cancel()
|
||||
self.recognitionTask = nil
|
||||
self.recognitionRequest?.endAudio()
|
||||
self.recognitionRequest = nil
|
||||
self.audioEngine.inputNode.removeTap(onBus: 0)
|
||||
self.audioEngine.stop()
|
||||
self.speechRecognizer = nil
|
||||
}
|
||||
|
||||
private nonisolated static func makeAudioTapAppendCallback(request: SpeechRequest) -> AVAudioNodeTapBlock {
|
||||
{ buffer, _ in
|
||||
request.append(buffer)
|
||||
}
|
||||
}
|
||||
|
||||
private func handleTranscript(transcript: String, isFinal: Bool) async {
|
||||
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if self.isSpeaking, self.interruptOnSpeech {
|
||||
if self.shouldInterrupt(with: trimmed) {
|
||||
self.stopSpeaking()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
guard self.isListening else { return }
|
||||
if !trimmed.isEmpty {
|
||||
self.lastTranscript = trimmed
|
||||
self.lastHeard = Date()
|
||||
}
|
||||
if isFinal {
|
||||
self.lastTranscript = trimmed
|
||||
}
|
||||
}
|
||||
|
||||
private func startSilenceMonitor() {
|
||||
self.silenceTask?.cancel()
|
||||
self.silenceTask = Task { [weak self] in
|
||||
guard let self else { return }
|
||||
while self.isEnabled {
|
||||
try? await Task.sleep(nanoseconds: 200_000_000)
|
||||
await self.checkSilence()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func checkSilence() async {
|
||||
guard self.isListening, !self.isSpeaking else { return }
|
||||
let transcript = self.lastTranscript.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !transcript.isEmpty else { return }
|
||||
guard let lastHeard else { return }
|
||||
if Date().timeIntervalSince(lastHeard) < self.silenceWindow { return }
|
||||
await self.finalizeTranscript(transcript)
|
||||
}
|
||||
|
||||
private func finalizeTranscript(_ transcript: String) async {
|
||||
self.isListening = false
|
||||
self.statusText = "Thinking…"
|
||||
self.lastTranscript = ""
|
||||
self.lastHeard = nil
|
||||
self.stopRecognition()
|
||||
|
||||
await self.reloadConfig()
|
||||
let prompt = self.buildPrompt(transcript: transcript)
|
||||
guard let gateway else {
|
||||
self.statusText = "Gateway not connected"
|
||||
self.logger.warning("finalize: gateway not connected")
|
||||
await self.start()
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
let startedAt = Date().timeIntervalSince1970
|
||||
let sessionKey = self.mainSessionKey
|
||||
await self.subscribeChatIfNeeded(sessionKey: sessionKey)
|
||||
self.logger.info(
|
||||
"chat.send start sessionKey=\(sessionKey, privacy: .public) chars=\(prompt.count, privacy: .public)")
|
||||
let runId = try await self.sendChat(prompt, gateway: gateway)
|
||||
self.logger.info("chat.send ok runId=\(runId, privacy: .public)")
|
||||
let completion = await self.waitForChatCompletion(runId: runId, gateway: gateway, timeoutSeconds: 120)
|
||||
if completion == .timeout {
|
||||
self.logger.warning(
|
||||
"chat completion timeout runId=\(runId, privacy: .public); attempting history fallback")
|
||||
} else if completion == .aborted {
|
||||
self.statusText = "Aborted"
|
||||
self.logger.warning("chat completion aborted runId=\(runId, privacy: .public)")
|
||||
await self.start()
|
||||
return
|
||||
} else if completion == .error {
|
||||
self.statusText = "Chat error"
|
||||
self.logger.warning("chat completion error runId=\(runId, privacy: .public)")
|
||||
await self.start()
|
||||
return
|
||||
}
|
||||
|
||||
guard let assistantText = try await self.waitForAssistantText(
|
||||
gateway: gateway,
|
||||
since: startedAt,
|
||||
timeoutSeconds: completion == .final ? 12 : 25)
|
||||
else {
|
||||
self.statusText = "No reply"
|
||||
self.logger.warning("assistant text timeout runId=\(runId, privacy: .public)")
|
||||
await self.start()
|
||||
return
|
||||
}
|
||||
self.logger.info("assistant text ok chars=\(assistantText.count, privacy: .public)")
|
||||
await self.playAssistant(text: assistantText)
|
||||
} catch {
|
||||
self.statusText = "Talk failed: \(error.localizedDescription)"
|
||||
self.logger.error("finalize failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
|
||||
await self.start()
|
||||
}
|
||||
|
||||
private func subscribeChatIfNeeded(sessionKey: String) async {
|
||||
let key = sessionKey.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !key.isEmpty else { return }
|
||||
guard let gateway else { return }
|
||||
guard !self.chatSubscribedSessionKeys.contains(key) else { return }
|
||||
|
||||
let payload = "{\"sessionKey\":\"\(key)\"}"
|
||||
await gateway.sendEvent(event: "chat.subscribe", payloadJSON: payload)
|
||||
self.chatSubscribedSessionKeys.insert(key)
|
||||
self.logger.info("chat.subscribe ok sessionKey=\(key, privacy: .public)")
|
||||
}
|
||||
|
||||
private func unsubscribeAllChats() async {
|
||||
guard let gateway else { return }
|
||||
let keys = self.chatSubscribedSessionKeys
|
||||
self.chatSubscribedSessionKeys.removeAll()
|
||||
for key in keys {
|
||||
let payload = "{\"sessionKey\":\"\(key)\"}"
|
||||
await gateway.sendEvent(event: "chat.unsubscribe", payloadJSON: payload)
|
||||
}
|
||||
}
|
||||
|
||||
private func buildPrompt(transcript: String) -> String {
|
||||
let interrupted = self.lastInterruptedAtSeconds
|
||||
self.lastInterruptedAtSeconds = nil
|
||||
return TalkPromptBuilder.build(transcript: transcript, interruptedAtSeconds: interrupted)
|
||||
}
|
||||
|
||||
private enum ChatCompletionState: CustomStringConvertible {
|
||||
case final
|
||||
case aborted
|
||||
case error
|
||||
case timeout
|
||||
|
||||
var description: String {
|
||||
switch self {
|
||||
case .final: "final"
|
||||
case .aborted: "aborted"
|
||||
case .error: "error"
|
||||
case .timeout: "timeout"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func sendChat(_ message: String, gateway: GatewayNodeSession) async throws -> String {
|
||||
struct SendResponse: Decodable { let runId: String }
|
||||
let payload: [String: Any] = [
|
||||
"sessionKey": self.mainSessionKey,
|
||||
"message": message,
|
||||
"thinking": "low",
|
||||
"timeoutMs": 30000,
|
||||
"idempotencyKey": UUID().uuidString,
|
||||
]
|
||||
let data = try JSONSerialization.data(withJSONObject: payload)
|
||||
guard let json = String(bytes: data, encoding: .utf8) else {
|
||||
throw NSError(
|
||||
domain: "TalkModeManager",
|
||||
code: 1,
|
||||
userInfo: [NSLocalizedDescriptionKey: "Failed to encode chat payload"])
|
||||
}
|
||||
let res = try await gateway.request(method: "chat.send", paramsJSON: json, timeoutSeconds: 30)
|
||||
let decoded = try JSONDecoder().decode(SendResponse.self, from: res)
|
||||
return decoded.runId
|
||||
}
|
||||
|
||||
private func waitForChatCompletion(
|
||||
runId: String,
|
||||
gateway: GatewayNodeSession,
|
||||
timeoutSeconds: Int = 120) async -> ChatCompletionState
|
||||
{
|
||||
let stream = await gateway.subscribeServerEvents(bufferingNewest: 200)
|
||||
return await withTaskGroup(of: ChatCompletionState.self) { group in
|
||||
group.addTask { [runId] in
|
||||
for await evt in stream {
|
||||
if Task.isCancelled { return .timeout }
|
||||
guard evt.event == "chat", let payload = evt.payload else { continue }
|
||||
guard let chatEvent = try? GatewayPayloadDecoding.decode(payload, as: ChatEvent.self) else {
|
||||
continue
|
||||
}
|
||||
guard chatEvent.runid == runId else { continue }
|
||||
if let state = chatEvent.state.value as? String {
|
||||
switch state {
|
||||
case "final": return .final
|
||||
case "aborted": return .aborted
|
||||
case "error": return .error
|
||||
default: break
|
||||
}
|
||||
}
|
||||
}
|
||||
return .timeout
|
||||
}
|
||||
group.addTask {
|
||||
try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds) * 1_000_000_000)
|
||||
return .timeout
|
||||
}
|
||||
let result = await group.next() ?? .timeout
|
||||
group.cancelAll()
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
private func waitForAssistantText(
|
||||
gateway: GatewayNodeSession,
|
||||
since: Double,
|
||||
timeoutSeconds: Int) async throws -> String?
|
||||
{
|
||||
let deadline = Date().addingTimeInterval(TimeInterval(timeoutSeconds))
|
||||
while Date() < deadline {
|
||||
if let text = try await self.fetchLatestAssistantText(gateway: gateway, since: since) {
|
||||
return text
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 300_000_000)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private func fetchLatestAssistantText(gateway: GatewayNodeSession, since: Double? = nil) async throws -> String? {
|
||||
let res = try await gateway.request(
|
||||
method: "chat.history",
|
||||
paramsJSON: "{\"sessionKey\":\"\(self.mainSessionKey)\"}",
|
||||
timeoutSeconds: 15)
|
||||
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return nil }
|
||||
guard let messages = json["messages"] as? [[String: Any]] else { return nil }
|
||||
for msg in messages.reversed() {
|
||||
guard (msg["role"] as? String) == "assistant" else { continue }
|
||||
if let since, let timestamp = msg["timestamp"] as? Double,
|
||||
TalkHistoryTimestamp.isAfter(timestamp, sinceSeconds: since) == false
|
||||
{
|
||||
continue
|
||||
}
|
||||
guard let content = msg["content"] as? [[String: Any]] else { continue }
|
||||
let text = content.compactMap { $0["text"] as? String }.joined(separator: "\n")
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !trimmed.isEmpty { return trimmed }
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private func playAssistant(text: String) async {
|
||||
let parsed = TalkDirectiveParser.parse(text)
|
||||
let directive = parsed.directive
|
||||
let cleaned = parsed.stripped.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !cleaned.isEmpty else { return }
|
||||
|
||||
let requestedVoice = directive?.voiceId?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let resolvedVoice = self.resolveVoiceAlias(requestedVoice)
|
||||
if requestedVoice?.isEmpty == false, resolvedVoice == nil {
|
||||
self.logger.warning("unknown voice alias \(requestedVoice ?? "?", privacy: .public)")
|
||||
}
|
||||
if let voice = resolvedVoice {
|
||||
if directive?.once != true {
|
||||
self.currentVoiceId = voice
|
||||
self.voiceOverrideActive = true
|
||||
}
|
||||
}
|
||||
if let model = directive?.modelId {
|
||||
if directive?.once != true {
|
||||
self.currentModelId = model
|
||||
self.modelOverrideActive = true
|
||||
}
|
||||
}
|
||||
|
||||
self.statusText = "Generating voice…"
|
||||
self.isSpeaking = true
|
||||
self.lastSpokenText = cleaned
|
||||
|
||||
do {
|
||||
let started = Date()
|
||||
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
|
||||
|
||||
let resolvedKey =
|
||||
(self.apiKey?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false ? self.apiKey : nil) ??
|
||||
ProcessInfo.processInfo.environment["ELEVENLABS_API_KEY"]
|
||||
let apiKey = resolvedKey?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let preferredVoice = resolvedVoice ?? self.currentVoiceId ?? self.defaultVoiceId
|
||||
let voiceId: String? = if let apiKey, !apiKey.isEmpty {
|
||||
await self.resolveVoiceId(preferred: preferredVoice, apiKey: apiKey)
|
||||
} else {
|
||||
nil
|
||||
}
|
||||
let canUseElevenLabs = (voiceId?.isEmpty == false) && (apiKey?.isEmpty == false)
|
||||
|
||||
if canUseElevenLabs, let voiceId, let apiKey {
|
||||
let desiredOutputFormat = (directive?.outputFormat ?? self.defaultOutputFormat)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let requestedOutputFormat = (desiredOutputFormat?.isEmpty == false) ? desiredOutputFormat : nil
|
||||
let outputFormat = ElevenLabsTTSClient.validatedOutputFormat(requestedOutputFormat ?? "pcm_44100")
|
||||
if outputFormat == nil, let requestedOutputFormat {
|
||||
self.logger.warning(
|
||||
"talk output_format unsupported for local playback: \(requestedOutputFormat, privacy: .public)")
|
||||
}
|
||||
|
||||
let modelId = directive?.modelId ?? self.currentModelId ?? self.defaultModelId
|
||||
func makeRequest(outputFormat: String?) -> ElevenLabsTTSRequest {
|
||||
ElevenLabsTTSRequest(
|
||||
text: cleaned,
|
||||
modelId: modelId,
|
||||
outputFormat: outputFormat,
|
||||
speed: TalkTTSValidation.resolveSpeed(speed: directive?.speed, rateWPM: directive?.rateWPM),
|
||||
stability: TalkTTSValidation.validatedStability(directive?.stability, modelId: modelId),
|
||||
similarity: TalkTTSValidation.validatedUnit(directive?.similarity),
|
||||
style: TalkTTSValidation.validatedUnit(directive?.style),
|
||||
speakerBoost: directive?.speakerBoost,
|
||||
seed: TalkTTSValidation.validatedSeed(directive?.seed),
|
||||
normalize: ElevenLabsTTSClient.validatedNormalize(directive?.normalize),
|
||||
language: language,
|
||||
latencyTier: TalkTTSValidation.validatedLatencyTier(directive?.latencyTier))
|
||||
}
|
||||
|
||||
let request = makeRequest(outputFormat: outputFormat)
|
||||
|
||||
let client = ElevenLabsTTSClient(apiKey: apiKey)
|
||||
let stream = client.streamSynthesize(voiceId: voiceId, request: request)
|
||||
|
||||
if self.interruptOnSpeech {
|
||||
do {
|
||||
try self.startRecognition()
|
||||
} catch {
|
||||
self.logger.warning(
|
||||
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
self.statusText = "Speaking…"
|
||||
let sampleRate = TalkTTSValidation.pcmSampleRate(from: outputFormat)
|
||||
let result: StreamingPlaybackResult
|
||||
if let sampleRate {
|
||||
self.lastPlaybackWasPCM = true
|
||||
var playback = await self.pcmPlayer.play(stream: stream, sampleRate: sampleRate)
|
||||
if !playback.finished, playback.interruptedAt == nil {
|
||||
let mp3Format = ElevenLabsTTSClient.validatedOutputFormat("mp3_44100")
|
||||
self.logger.warning("pcm playback failed; retrying mp3")
|
||||
self.lastPlaybackWasPCM = false
|
||||
let mp3Stream = client.streamSynthesize(
|
||||
voiceId: voiceId,
|
||||
request: makeRequest(outputFormat: mp3Format))
|
||||
playback = await self.mp3Player.play(stream: mp3Stream)
|
||||
}
|
||||
result = playback
|
||||
} else {
|
||||
self.lastPlaybackWasPCM = false
|
||||
result = await self.mp3Player.play(stream: stream)
|
||||
}
|
||||
let duration = Date().timeIntervalSince(started)
|
||||
self.logger.info("elevenlabs stream finished=\(result.finished, privacy: .public) dur=\(duration, privacy: .public)s")
|
||||
if !result.finished, let interruptedAt = result.interruptedAt {
|
||||
self.lastInterruptedAtSeconds = interruptedAt
|
||||
}
|
||||
} else {
|
||||
self.logger.warning("tts unavailable; falling back to system voice (missing key or voiceId)")
|
||||
if self.interruptOnSpeech {
|
||||
do {
|
||||
try self.startRecognition()
|
||||
} catch {
|
||||
self.logger.warning(
|
||||
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
self.statusText = "Speaking (System)…"
|
||||
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
|
||||
}
|
||||
} catch {
|
||||
self.logger.error(
|
||||
"tts failed: \(error.localizedDescription, privacy: .public); falling back to system voice")
|
||||
do {
|
||||
if self.interruptOnSpeech {
|
||||
do {
|
||||
try self.startRecognition()
|
||||
} catch {
|
||||
self.logger.warning(
|
||||
"startRecognition during speak failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
self.statusText = "Speaking (System)…"
|
||||
let language = ElevenLabsTTSClient.validatedLanguage(directive?.language)
|
||||
try await TalkSystemSpeechSynthesizer.shared.speak(text: cleaned, language: language)
|
||||
} catch {
|
||||
self.statusText = "Speak failed: \(error.localizedDescription)"
|
||||
self.logger.error("system voice failed: \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
}
|
||||
|
||||
self.stopRecognition()
|
||||
self.isSpeaking = false
|
||||
}
|
||||
|
||||
private func stopSpeaking(storeInterruption: Bool = true) {
|
||||
guard self.isSpeaking else { return }
|
||||
let interruptedAt = self.lastPlaybackWasPCM
|
||||
? self.pcmPlayer.stop()
|
||||
: self.mp3Player.stop()
|
||||
if storeInterruption {
|
||||
self.lastInterruptedAtSeconds = interruptedAt
|
||||
}
|
||||
_ = self.lastPlaybackWasPCM
|
||||
? self.mp3Player.stop()
|
||||
: self.pcmPlayer.stop()
|
||||
TalkSystemSpeechSynthesizer.shared.stop()
|
||||
self.isSpeaking = false
|
||||
}
|
||||
|
||||
private func shouldInterrupt(with transcript: String) -> Bool {
|
||||
let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard trimmed.count >= 3 else { return false }
|
||||
if let spoken = self.lastSpokenText?.lowercased(), spoken.contains(trimmed.lowercased()) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
private func resolveVoiceAlias(_ value: String?) -> String? {
|
||||
let trimmed = (value ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return nil }
|
||||
let normalized = trimmed.lowercased()
|
||||
if let mapped = self.voiceAliases[normalized] { return mapped }
|
||||
if self.voiceAliases.values.contains(where: { $0.caseInsensitiveCompare(trimmed) == .orderedSame }) {
|
||||
return trimmed
|
||||
}
|
||||
return Self.isLikelyVoiceId(trimmed) ? trimmed : nil
|
||||
}
|
||||
|
||||
private func resolveVoiceId(preferred: String?, apiKey: String) async -> String? {
|
||||
let trimmed = preferred?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
if !trimmed.isEmpty {
|
||||
if let resolved = self.resolveVoiceAlias(trimmed) { return resolved }
|
||||
self.logger.warning("unknown voice alias \(trimmed, privacy: .public)")
|
||||
}
|
||||
if let fallbackVoiceId { return fallbackVoiceId }
|
||||
|
||||
do {
|
||||
let voices = try await ElevenLabsTTSClient(apiKey: apiKey).listVoices()
|
||||
guard let first = voices.first else {
|
||||
self.logger.warning("elevenlabs voices list empty")
|
||||
return nil
|
||||
}
|
||||
self.fallbackVoiceId = first.voiceId
|
||||
if self.defaultVoiceId == nil {
|
||||
self.defaultVoiceId = first.voiceId
|
||||
}
|
||||
if !self.voiceOverrideActive {
|
||||
self.currentVoiceId = first.voiceId
|
||||
}
|
||||
let name = first.name ?? "unknown"
|
||||
self.logger
|
||||
.info("default voice selected \(name, privacy: .public) (\(first.voiceId, privacy: .public))")
|
||||
return first.voiceId
|
||||
} catch {
|
||||
self.logger.error("elevenlabs list voices failed: \(error.localizedDescription, privacy: .public)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
private static func isLikelyVoiceId(_ value: String) -> Bool {
|
||||
guard value.count >= 10 else { return false }
|
||||
return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "-" || $0 == "_" }
|
||||
}
|
||||
|
||||
private func reloadConfig() async {
|
||||
guard let gateway else { return }
|
||||
do {
|
||||
let res = try await gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8)
|
||||
guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return }
|
||||
guard let config = json["config"] as? [String: Any] else { return }
|
||||
let talk = config["talk"] as? [String: Any]
|
||||
let session = config["session"] as? [String: Any]
|
||||
let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String)
|
||||
if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) {
|
||||
self.mainSessionKey = mainKey
|
||||
}
|
||||
self.defaultVoiceId = (talk?["voiceId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if let aliases = talk?["voiceAliases"] as? [String: Any] {
|
||||
var resolved: [String: String] = [:]
|
||||
for (key, value) in aliases {
|
||||
guard let id = value as? String else { continue }
|
||||
let normalizedKey = key.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
||||
let trimmedId = id.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !normalizedKey.isEmpty, !trimmedId.isEmpty else { continue }
|
||||
resolved[normalizedKey] = trimmedId
|
||||
}
|
||||
self.voiceAliases = resolved
|
||||
} else {
|
||||
self.voiceAliases = [:]
|
||||
}
|
||||
if !self.voiceOverrideActive {
|
||||
self.currentVoiceId = self.defaultVoiceId
|
||||
}
|
||||
let model = (talk?["modelId"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.defaultModelId = (model?.isEmpty == false) ? model : Self.defaultModelIdFallback
|
||||
if !self.modelOverrideActive {
|
||||
self.currentModelId = self.defaultModelId
|
||||
}
|
||||
self.defaultOutputFormat = (talk?["outputFormat"] as? String)?
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
self.apiKey = (talk?["apiKey"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if let interrupt = talk?["interruptOnSpeech"] as? Bool {
|
||||
self.interruptOnSpeech = interrupt
|
||||
}
|
||||
} catch {
|
||||
self.defaultModelId = Self.defaultModelIdFallback
|
||||
if !self.modelOverrideActive {
|
||||
self.currentModelId = self.defaultModelId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func configureAudioSession() throws {
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try session.setCategory(.playAndRecord, mode: .voiceChat, options: [
|
||||
.duckOthers,
|
||||
.mixWithOthers,
|
||||
.allowBluetoothHFP,
|
||||
.defaultToSpeaker,
|
||||
])
|
||||
try session.setActive(true, options: [])
|
||||
}
|
||||
|
||||
private nonisolated static func requestMicrophonePermission() async -> Bool {
|
||||
await withCheckedContinuation(isolation: nil) { cont in
|
||||
AVAudioApplication.requestRecordPermission { ok in
|
||||
cont.resume(returning: ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func requestSpeechPermission() async -> Bool {
|
||||
await withCheckedContinuation(isolation: nil) { cont in
|
||||
SFSpeechRecognizer.requestAuthorization { status in
|
||||
cont.resume(returning: status == .authorized)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import SwiftUI
|
||||
|
||||
struct TalkOrbOverlay: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
@State private var pulse: Bool = false
|
||||
|
||||
var body: some View {
|
||||
let seam = self.appModel.seamColor
|
||||
let status = self.appModel.talkMode.statusText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
VStack(spacing: 14) {
|
||||
ZStack {
|
||||
Circle()
|
||||
.stroke(seam.opacity(0.26), lineWidth: 2)
|
||||
.frame(width: 320, height: 320)
|
||||
.scaleEffect(self.pulse ? 1.15 : 0.96)
|
||||
.opacity(self.pulse ? 0.0 : 1.0)
|
||||
.animation(.easeOut(duration: 1.3).repeatForever(autoreverses: false), value: self.pulse)
|
||||
|
||||
Circle()
|
||||
.stroke(seam.opacity(0.18), lineWidth: 2)
|
||||
.frame(width: 320, height: 320)
|
||||
.scaleEffect(self.pulse ? 1.45 : 1.02)
|
||||
.opacity(self.pulse ? 0.0 : 0.9)
|
||||
.animation(.easeOut(duration: 1.9).repeatForever(autoreverses: false).delay(0.2), value: self.pulse)
|
||||
|
||||
Circle()
|
||||
.fill(
|
||||
RadialGradient(
|
||||
colors: [
|
||||
seam.opacity(0.95),
|
||||
seam.opacity(0.40),
|
||||
Color.black.opacity(0.55),
|
||||
],
|
||||
center: .center,
|
||||
startRadius: 1,
|
||||
endRadius: 112))
|
||||
.frame(width: 190, height: 190)
|
||||
.overlay(
|
||||
Circle()
|
||||
.stroke(seam.opacity(0.35), lineWidth: 1))
|
||||
.shadow(color: seam.opacity(0.32), radius: 26, x: 0, y: 0)
|
||||
.shadow(color: Color.black.opacity(0.50), radius: 22, x: 0, y: 10)
|
||||
}
|
||||
.contentShape(Circle())
|
||||
.onTapGesture {
|
||||
self.appModel.talkMode.userTappedOrb()
|
||||
}
|
||||
|
||||
if !status.isEmpty, status != "Off" {
|
||||
Text(status)
|
||||
.font(.system(.footnote, design: .rounded).weight(.semibold))
|
||||
.foregroundStyle(Color.white.opacity(0.92))
|
||||
.padding(.horizontal, 12)
|
||||
.padding(.vertical, 8)
|
||||
.background(
|
||||
Capsule()
|
||||
.fill(Color.black.opacity(0.40))
|
||||
.overlay(
|
||||
Capsule().stroke(seam.opacity(0.22), lineWidth: 1)))
|
||||
}
|
||||
}
|
||||
.padding(28)
|
||||
.onAppear {
|
||||
self.pulse = true
|
||||
}
|
||||
.accessibilityElement(children: .combine)
|
||||
.accessibilityLabel("Talk Mode \(status)")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import SwiftUI
|
||||
|
||||
struct VoiceTab: View {
|
||||
@Environment(NodeAppModel.self) private var appModel
|
||||
@Environment(VoiceWakeManager.self) private var voiceWake
|
||||
@AppStorage("voiceWake.enabled") private var voiceWakeEnabled: Bool = false
|
||||
@AppStorage("talk.enabled") private var talkEnabled: Bool = false
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
List {
|
||||
Section("Status") {
|
||||
LabeledContent("Voice Wake", value: self.voiceWakeEnabled ? "Enabled" : "Disabled")
|
||||
LabeledContent("Listener", value: self.voiceWake.isListening ? "Listening" : "Idle")
|
||||
Text(self.voiceWake.statusText)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
LabeledContent("Talk Mode", value: self.talkEnabled ? "Enabled" : "Disabled")
|
||||
}
|
||||
|
||||
Section("Notes") {
|
||||
let triggers = self.voiceWake.activeTriggerWords
|
||||
Group {
|
||||
if triggers.isEmpty {
|
||||
Text("Add wake words in Settings.")
|
||||
} else if triggers.count == 1 {
|
||||
Text("Say “\(triggers[0]) …” to trigger.")
|
||||
} else if triggers.count == 2 {
|
||||
Text("Say “\(triggers[0]) …” or “\(triggers[1]) …” to trigger.")
|
||||
} else {
|
||||
Text("Say “\(triggers.joined(separator: " …”, “")) …” to trigger.")
|
||||
}
|
||||
}
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
.navigationTitle("Voice")
|
||||
.onChange(of: self.voiceWakeEnabled) { _, newValue in
|
||||
self.appModel.setVoiceWakeEnabled(newValue)
|
||||
}
|
||||
.onChange(of: self.talkEnabled) { _, newValue in
|
||||
self.appModel.setTalkEnabled(newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
import AVFAudio
|
||||
import Foundation
|
||||
import Observation
|
||||
import Speech
|
||||
import SwabbleKit
|
||||
|
||||
private func makeAudioTapEnqueueCallback(queue: AudioBufferQueue) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
|
||||
{ buffer, _ in
|
||||
// This callback is invoked on a realtime audio thread/queue. Keep it tiny and nonisolated.
|
||||
queue.enqueueCopy(of: buffer)
|
||||
}
|
||||
}
|
||||
|
||||
private final class AudioBufferQueue: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var buffers: [AVAudioPCMBuffer] = []
|
||||
|
||||
func enqueueCopy(of buffer: AVAudioPCMBuffer) {
|
||||
guard let copy = buffer.deepCopy() else { return }
|
||||
self.lock.lock()
|
||||
self.buffers.append(copy)
|
||||
self.lock.unlock()
|
||||
}
|
||||
|
||||
func drain() -> [AVAudioPCMBuffer] {
|
||||
self.lock.lock()
|
||||
let drained = self.buffers
|
||||
self.buffers.removeAll(keepingCapacity: true)
|
||||
self.lock.unlock()
|
||||
return drained
|
||||
}
|
||||
|
||||
func clear() {
|
||||
self.lock.lock()
|
||||
self.buffers.removeAll(keepingCapacity: false)
|
||||
self.lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
extension AVAudioPCMBuffer {
|
||||
fileprivate func deepCopy() -> AVAudioPCMBuffer? {
|
||||
let format = self.format
|
||||
let frameLength = self.frameLength
|
||||
guard let copy = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameLength) else {
|
||||
return nil
|
||||
}
|
||||
copy.frameLength = frameLength
|
||||
|
||||
if let src = self.floatChannelData, let dst = copy.floatChannelData {
|
||||
let channels = Int(format.channelCount)
|
||||
let frames = Int(frameLength)
|
||||
for ch in 0..<channels {
|
||||
dst[ch].update(from: src[ch], count: frames)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
if let src = self.int16ChannelData, let dst = copy.int16ChannelData {
|
||||
let channels = Int(format.channelCount)
|
||||
let frames = Int(frameLength)
|
||||
for ch in 0..<channels {
|
||||
dst[ch].update(from: src[ch], count: frames)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
if let src = self.int32ChannelData, let dst = copy.int32ChannelData {
|
||||
let channels = Int(format.channelCount)
|
||||
let frames = Int(frameLength)
|
||||
for ch in 0..<channels {
|
||||
dst[ch].update(from: src[ch], count: frames)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
@Observable
|
||||
final class VoiceWakeManager: NSObject {
|
||||
var isEnabled: Bool = false
|
||||
var isListening: Bool = false
|
||||
var statusText: String = "Off"
|
||||
var triggerWords: [String] = VoiceWakePreferences.loadTriggerWords()
|
||||
var lastTriggeredCommand: String?
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private var speechRecognizer: SFSpeechRecognizer?
|
||||
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
||||
private var recognitionTask: SFSpeechRecognitionTask?
|
||||
private var tapQueue: AudioBufferQueue?
|
||||
private var tapDrainTask: Task<Void, Never>?
|
||||
|
||||
private var lastDispatched: String?
|
||||
private var onCommand: (@Sendable (String) async -> Void)?
|
||||
private var userDefaultsObserver: NSObjectProtocol?
|
||||
|
||||
override init() {
|
||||
super.init()
|
||||
self.triggerWords = VoiceWakePreferences.loadTriggerWords()
|
||||
self.userDefaultsObserver = NotificationCenter.default.addObserver(
|
||||
forName: UserDefaults.didChangeNotification,
|
||||
object: UserDefaults.standard,
|
||||
queue: .main,
|
||||
using: { [weak self] _ in
|
||||
Task { @MainActor in
|
||||
self?.handleUserDefaultsDidChange()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@MainActor deinit {
|
||||
if let userDefaultsObserver = self.userDefaultsObserver {
|
||||
NotificationCenter.default.removeObserver(userDefaultsObserver)
|
||||
}
|
||||
}
|
||||
|
||||
var activeTriggerWords: [String] {
|
||||
VoiceWakePreferences.sanitizeTriggerWords(self.triggerWords)
|
||||
}
|
||||
|
||||
private func handleUserDefaultsDidChange() {
|
||||
let updated = VoiceWakePreferences.loadTriggerWords()
|
||||
if updated != self.triggerWords {
|
||||
self.triggerWords = updated
|
||||
}
|
||||
}
|
||||
|
||||
func configure(onCommand: @escaping @Sendable (String) async -> Void) {
|
||||
self.onCommand = onCommand
|
||||
}
|
||||
|
||||
func setEnabled(_ enabled: Bool) {
|
||||
self.isEnabled = enabled
|
||||
if enabled {
|
||||
Task { await self.start() }
|
||||
} else {
|
||||
self.stop()
|
||||
}
|
||||
}
|
||||
|
||||
func start() async {
|
||||
guard self.isEnabled else { return }
|
||||
if self.isListening { return }
|
||||
|
||||
if ProcessInfo.processInfo.environment["SIMULATOR_DEVICE_NAME"] != nil ||
|
||||
ProcessInfo.processInfo.environment["SIMULATOR_UDID"] != nil
|
||||
{
|
||||
// The iOS Simulator’s audio stack is unreliable for long-running microphone capture.
|
||||
// (We’ve observed CoreAudio deadlocks after TCC permission prompts.)
|
||||
self.isListening = false
|
||||
self.statusText = "Voice Wake isn’t supported on Simulator"
|
||||
return
|
||||
}
|
||||
|
||||
self.statusText = "Requesting permissions…"
|
||||
|
||||
let micOk = await Self.requestMicrophonePermission()
|
||||
guard micOk else {
|
||||
self.statusText = "Microphone permission denied"
|
||||
self.isListening = false
|
||||
return
|
||||
}
|
||||
|
||||
let speechOk = await Self.requestSpeechPermission()
|
||||
guard speechOk else {
|
||||
self.statusText = "Speech recognition permission denied"
|
||||
self.isListening = false
|
||||
return
|
||||
}
|
||||
|
||||
self.speechRecognizer = SFSpeechRecognizer()
|
||||
guard self.speechRecognizer != nil else {
|
||||
self.statusText = "Speech recognizer unavailable"
|
||||
self.isListening = false
|
||||
return
|
||||
}
|
||||
|
||||
do {
|
||||
try Self.configureAudioSession()
|
||||
try self.startRecognition()
|
||||
self.isListening = true
|
||||
self.statusText = "Listening"
|
||||
} catch {
|
||||
self.isListening = false
|
||||
self.statusText = "Start failed: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
self.isEnabled = false
|
||||
self.isListening = false
|
||||
self.statusText = "Off"
|
||||
|
||||
self.tapDrainTask?.cancel()
|
||||
self.tapDrainTask = nil
|
||||
self.tapQueue?.clear()
|
||||
self.tapQueue = nil
|
||||
|
||||
self.recognitionTask?.cancel()
|
||||
self.recognitionTask = nil
|
||||
self.recognitionRequest = nil
|
||||
|
||||
if self.audioEngine.isRunning {
|
||||
self.audioEngine.stop()
|
||||
self.audioEngine.inputNode.removeTap(onBus: 0)
|
||||
}
|
||||
|
||||
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||
}
|
||||
|
||||
/// Temporarily releases the microphone so other subsystems (e.g. camera video capture) can record audio.
|
||||
/// Returns `true` when listening was active and was suspended.
|
||||
func suspendForExternalAudioCapture() -> Bool {
|
||||
guard self.isEnabled, self.isListening else { return false }
|
||||
|
||||
self.isListening = false
|
||||
self.statusText = "Paused"
|
||||
|
||||
self.tapDrainTask?.cancel()
|
||||
self.tapDrainTask = nil
|
||||
self.tapQueue?.clear()
|
||||
self.tapQueue = nil
|
||||
|
||||
self.recognitionTask?.cancel()
|
||||
self.recognitionTask = nil
|
||||
self.recognitionRequest = nil
|
||||
|
||||
if self.audioEngine.isRunning {
|
||||
self.audioEngine.stop()
|
||||
self.audioEngine.inputNode.removeTap(onBus: 0)
|
||||
}
|
||||
|
||||
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||
return true
|
||||
}
|
||||
|
||||
func resumeAfterExternalAudioCapture(wasSuspended: Bool) {
|
||||
guard wasSuspended else { return }
|
||||
Task { await self.start() }
|
||||
}
|
||||
|
||||
private func startRecognition() throws {
|
||||
self.recognitionTask?.cancel()
|
||||
self.recognitionTask = nil
|
||||
self.tapDrainTask?.cancel()
|
||||
self.tapDrainTask = nil
|
||||
self.tapQueue?.clear()
|
||||
self.tapQueue = nil
|
||||
|
||||
let request = SFSpeechAudioBufferRecognitionRequest()
|
||||
request.shouldReportPartialResults = true
|
||||
self.recognitionRequest = request
|
||||
|
||||
let inputNode = self.audioEngine.inputNode
|
||||
inputNode.removeTap(onBus: 0)
|
||||
|
||||
let recordingFormat = inputNode.outputFormat(forBus: 0)
|
||||
|
||||
let queue = AudioBufferQueue()
|
||||
self.tapQueue = queue
|
||||
let tapBlock: @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void = makeAudioTapEnqueueCallback(queue: queue)
|
||||
inputNode.installTap(
|
||||
onBus: 0,
|
||||
bufferSize: 1024,
|
||||
format: recordingFormat,
|
||||
block: tapBlock)
|
||||
|
||||
self.audioEngine.prepare()
|
||||
try self.audioEngine.start()
|
||||
|
||||
let handler = self.makeRecognitionResultHandler()
|
||||
self.recognitionTask = self.speechRecognizer?.recognitionTask(with: request, resultHandler: handler)
|
||||
|
||||
self.tapDrainTask = Task { [weak self] in
|
||||
guard let self, let queue = self.tapQueue else { return }
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(nanoseconds: 40_000_000)
|
||||
let drained = queue.drain()
|
||||
if drained.isEmpty { continue }
|
||||
for buf in drained {
|
||||
request.append(buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated func makeRecognitionResultHandler() -> @Sendable (SFSpeechRecognitionResult?, Error?) -> Void {
|
||||
{ [weak self] result, error in
|
||||
let transcript = result?.bestTranscription.formattedString
|
||||
let segments = result.flatMap { result in
|
||||
transcript.map { WakeWordSpeechSegments.from(transcription: result.bestTranscription, transcript: $0) }
|
||||
} ?? []
|
||||
let errorText = error?.localizedDescription
|
||||
|
||||
Task { @MainActor in
|
||||
self?.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
|
||||
if let errorText {
|
||||
self.statusText = "Recognizer error: \(errorText)"
|
||||
self.isListening = false
|
||||
|
||||
let shouldRestart = self.isEnabled
|
||||
if shouldRestart {
|
||||
Task {
|
||||
try? await Task.sleep(nanoseconds: 700_000_000)
|
||||
await self.start()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
guard let transcript else { return }
|
||||
guard let cmd = self.extractCommand(from: transcript, segments: segments) else { return }
|
||||
|
||||
if cmd == self.lastDispatched { return }
|
||||
self.lastDispatched = cmd
|
||||
self.lastTriggeredCommand = cmd
|
||||
self.statusText = "Triggered"
|
||||
|
||||
Task { [weak self] in
|
||||
guard let self else { return }
|
||||
await self.onCommand?(cmd)
|
||||
await self.startIfEnabled()
|
||||
}
|
||||
}
|
||||
|
||||
private func startIfEnabled() async {
|
||||
let shouldRestart = self.isEnabled
|
||||
if shouldRestart {
|
||||
await self.start()
|
||||
}
|
||||
}
|
||||
|
||||
private func extractCommand(from transcript: String, segments: [WakeWordSegment]) -> String? {
|
||||
Self.extractCommand(from: transcript, segments: segments, triggers: self.activeTriggerWords)
|
||||
}
|
||||
|
||||
nonisolated static func extractCommand(
|
||||
from transcript: String,
|
||||
segments: [WakeWordSegment],
|
||||
triggers: [String],
|
||||
minPostTriggerGap: TimeInterval = 0.45) -> String?
|
||||
{
|
||||
let config = WakeWordGateConfig(triggers: triggers, minPostTriggerGap: minPostTriggerGap)
|
||||
return WakeWordGate.match(transcript: transcript, segments: segments, config: config)?.command
|
||||
}
|
||||
|
||||
private static func configureAudioSession() throws {
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try session.setCategory(.playAndRecord, mode: .measurement, options: [
|
||||
.duckOthers,
|
||||
.mixWithOthers,
|
||||
.allowBluetoothHFP,
|
||||
.defaultToSpeaker,
|
||||
])
|
||||
try session.setActive(true, options: [])
|
||||
}
|
||||
|
||||
private nonisolated static func requestMicrophonePermission() async -> Bool {
|
||||
await withCheckedContinuation(isolation: nil) { cont in
|
||||
AVAudioApplication.requestRecordPermission { ok in
|
||||
cont.resume(returning: ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private nonisolated static func requestSpeechPermission() async -> Bool {
|
||||
await withCheckedContinuation(isolation: nil) { cont in
|
||||
SFSpeechRecognizer.requestAuthorization { status in
|
||||
cont.resume(returning: status == .authorized)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
extension VoiceWakeManager {
|
||||
func _test_handleRecognitionCallback(transcript: String?, segments: [WakeWordSegment], errorText: String?) {
|
||||
self.handleRecognitionCallback(transcript: transcript, segments: segments, errorText: errorText)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,44 @@
|
||||
import Foundation
|
||||
|
||||
enum VoiceWakePreferences {
|
||||
static let enabledKey = "voiceWake.enabled"
|
||||
static let triggerWordsKey = "voiceWake.triggerWords"
|
||||
|
||||
// Keep defaults aligned with the mac app.
|
||||
static let defaultTriggerWords: [String] = ["clawd", "claude"]
|
||||
static let maxWords = 32
|
||||
static let maxWordLength = 64
|
||||
|
||||
static func decodeGatewayTriggers(from payloadJSON: String) -> [String]? {
|
||||
guard let data = payloadJSON.data(using: .utf8) else { return nil }
|
||||
return self.decodeGatewayTriggers(from: data)
|
||||
}
|
||||
|
||||
static func decodeGatewayTriggers(from data: Data) -> [String]? {
|
||||
struct Payload: Decodable { var triggers: [String] }
|
||||
guard let decoded = try? JSONDecoder().decode(Payload.self, from: data) else { return nil }
|
||||
return self.sanitizeTriggerWords(decoded.triggers)
|
||||
}
|
||||
|
||||
static func loadTriggerWords(defaults: UserDefaults = .standard) -> [String] {
|
||||
defaults.stringArray(forKey: self.triggerWordsKey) ?? self.defaultTriggerWords
|
||||
}
|
||||
|
||||
static func saveTriggerWords(_ words: [String], defaults: UserDefaults = .standard) {
|
||||
defaults.set(words, forKey: self.triggerWordsKey)
|
||||
}
|
||||
|
||||
static func sanitizeTriggerWords(_ words: [String]) -> [String] {
|
||||
let cleaned = words
|
||||
.map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
|
||||
.filter { !$0.isEmpty }
|
||||
.prefix(Self.maxWords)
|
||||
.map { String($0.prefix(Self.maxWordLength)) }
|
||||
return cleaned.isEmpty ? Self.defaultTriggerWords : cleaned
|
||||
}
|
||||
|
||||
static func displayString(for words: [String]) -> String {
|
||||
let sanitized = self.sanitizeTriggerWords(words)
|
||||
return sanitized.joined(separator: ", ")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
Sources/Gateway/GatewayConnectionController.swift
|
||||
Sources/Gateway/GatewayDiscoveryDebugLogView.swift
|
||||
Sources/Gateway/GatewayDiscoveryModel.swift
|
||||
Sources/Gateway/GatewaySettingsStore.swift
|
||||
Sources/Gateway/KeychainStore.swift
|
||||
Sources/Camera/CameraController.swift
|
||||
Sources/Chat/ChatSheet.swift
|
||||
Sources/Chat/IOSGatewayChatTransport.swift
|
||||
Sources/ClawdbotApp.swift
|
||||
Sources/Location/LocationService.swift
|
||||
Sources/Model/NodeAppModel.swift
|
||||
Sources/RootCanvas.swift
|
||||
Sources/RootTabs.swift
|
||||
Sources/Screen/ScreenController.swift
|
||||
Sources/Screen/ScreenRecordService.swift
|
||||
Sources/Screen/ScreenTab.swift
|
||||
Sources/Screen/ScreenWebView.swift
|
||||
Sources/SessionKey.swift
|
||||
Sources/Settings/SettingsNetworkingHelpers.swift
|
||||
Sources/Settings/SettingsTab.swift
|
||||
Sources/Settings/VoiceWakeWordsSettingsView.swift
|
||||
Sources/Status/StatusPill.swift
|
||||
Sources/Status/VoiceWakeToast.swift
|
||||
Sources/Voice/VoiceTab.swift
|
||||
Sources/Voice/VoiceWakeManager.swift
|
||||
Sources/Voice/VoiceWakePreferences.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatComposer.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatMarkdownRenderer.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatMarkdownPreprocessor.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatMessageViews.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatModels.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatPayloadDecoding.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatSessions.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatSheets.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatTheme.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatTransport.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatView.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotChatUI/ChatViewModel.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/AnyCodable.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/BonjourEscapes.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/BonjourTypes.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/BridgeFrames.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/CameraCommands.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/CanvasA2UIAction.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/CanvasA2UICommands.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/CanvasA2UIJSONL.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/CanvasCommandParams.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/CanvasCommands.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/Capabilities.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/ClawdbotKitResources.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/DeepLinks.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/JPEGTranscoder.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/NodeError.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/ScreenCommands.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/StoragePaths.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/SystemCommands.swift
|
||||
../shared/MoltbotKit/Sources/MoltbotKit/TalkDirective.swift
|
||||
../../Swabble/Sources/SwabbleKit/WakeWordGate.swift
|
||||
Sources/Voice/TalkModeManager.swift
|
||||
Sources/Voice/TalkOrbOverlay.swift
|
||||
@@ -0,0 +1,31 @@
|
||||
import SwiftUI
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct AppCoverageTests {
|
||||
@Test @MainActor func nodeAppModelUpdatesBackgroundedState() {
|
||||
let appModel = NodeAppModel()
|
||||
|
||||
appModel.setScenePhase(.background)
|
||||
#expect(appModel.isBackgrounded == true)
|
||||
|
||||
appModel.setScenePhase(.inactive)
|
||||
#expect(appModel.isBackgrounded == false)
|
||||
|
||||
appModel.setScenePhase(.active)
|
||||
#expect(appModel.isBackgrounded == false)
|
||||
}
|
||||
|
||||
@Test @MainActor func voiceWakeStartReportsUnsupportedOnSimulator() async {
|
||||
let voiceWake = VoiceWakeManager()
|
||||
voiceWake.isEnabled = true
|
||||
|
||||
await voiceWake.start()
|
||||
|
||||
#expect(voiceWake.isListening == false)
|
||||
#expect(voiceWake.statusText.contains("Simulator"))
|
||||
|
||||
voiceWake.stop()
|
||||
#expect(voiceWake.statusText == "Off")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct CameraControllerClampTests {
|
||||
@Test func clampQualityDefaultsAndBounds() {
|
||||
#expect(CameraController.clampQuality(nil) == 0.9)
|
||||
#expect(CameraController.clampQuality(0.0) == 0.05)
|
||||
#expect(CameraController.clampQuality(0.049) == 0.05)
|
||||
#expect(CameraController.clampQuality(0.05) == 0.05)
|
||||
#expect(CameraController.clampQuality(0.5) == 0.5)
|
||||
#expect(CameraController.clampQuality(1.0) == 1.0)
|
||||
#expect(CameraController.clampQuality(1.1) == 1.0)
|
||||
}
|
||||
|
||||
@Test func clampDurationDefaultsAndBounds() {
|
||||
#expect(CameraController.clampDurationMs(nil) == 3000)
|
||||
#expect(CameraController.clampDurationMs(0) == 250)
|
||||
#expect(CameraController.clampDurationMs(249) == 250)
|
||||
#expect(CameraController.clampDurationMs(250) == 250)
|
||||
#expect(CameraController.clampDurationMs(1000) == 1000)
|
||||
#expect(CameraController.clampDurationMs(60000) == 60000)
|
||||
#expect(CameraController.clampDurationMs(60001) == 60000)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct CameraControllerErrorTests {
|
||||
@Test func errorDescriptionsAreStable() {
|
||||
#expect(CameraController.CameraError.cameraUnavailable.errorDescription == "Camera unavailable")
|
||||
#expect(CameraController.CameraError.microphoneUnavailable.errorDescription == "Microphone unavailable")
|
||||
#expect(CameraController.CameraError.permissionDenied(kind: "Camera")
|
||||
.errorDescription == "Camera permission denied")
|
||||
#expect(CameraController.CameraError.invalidParams("bad").errorDescription == "bad")
|
||||
#expect(CameraController.CameraError.captureFailed("nope").errorDescription == "nope")
|
||||
#expect(CameraController.CameraError.exportFailed("export").errorDescription == "export")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import MoltbotKit
|
||||
import Foundation
|
||||
import Testing
|
||||
|
||||
@Suite struct DeepLinkParserTests {
|
||||
@Test func parseRejectsUnknownHost() {
|
||||
let url = URL(string: "moltbot://nope?message=hi")!
|
||||
#expect(DeepLinkParser.parse(url) == nil)
|
||||
}
|
||||
|
||||
@Test func parseHostIsCaseInsensitive() {
|
||||
let url = URL(string: "moltbot://AGENT?message=Hello")!
|
||||
#expect(DeepLinkParser.parse(url) == .agent(.init(
|
||||
message: "Hello",
|
||||
sessionKey: nil,
|
||||
thinking: nil,
|
||||
deliver: false,
|
||||
to: nil,
|
||||
channel: nil,
|
||||
timeoutSeconds: nil,
|
||||
key: nil)))
|
||||
}
|
||||
|
||||
@Test func parseRejectsNonMoltbotScheme() {
|
||||
let url = URL(string: "https://example.com/agent?message=hi")!
|
||||
#expect(DeepLinkParser.parse(url) == nil)
|
||||
}
|
||||
|
||||
@Test func parseRejectsEmptyMessage() {
|
||||
let url = URL(string: "moltbot://agent?message=%20%20%0A")!
|
||||
#expect(DeepLinkParser.parse(url) == nil)
|
||||
}
|
||||
|
||||
@Test func parseAgentLinkParsesCommonFields() {
|
||||
let url =
|
||||
URL(string: "moltbot://agent?message=Hello&deliver=1&sessionKey=node-test&thinking=low&timeoutSeconds=30")!
|
||||
#expect(
|
||||
DeepLinkParser.parse(url) == .agent(
|
||||
.init(
|
||||
message: "Hello",
|
||||
sessionKey: "node-test",
|
||||
thinking: "low",
|
||||
deliver: true,
|
||||
to: nil,
|
||||
channel: nil,
|
||||
timeoutSeconds: 30,
|
||||
key: nil)))
|
||||
}
|
||||
|
||||
@Test func parseAgentLinkParsesTargetRoutingFields() {
|
||||
let url =
|
||||
URL(
|
||||
string: "moltbot://agent?message=Hello%20World&deliver=1&to=%2B15551234567&channel=whatsapp&key=secret")!
|
||||
#expect(
|
||||
DeepLinkParser.parse(url) == .agent(
|
||||
.init(
|
||||
message: "Hello World",
|
||||
sessionKey: nil,
|
||||
thinking: nil,
|
||||
deliver: true,
|
||||
to: "+15551234567",
|
||||
channel: "whatsapp",
|
||||
timeoutSeconds: nil,
|
||||
key: "secret")))
|
||||
}
|
||||
|
||||
@Test func parseRejectsNegativeTimeoutSeconds() {
|
||||
let url = URL(string: "moltbot://agent?message=Hello&timeoutSeconds=-1")!
|
||||
#expect(DeepLinkParser.parse(url) == .agent(.init(
|
||||
message: "Hello",
|
||||
sessionKey: nil,
|
||||
thinking: nil,
|
||||
deliver: false,
|
||||
to: nil,
|
||||
channel: nil,
|
||||
timeoutSeconds: nil,
|
||||
key: nil)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import MoltbotKit
|
||||
import Foundation
|
||||
import Testing
|
||||
import UIKit
|
||||
@testable import Moltbot
|
||||
|
||||
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
|
||||
let defaults = UserDefaults.standard
|
||||
var snapshot: [String: Any?] = [:]
|
||||
for key in updates.keys {
|
||||
snapshot[key] = defaults.object(forKey: key)
|
||||
}
|
||||
for (key, value) in updates {
|
||||
if let value {
|
||||
defaults.set(value, forKey: key)
|
||||
} else {
|
||||
defaults.removeObject(forKey: key)
|
||||
}
|
||||
}
|
||||
defer {
|
||||
for (key, value) in snapshot {
|
||||
if let value {
|
||||
defaults.set(value, forKey: key)
|
||||
} else {
|
||||
defaults.removeObject(forKey: key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return try body()
|
||||
}
|
||||
|
||||
@Suite(.serialized) struct GatewayConnectionControllerTests {
|
||||
@Test @MainActor func resolvedDisplayNameSetsDefaultWhenMissing() {
|
||||
let defaults = UserDefaults.standard
|
||||
let displayKey = "node.displayName"
|
||||
|
||||
withUserDefaults([displayKey: nil, "node.instanceId": "ios-test"]) {
|
||||
let appModel = NodeAppModel()
|
||||
let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false)
|
||||
|
||||
let resolved = controller._test_resolvedDisplayName(defaults: defaults)
|
||||
#expect(!resolved.isEmpty)
|
||||
#expect(defaults.string(forKey: displayKey) == resolved)
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func currentCapsReflectToggles() {
|
||||
withUserDefaults([
|
||||
"node.instanceId": "ios-test",
|
||||
"node.displayName": "Test Node",
|
||||
"camera.enabled": true,
|
||||
"location.enabledMode": MoltbotLocationMode.always.rawValue,
|
||||
VoiceWakePreferences.enabledKey: true,
|
||||
]) {
|
||||
let appModel = NodeAppModel()
|
||||
let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false)
|
||||
let caps = Set(controller._test_currentCaps())
|
||||
|
||||
#expect(caps.contains(MoltbotCapability.canvas.rawValue))
|
||||
#expect(caps.contains(MoltbotCapability.screen.rawValue))
|
||||
#expect(caps.contains(MoltbotCapability.camera.rawValue))
|
||||
#expect(caps.contains(MoltbotCapability.location.rawValue))
|
||||
#expect(caps.contains(MoltbotCapability.voiceWake.rawValue))
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func currentCommandsIncludeLocationWhenEnabled() {
|
||||
withUserDefaults([
|
||||
"node.instanceId": "ios-test",
|
||||
"location.enabledMode": MoltbotLocationMode.whileUsing.rawValue,
|
||||
]) {
|
||||
let appModel = NodeAppModel()
|
||||
let controller = GatewayConnectionController(appModel: appModel, startDiscovery: false)
|
||||
let commands = Set(controller._test_currentCommands())
|
||||
|
||||
#expect(commands.contains(MoltbotLocationCommand.get.rawValue))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite(.serialized) struct GatewayDiscoveryModelTests {
|
||||
@Test @MainActor func debugLoggingCapturesLifecycleAndResets() {
|
||||
let model = GatewayDiscoveryModel()
|
||||
|
||||
#expect(model.debugLog.isEmpty)
|
||||
#expect(model.statusText == "Idle")
|
||||
|
||||
model.setDebugLoggingEnabled(true)
|
||||
#expect(model.debugLog.count >= 2)
|
||||
|
||||
model.stop()
|
||||
#expect(model.statusText == "Stopped")
|
||||
#expect(model.gateways.isEmpty)
|
||||
#expect(model.debugLog.count >= 3)
|
||||
|
||||
model.setDebugLoggingEnabled(false)
|
||||
#expect(model.debugLog.isEmpty)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import MoltbotKit
|
||||
import Network
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct GatewayEndpointIDTests {
|
||||
@Test func stableIDForServiceDecodesAndNormalizesName() {
|
||||
let endpoint = NWEndpoint.service(
|
||||
name: "Moltbot\\032Gateway \\032 Node\n",
|
||||
type: "_moltbot-gw._tcp",
|
||||
domain: "local.",
|
||||
interface: nil)
|
||||
|
||||
#expect(GatewayEndpointID.stableID(endpoint) == "_moltbot-gw._tcp|local.|Moltbot Gateway Node")
|
||||
}
|
||||
|
||||
@Test func stableIDForNonServiceUsesEndpointDescription() {
|
||||
let endpoint = NWEndpoint.hostPort(host: NWEndpoint.Host("127.0.0.1"), port: 4242)
|
||||
#expect(GatewayEndpointID.stableID(endpoint) == String(describing: endpoint))
|
||||
}
|
||||
|
||||
@Test func prettyDescriptionDecodesBonjourEscapes() {
|
||||
let endpoint = NWEndpoint.service(
|
||||
name: "Moltbot\\032Gateway",
|
||||
type: "_moltbot-gw._tcp",
|
||||
domain: "local.",
|
||||
interface: nil)
|
||||
|
||||
let pretty = GatewayEndpointID.prettyDescription(endpoint)
|
||||
#expect(pretty == BonjourEscapes.decode(String(describing: endpoint)))
|
||||
#expect(!pretty.localizedCaseInsensitiveContains("\\032"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
private struct KeychainEntry: Hashable {
|
||||
let service: String
|
||||
let account: String
|
||||
}
|
||||
|
||||
private let gatewayService = "bot.molt.gateway"
|
||||
private let nodeService = "bot.molt.node"
|
||||
private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId")
|
||||
private let preferredGatewayEntry = KeychainEntry(service: gatewayService, account: "preferredStableID")
|
||||
private let lastGatewayEntry = KeychainEntry(service: gatewayService, account: "lastDiscoveredStableID")
|
||||
|
||||
private func snapshotDefaults(_ keys: [String]) -> [String: Any?] {
|
||||
let defaults = UserDefaults.standard
|
||||
var snapshot: [String: Any?] = [:]
|
||||
for key in keys {
|
||||
snapshot[key] = defaults.object(forKey: key)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
private func applyDefaults(_ values: [String: Any?]) {
|
||||
let defaults = UserDefaults.standard
|
||||
for (key, value) in values {
|
||||
if let value {
|
||||
defaults.set(value, forKey: key)
|
||||
} else {
|
||||
defaults.removeObject(forKey: key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func restoreDefaults(_ snapshot: [String: Any?]) {
|
||||
applyDefaults(snapshot)
|
||||
}
|
||||
|
||||
private func snapshotKeychain(_ entries: [KeychainEntry]) -> [KeychainEntry: String?] {
|
||||
var snapshot: [KeychainEntry: String?] = [:]
|
||||
for entry in entries {
|
||||
snapshot[entry] = KeychainStore.loadString(service: entry.service, account: entry.account)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
private func applyKeychain(_ values: [KeychainEntry: String?]) {
|
||||
for (entry, value) in values {
|
||||
if let value {
|
||||
_ = KeychainStore.saveString(value, service: entry.service, account: entry.account)
|
||||
} else {
|
||||
_ = KeychainStore.delete(service: entry.service, account: entry.account)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func restoreKeychain(_ snapshot: [KeychainEntry: String?]) {
|
||||
applyKeychain(snapshot)
|
||||
}
|
||||
|
||||
@Suite(.serialized) struct GatewaySettingsStoreTests {
|
||||
@Test func bootstrapCopiesDefaultsToKeychainWhenMissing() {
|
||||
let defaultsKeys = [
|
||||
"node.instanceId",
|
||||
"gateway.preferredStableID",
|
||||
"gateway.lastDiscoveredStableID",
|
||||
]
|
||||
let entries = [instanceIdEntry, preferredGatewayEntry, lastGatewayEntry]
|
||||
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
|
||||
let keychainSnapshot = snapshotKeychain(entries)
|
||||
defer {
|
||||
restoreDefaults(defaultsSnapshot)
|
||||
restoreKeychain(keychainSnapshot)
|
||||
}
|
||||
|
||||
applyDefaults([
|
||||
"node.instanceId": "node-test",
|
||||
"gateway.preferredStableID": "preferred-test",
|
||||
"gateway.lastDiscoveredStableID": "last-test",
|
||||
])
|
||||
applyKeychain([
|
||||
instanceIdEntry: nil,
|
||||
preferredGatewayEntry: nil,
|
||||
lastGatewayEntry: nil,
|
||||
])
|
||||
|
||||
GatewaySettingsStore.bootstrapPersistence()
|
||||
|
||||
#expect(KeychainStore.loadString(service: nodeService, account: "instanceId") == "node-test")
|
||||
#expect(KeychainStore.loadString(service: gatewayService, account: "preferredStableID") == "preferred-test")
|
||||
#expect(KeychainStore.loadString(service: gatewayService, account: "lastDiscoveredStableID") == "last-test")
|
||||
}
|
||||
|
||||
@Test func bootstrapCopiesKeychainToDefaultsWhenMissing() {
|
||||
let defaultsKeys = [
|
||||
"node.instanceId",
|
||||
"gateway.preferredStableID",
|
||||
"gateway.lastDiscoveredStableID",
|
||||
]
|
||||
let entries = [instanceIdEntry, preferredGatewayEntry, lastGatewayEntry]
|
||||
let defaultsSnapshot = snapshotDefaults(defaultsKeys)
|
||||
let keychainSnapshot = snapshotKeychain(entries)
|
||||
defer {
|
||||
restoreDefaults(defaultsSnapshot)
|
||||
restoreKeychain(keychainSnapshot)
|
||||
}
|
||||
|
||||
applyDefaults([
|
||||
"node.instanceId": nil,
|
||||
"gateway.preferredStableID": nil,
|
||||
"gateway.lastDiscoveredStableID": nil,
|
||||
])
|
||||
applyKeychain([
|
||||
instanceIdEntry: "node-from-keychain",
|
||||
preferredGatewayEntry: "preferred-from-keychain",
|
||||
lastGatewayEntry: "last-from-keychain",
|
||||
])
|
||||
|
||||
GatewaySettingsStore.bootstrapPersistence()
|
||||
|
||||
let defaults = UserDefaults.standard
|
||||
#expect(defaults.string(forKey: "node.instanceId") == "node-from-keychain")
|
||||
#expect(defaults.string(forKey: "gateway.preferredStableID") == "preferred-from-keychain")
|
||||
#expect(defaults.string(forKey: "gateway.lastDiscoveredStableID") == "last-from-keychain")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import MoltbotKit
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct IOSGatewayChatTransportTests {
|
||||
@Test func requestsFailFastWhenGatewayNotConnected() async {
|
||||
let gateway = GatewayNodeSession()
|
||||
let transport = IOSGatewayChatTransport(gateway: gateway)
|
||||
|
||||
do {
|
||||
_ = try await transport.requestHistory(sessionKey: "node-test")
|
||||
Issue.record("Expected requestHistory to throw when gateway not connected")
|
||||
} catch {}
|
||||
|
||||
do {
|
||||
_ = try await transport.sendMessage(
|
||||
sessionKey: "node-test",
|
||||
message: "hello",
|
||||
thinking: "low",
|
||||
idempotencyKey: "idempotency",
|
||||
attachments: [])
|
||||
Issue.record("Expected sendMessage to throw when gateway not connected")
|
||||
} catch {}
|
||||
|
||||
do {
|
||||
_ = try await transport.requestHealth(timeoutMs: 250)
|
||||
Issue.record("Expected requestHealth to throw when gateway not connected")
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
24
docker-compose/ez-assistant/apps/ios/Tests/Info.plist
Normal file
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDevelopmentRegion</key>
|
||||
<string>$(DEVELOPMENT_LANGUAGE)</string>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>MoltbotTests</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>BNDL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>2026.1.26</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>20260126</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,22 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct KeychainStoreTests {
|
||||
@Test func saveLoadUpdateDeleteRoundTrip() {
|
||||
let service = "bot.molt.tests.\(UUID().uuidString)"
|
||||
let account = "value"
|
||||
|
||||
#expect(KeychainStore.delete(service: service, account: account))
|
||||
#expect(KeychainStore.loadString(service: service, account: account) == nil)
|
||||
|
||||
#expect(KeychainStore.saveString("first", service: service, account: account))
|
||||
#expect(KeychainStore.loadString(service: service, account: account) == "first")
|
||||
|
||||
#expect(KeychainStore.saveString("second", service: service, account: account))
|
||||
#expect(KeychainStore.loadString(service: service, account: account) == "second")
|
||||
|
||||
#expect(KeychainStore.delete(service: service, account: account))
|
||||
#expect(KeychainStore.loadString(service: service, account: account) == nil)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import MoltbotKit
|
||||
import Foundation
|
||||
import Testing
|
||||
import UIKit
|
||||
@testable import Moltbot
|
||||
|
||||
private func withUserDefaults<T>(_ updates: [String: Any?], _ body: () throws -> T) rethrows -> T {
|
||||
let defaults = UserDefaults.standard
|
||||
var snapshot: [String: Any?] = [:]
|
||||
for key in updates.keys {
|
||||
snapshot[key] = defaults.object(forKey: key)
|
||||
}
|
||||
for (key, value) in updates {
|
||||
if let value {
|
||||
defaults.set(value, forKey: key)
|
||||
} else {
|
||||
defaults.removeObject(forKey: key)
|
||||
}
|
||||
}
|
||||
defer {
|
||||
for (key, value) in snapshot {
|
||||
if let value {
|
||||
defaults.set(value, forKey: key)
|
||||
} else {
|
||||
defaults.removeObject(forKey: key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return try body()
|
||||
}
|
||||
|
||||
@Suite(.serialized) struct NodeAppModelInvokeTests {
|
||||
@Test @MainActor func decodeParamsFailsWithoutJSON() {
|
||||
#expect(throws: Error.self) {
|
||||
_ = try NodeAppModel._test_decodeParams(MoltbotCanvasNavigateParams.self, from: nil)
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func encodePayloadEmitsJSON() throws {
|
||||
struct Payload: Codable, Equatable {
|
||||
var value: String
|
||||
}
|
||||
let json = try NodeAppModel._test_encodePayload(Payload(value: "ok"))
|
||||
#expect(json.contains("\"value\""))
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeRejectsBackgroundCommands() async {
|
||||
let appModel = NodeAppModel()
|
||||
appModel.setScenePhase(.background)
|
||||
|
||||
let req = BridgeInvokeRequest(id: "bg", command: MoltbotCanvasCommand.present.rawValue)
|
||||
let res = await appModel._test_handleInvoke(req)
|
||||
#expect(res.ok == false)
|
||||
#expect(res.error?.code == .backgroundUnavailable)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeRejectsCameraWhenDisabled() async {
|
||||
let appModel = NodeAppModel()
|
||||
let req = BridgeInvokeRequest(id: "cam", command: MoltbotCameraCommand.snap.rawValue)
|
||||
|
||||
let defaults = UserDefaults.standard
|
||||
let key = "camera.enabled"
|
||||
let previous = defaults.object(forKey: key)
|
||||
defaults.set(false, forKey: key)
|
||||
defer {
|
||||
if let previous {
|
||||
defaults.set(previous, forKey: key)
|
||||
} else {
|
||||
defaults.removeObject(forKey: key)
|
||||
}
|
||||
}
|
||||
|
||||
let res = await appModel._test_handleInvoke(req)
|
||||
#expect(res.ok == false)
|
||||
#expect(res.error?.code == .unavailable)
|
||||
#expect(res.error?.message.contains("CAMERA_DISABLED") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeRejectsInvalidScreenFormat() async {
|
||||
let appModel = NodeAppModel()
|
||||
let params = MoltbotScreenRecordParams(format: "gif")
|
||||
let data = try? JSONEncoder().encode(params)
|
||||
let json = data.flatMap { String(data: $0, encoding: .utf8) }
|
||||
|
||||
let req = BridgeInvokeRequest(
|
||||
id: "screen",
|
||||
command: MoltbotScreenCommand.record.rawValue,
|
||||
paramsJSON: json)
|
||||
|
||||
let res = await appModel._test_handleInvoke(req)
|
||||
#expect(res.ok == false)
|
||||
#expect(res.error?.message.contains("screen format must be mp4") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeCanvasCommandsUpdateScreen() async throws {
|
||||
let appModel = NodeAppModel()
|
||||
appModel.screen.navigate(to: "http://example.com")
|
||||
|
||||
let present = BridgeInvokeRequest(id: "present", command: MoltbotCanvasCommand.present.rawValue)
|
||||
let presentRes = await appModel._test_handleInvoke(present)
|
||||
#expect(presentRes.ok == true)
|
||||
#expect(appModel.screen.urlString.isEmpty)
|
||||
|
||||
let navigateParams = MoltbotCanvasNavigateParams(url: "http://localhost:18789/")
|
||||
let navData = try JSONEncoder().encode(navigateParams)
|
||||
let navJSON = String(decoding: navData, as: UTF8.self)
|
||||
let navigate = BridgeInvokeRequest(
|
||||
id: "nav",
|
||||
command: MoltbotCanvasCommand.navigate.rawValue,
|
||||
paramsJSON: navJSON)
|
||||
let navRes = await appModel._test_handleInvoke(navigate)
|
||||
#expect(navRes.ok == true)
|
||||
#expect(appModel.screen.urlString == "http://localhost:18789/")
|
||||
|
||||
let evalParams = MoltbotCanvasEvalParams(javaScript: "1+1")
|
||||
let evalData = try JSONEncoder().encode(evalParams)
|
||||
let evalJSON = String(decoding: evalData, as: UTF8.self)
|
||||
let eval = BridgeInvokeRequest(
|
||||
id: "eval",
|
||||
command: MoltbotCanvasCommand.evalJS.rawValue,
|
||||
paramsJSON: evalJSON)
|
||||
let evalRes = await appModel._test_handleInvoke(eval)
|
||||
#expect(evalRes.ok == true)
|
||||
let payloadData = try #require(evalRes.payloadJSON?.data(using: .utf8))
|
||||
let payload = try JSONSerialization.jsonObject(with: payloadData) as? [String: Any]
|
||||
#expect(payload?["result"] as? String == "2")
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeA2UICommandsFailWhenHostMissing() async throws {
|
||||
let appModel = NodeAppModel()
|
||||
|
||||
let reset = BridgeInvokeRequest(id: "reset", command: MoltbotCanvasA2UICommand.reset.rawValue)
|
||||
let resetRes = await appModel._test_handleInvoke(reset)
|
||||
#expect(resetRes.ok == false)
|
||||
#expect(resetRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
|
||||
|
||||
let jsonl = "{\"beginRendering\":{}}"
|
||||
let pushParams = MoltbotCanvasA2UIPushJSONLParams(jsonl: jsonl)
|
||||
let pushData = try JSONEncoder().encode(pushParams)
|
||||
let pushJSON = String(decoding: pushData, as: UTF8.self)
|
||||
let push = BridgeInvokeRequest(
|
||||
id: "push",
|
||||
command: MoltbotCanvasA2UICommand.pushJSONL.rawValue,
|
||||
paramsJSON: pushJSON)
|
||||
let pushRes = await appModel._test_handleInvoke(push)
|
||||
#expect(pushRes.ok == false)
|
||||
#expect(pushRes.error?.message.contains("A2UI_HOST_NOT_CONFIGURED") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleInvokeUnknownCommandReturnsInvalidRequest() async {
|
||||
let appModel = NodeAppModel()
|
||||
let req = BridgeInvokeRequest(id: "unknown", command: "nope")
|
||||
let res = await appModel._test_handleInvoke(req)
|
||||
#expect(res.ok == false)
|
||||
#expect(res.error?.code == .invalidRequest)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleDeepLinkSetsErrorWhenNotConnected() async {
|
||||
let appModel = NodeAppModel()
|
||||
let url = URL(string: "moltbot://agent?message=hello")!
|
||||
await appModel.handleDeepLink(url: url)
|
||||
#expect(appModel.screen.errorText?.contains("Gateway not connected") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleDeepLinkRejectsOversizedMessage() async {
|
||||
let appModel = NodeAppModel()
|
||||
let msg = String(repeating: "a", count: 20001)
|
||||
let url = URL(string: "moltbot://agent?message=\(msg)")!
|
||||
await appModel.handleDeepLink(url: url)
|
||||
#expect(appModel.screen.errorText?.contains("Deep link too large") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func sendVoiceTranscriptThrowsWhenGatewayOffline() async {
|
||||
let appModel = NodeAppModel()
|
||||
await #expect(throws: Error.self) {
|
||||
try await appModel.sendVoiceTranscript(text: "hello", sessionKey: "main")
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func canvasA2UIActionDispatchesStatus() async {
|
||||
let appModel = NodeAppModel()
|
||||
let body: [String: Any] = [
|
||||
"userAction": [
|
||||
"name": "tap",
|
||||
"id": "action-1",
|
||||
"surfaceId": "main",
|
||||
"sourceComponentId": "button-1",
|
||||
"context": ["value": "ok"],
|
||||
],
|
||||
]
|
||||
await appModel._test_handleCanvasA2UIAction(body: body)
|
||||
#expect(appModel.screen.urlString.isEmpty)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import Testing
|
||||
import WebKit
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct ScreenControllerTests {
|
||||
@Test @MainActor func canvasModeConfiguresWebViewForTouch() {
|
||||
let screen = ScreenController()
|
||||
|
||||
#expect(screen.webView.isOpaque == true)
|
||||
#expect(screen.webView.backgroundColor == .black)
|
||||
|
||||
let scrollView = screen.webView.scrollView
|
||||
#expect(scrollView.backgroundColor == .black)
|
||||
#expect(scrollView.contentInsetAdjustmentBehavior == .never)
|
||||
#expect(scrollView.isScrollEnabled == false)
|
||||
#expect(scrollView.bounces == false)
|
||||
}
|
||||
|
||||
@Test @MainActor func navigateEnablesScrollForWebPages() {
|
||||
let screen = ScreenController()
|
||||
screen.navigate(to: "https://example.com")
|
||||
|
||||
let scrollView = screen.webView.scrollView
|
||||
#expect(scrollView.isScrollEnabled == true)
|
||||
#expect(scrollView.bounces == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func navigateSlashShowsDefaultCanvas() {
|
||||
let screen = ScreenController()
|
||||
screen.navigate(to: "/")
|
||||
|
||||
#expect(screen.urlString.isEmpty)
|
||||
}
|
||||
|
||||
@Test @MainActor func evalExecutesJavaScript() async throws {
|
||||
let screen = ScreenController()
|
||||
let deadline = ContinuousClock().now.advanced(by: .seconds(3))
|
||||
|
||||
while true {
|
||||
do {
|
||||
let result = try await screen.eval(javaScript: "1+1")
|
||||
#expect(result == "2")
|
||||
return
|
||||
} catch {
|
||||
if ContinuousClock().now >= deadline {
|
||||
throw error
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 100_000_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test @MainActor func localNetworkCanvasURLsAreAllowed() {
|
||||
let screen = ScreenController()
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://localhost:18789/")!) == true)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://clawd.local:18789/")!) == true)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://peters-mac-studio-1:18789/")!) == true)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "https://peters-mac-studio-1.ts.net:18789/")!) == true)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://192.168.0.10:18789/")!) == true)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://10.0.0.10:18789/")!) == true)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://100.123.224.76:18789/")!) == true) // Tailscale CGNAT
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "https://example.com/")!) == false)
|
||||
#expect(screen.isLocalNetworkCanvasURL(URL(string: "http://8.8.8.8/")!) == false)
|
||||
}
|
||||
|
||||
@Test func parseA2UIActionBodyAcceptsJSONString() throws {
|
||||
let body = ScreenController.parseA2UIActionBody("{\"userAction\":{\"name\":\"hello\"}}")
|
||||
let userAction = try #require(body?["userAction"] as? [String: Any])
|
||||
#expect(userAction["name"] as? String == "hello")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite(.serialized) struct ScreenRecordServiceTests {
|
||||
@Test func clampDefaultsAndBounds() {
|
||||
#expect(ScreenRecordService._test_clampDurationMs(nil) == 10000)
|
||||
#expect(ScreenRecordService._test_clampDurationMs(0) == 250)
|
||||
#expect(ScreenRecordService._test_clampDurationMs(60001) == 60000)
|
||||
|
||||
#expect(ScreenRecordService._test_clampFps(nil) == 10)
|
||||
#expect(ScreenRecordService._test_clampFps(0) == 1)
|
||||
#expect(ScreenRecordService._test_clampFps(120) == 30)
|
||||
#expect(ScreenRecordService._test_clampFps(.infinity) == 10)
|
||||
}
|
||||
|
||||
@Test @MainActor func recordRejectsInvalidScreenIndex() async {
|
||||
let recorder = ScreenRecordService()
|
||||
do {
|
||||
_ = try await recorder.record(
|
||||
screenIndex: 1,
|
||||
durationMs: 250,
|
||||
fps: 5,
|
||||
includeAudio: false,
|
||||
outPath: nil)
|
||||
Issue.record("Expected invalid screen index to throw")
|
||||
} catch let error as ScreenRecordService.ScreenRecordError {
|
||||
#expect(error.localizedDescription.contains("Invalid screen index") == true)
|
||||
} catch {
|
||||
Issue.record("Unexpected error type: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct SettingsNetworkingHelpersTests {
|
||||
@Test func parseHostPortParsesIPv4() {
|
||||
#expect(SettingsNetworkingHelpers.parseHostPort(from: "127.0.0.1:8080") == .init(host: "127.0.0.1", port: 8080))
|
||||
}
|
||||
|
||||
@Test func parseHostPortParsesHostnameAndTrims() {
|
||||
#expect(SettingsNetworkingHelpers.parseHostPort(from: " example.com:80 \n") == .init(
|
||||
host: "example.com",
|
||||
port: 80))
|
||||
}
|
||||
|
||||
@Test func parseHostPortParsesBracketedIPv6() {
|
||||
#expect(
|
||||
SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]:443") ==
|
||||
.init(host: "2001:db8::1", port: 443))
|
||||
}
|
||||
|
||||
@Test func parseHostPortRejectsMissingPort() {
|
||||
#expect(SettingsNetworkingHelpers.parseHostPort(from: "example.com") == nil)
|
||||
#expect(SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]") == nil)
|
||||
}
|
||||
|
||||
@Test func parseHostPortRejectsInvalidPort() {
|
||||
#expect(SettingsNetworkingHelpers.parseHostPort(from: "example.com:lol") == nil)
|
||||
#expect(SettingsNetworkingHelpers.parseHostPort(from: "[2001:db8::1]:lol") == nil)
|
||||
}
|
||||
|
||||
@Test func httpURLStringFormatsIPv4AndPort() {
|
||||
#expect(SettingsNetworkingHelpers
|
||||
.httpURLString(host: "127.0.0.1", port: 8080, fallback: "fallback") == "http://127.0.0.1:8080")
|
||||
}
|
||||
|
||||
@Test func httpURLStringBracketsIPv6() {
|
||||
#expect(SettingsNetworkingHelpers
|
||||
.httpURLString(host: "2001:db8::1", port: 8080, fallback: "fallback") == "http://[2001:db8::1]:8080")
|
||||
}
|
||||
|
||||
@Test func httpURLStringLeavesAlreadyBracketedIPv6() {
|
||||
#expect(SettingsNetworkingHelpers
|
||||
.httpURLString(host: "[2001:db8::1]", port: 8080, fallback: "fallback") == "http://[2001:db8::1]:8080")
|
||||
}
|
||||
|
||||
@Test func httpURLStringFallsBackWhenMissingHostOrPort() {
|
||||
#expect(SettingsNetworkingHelpers.httpURLString(host: nil, port: 80, fallback: "x") == "http://x")
|
||||
#expect(SettingsNetworkingHelpers.httpURLString(host: "example.com", port: nil, fallback: "y") == "http://y")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import MoltbotKit
|
||||
import SwiftUI
|
||||
import Testing
|
||||
import UIKit
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct SwiftUIRenderSmokeTests {
|
||||
@MainActor private static func host(_ view: some View) -> UIWindow {
|
||||
let window = UIWindow(frame: UIScreen.main.bounds)
|
||||
window.rootViewController = UIHostingController(rootView: view)
|
||||
window.makeKeyAndVisible()
|
||||
window.rootViewController?.view.setNeedsLayout()
|
||||
window.rootViewController?.view.layoutIfNeeded()
|
||||
return window
|
||||
}
|
||||
|
||||
@Test @MainActor func statusPillConnectingBuildsAViewHierarchy() {
|
||||
let root = StatusPill(gateway: .connecting, voiceWakeEnabled: true, brighten: true) {}
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func statusPillDisconnectedBuildsAViewHierarchy() {
|
||||
let root = StatusPill(gateway: .disconnected, voiceWakeEnabled: false) {}
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func settingsTabBuildsAViewHierarchy() {
|
||||
let appModel = NodeAppModel()
|
||||
let gatewayController = GatewayConnectionController(appModel: appModel, startDiscovery: false)
|
||||
|
||||
let root = SettingsTab()
|
||||
.environment(appModel)
|
||||
.environment(appModel.voiceWake)
|
||||
.environment(gatewayController)
|
||||
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func rootTabsBuildAViewHierarchy() {
|
||||
let appModel = NodeAppModel()
|
||||
let gatewayController = GatewayConnectionController(appModel: appModel, startDiscovery: false)
|
||||
|
||||
let root = RootTabs()
|
||||
.environment(appModel)
|
||||
.environment(appModel.voiceWake)
|
||||
.environment(gatewayController)
|
||||
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func voiceTabBuildsAViewHierarchy() {
|
||||
let appModel = NodeAppModel()
|
||||
|
||||
let root = VoiceTab()
|
||||
.environment(appModel)
|
||||
.environment(appModel.voiceWake)
|
||||
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func voiceWakeWordsViewBuildsAViewHierarchy() {
|
||||
let appModel = NodeAppModel()
|
||||
let root = NavigationStack { VoiceWakeWordsSettingsView() }
|
||||
.environment(appModel)
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func chatSheetBuildsAViewHierarchy() {
|
||||
let appModel = NodeAppModel()
|
||||
let gateway = GatewayNodeSession()
|
||||
let root = ChatSheet(gateway: gateway, sessionKey: "test")
|
||||
.environment(appModel)
|
||||
.environment(appModel.voiceWake)
|
||||
_ = Self.host(root)
|
||||
}
|
||||
|
||||
@Test @MainActor func voiceWakeToastBuildsAViewHierarchy() {
|
||||
let root = VoiceWakeToast(command: "moltbot: do something")
|
||||
_ = Self.host(root)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct VoiceWakeGatewaySyncTests {
|
||||
@Test func decodeGatewayTriggersFromJSONSanitizes() {
|
||||
let payload = #"{"triggers":[" clawd ","", "computer"]}"#
|
||||
let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payload)
|
||||
#expect(triggers == ["clawd", "computer"])
|
||||
}
|
||||
|
||||
@Test func decodeGatewayTriggersFromJSONFallsBackWhenEmpty() {
|
||||
let payload = #"{"triggers":[" ",""]}"#
|
||||
let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: payload)
|
||||
#expect(triggers == VoiceWakePreferences.defaultTriggerWords)
|
||||
}
|
||||
|
||||
@Test func decodeGatewayTriggersFromInvalidJSONReturnsNil() {
|
||||
let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: "not json")
|
||||
#expect(triggers == nil)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import Foundation
|
||||
import SwabbleKit
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct VoiceWakeManagerExtractCommandTests {
|
||||
@Test func extractCommandReturnsNilWhenNoTriggerFound() {
|
||||
let transcript = "hello world"
|
||||
let segments = makeSegments(
|
||||
transcript: transcript,
|
||||
words: [("hello", 0.0, 0.1), ("world", 0.2, 0.1)])
|
||||
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
|
||||
}
|
||||
|
||||
@Test func extractCommandTrimsTokensAndResult() {
|
||||
let transcript = "hey clawd do thing"
|
||||
let segments = makeSegments(
|
||||
transcript: transcript,
|
||||
words: [
|
||||
("hey", 0.0, 0.1),
|
||||
("clawd", 0.2, 0.1),
|
||||
("do", 0.9, 0.1),
|
||||
("thing", 1.1, 0.1),
|
||||
])
|
||||
let cmd = VoiceWakeManager.extractCommand(
|
||||
from: transcript,
|
||||
segments: segments,
|
||||
triggers: [" clawd "],
|
||||
minPostTriggerGap: 0.3)
|
||||
#expect(cmd == "do thing")
|
||||
}
|
||||
|
||||
@Test func extractCommandReturnsNilWhenGapTooShort() {
|
||||
let transcript = "hey clawd do thing"
|
||||
let segments = makeSegments(
|
||||
transcript: transcript,
|
||||
words: [
|
||||
("hey", 0.0, 0.1),
|
||||
("clawd", 0.2, 0.1),
|
||||
("do", 0.35, 0.1),
|
||||
("thing", 0.5, 0.1),
|
||||
])
|
||||
let cmd = VoiceWakeManager.extractCommand(
|
||||
from: transcript,
|
||||
segments: segments,
|
||||
triggers: ["clawd"],
|
||||
minPostTriggerGap: 0.3)
|
||||
#expect(cmd == nil)
|
||||
}
|
||||
|
||||
@Test func extractCommandReturnsNilWhenNothingAfterTrigger() {
|
||||
let transcript = "hey clawd"
|
||||
let segments = makeSegments(
|
||||
transcript: transcript,
|
||||
words: [("hey", 0.0, 0.1), ("clawd", 0.2, 0.1)])
|
||||
#expect(VoiceWakeManager.extractCommand(from: transcript, segments: segments, triggers: ["clawd"]) == nil)
|
||||
}
|
||||
|
||||
@Test func extractCommandIgnoresEmptyTriggers() {
|
||||
let transcript = "hey clawd do thing"
|
||||
let segments = makeSegments(
|
||||
transcript: transcript,
|
||||
words: [
|
||||
("hey", 0.0, 0.1),
|
||||
("clawd", 0.2, 0.1),
|
||||
("do", 0.9, 0.1),
|
||||
("thing", 1.1, 0.1),
|
||||
])
|
||||
let cmd = VoiceWakeManager.extractCommand(
|
||||
from: transcript,
|
||||
segments: segments,
|
||||
triggers: ["", " ", "clawd"],
|
||||
minPostTriggerGap: 0.3)
|
||||
#expect(cmd == "do thing")
|
||||
}
|
||||
}
|
||||
|
||||
private func makeSegments(
|
||||
transcript: String,
|
||||
words: [(String, TimeInterval, TimeInterval)])
|
||||
-> [WakeWordSegment] {
|
||||
var searchStart = transcript.startIndex
|
||||
var output: [WakeWordSegment] = []
|
||||
for (word, start, duration) in words {
|
||||
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
|
||||
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
|
||||
if let range { searchStart = range.upperBound }
|
||||
}
|
||||
return output
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import Foundation
|
||||
import SwabbleKit
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite(.serialized) struct VoiceWakeManagerStateTests {
|
||||
@Test @MainActor func suspendAndResumeCycleUpdatesState() async {
|
||||
let manager = VoiceWakeManager()
|
||||
manager.isEnabled = true
|
||||
manager.isListening = true
|
||||
manager.statusText = "Listening"
|
||||
|
||||
let suspended = manager.suspendForExternalAudioCapture()
|
||||
#expect(suspended == true)
|
||||
#expect(manager.isListening == false)
|
||||
#expect(manager.statusText == "Paused")
|
||||
|
||||
manager.resumeAfterExternalAudioCapture(wasSuspended: true)
|
||||
try? await Task.sleep(nanoseconds: 900_000_000)
|
||||
#expect(manager.statusText.contains("Voice Wake") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleRecognitionCallbackRestartsOnError() async {
|
||||
let manager = VoiceWakeManager()
|
||||
manager.isEnabled = true
|
||||
manager.isListening = true
|
||||
|
||||
manager._test_handleRecognitionCallback(transcript: nil, segments: [], errorText: "boom")
|
||||
#expect(manager.statusText.contains("Recognizer error") == true)
|
||||
#expect(manager.isListening == false)
|
||||
|
||||
try? await Task.sleep(nanoseconds: 900_000_000)
|
||||
#expect(manager.statusText.contains("Voice Wake") == true)
|
||||
}
|
||||
|
||||
@Test @MainActor func handleRecognitionCallbackDispatchesCommand() async {
|
||||
let manager = VoiceWakeManager()
|
||||
manager.triggerWords = ["clawd"]
|
||||
manager.isEnabled = true
|
||||
|
||||
actor CaptureBox {
|
||||
var value: String?
|
||||
func set(_ next: String) { self.value = next }
|
||||
}
|
||||
let capture = CaptureBox()
|
||||
manager.configure { cmd in
|
||||
await capture.set(cmd)
|
||||
}
|
||||
|
||||
let transcript = "clawd hello"
|
||||
let clawdRange = transcript.range(of: "clawd")!
|
||||
let helloRange = transcript.range(of: "hello")!
|
||||
let segments = [
|
||||
WakeWordSegment(text: "clawd", start: 0.0, duration: 0.2, range: clawdRange),
|
||||
WakeWordSegment(text: "hello", start: 0.8, duration: 0.2, range: helloRange),
|
||||
]
|
||||
|
||||
manager._test_handleRecognitionCallback(transcript: transcript, segments: segments, errorText: nil)
|
||||
#expect(manager.lastTriggeredCommand == "hello")
|
||||
#expect(manager.statusText == "Triggered")
|
||||
|
||||
try? await Task.sleep(nanoseconds: 300_000_000)
|
||||
#expect(await capture.value == "hello")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import Moltbot
|
||||
|
||||
@Suite struct VoiceWakePreferencesTests {
|
||||
@Test func sanitizeTriggerWordsTrimsAndDropsEmpty() {
|
||||
#expect(VoiceWakePreferences.sanitizeTriggerWords([" clawd ", "", " \nclaude\t"]) == ["clawd", "claude"])
|
||||
}
|
||||
|
||||
@Test func sanitizeTriggerWordsFallsBackToDefaultsWhenEmpty() {
|
||||
#expect(VoiceWakePreferences.sanitizeTriggerWords(["", " "]) == VoiceWakePreferences.defaultTriggerWords)
|
||||
}
|
||||
|
||||
@Test func sanitizeTriggerWordsLimitsWordLength() {
|
||||
let long = String(repeating: "x", count: VoiceWakePreferences.maxWordLength + 5)
|
||||
let cleaned = VoiceWakePreferences.sanitizeTriggerWords(["ok", long])
|
||||
#expect(cleaned[1].count == VoiceWakePreferences.maxWordLength)
|
||||
}
|
||||
|
||||
@Test func sanitizeTriggerWordsLimitsWordCount() {
|
||||
let words = (1...VoiceWakePreferences.maxWords + 3).map { "w\($0)" }
|
||||
let cleaned = VoiceWakePreferences.sanitizeTriggerWords(words)
|
||||
#expect(cleaned.count == VoiceWakePreferences.maxWords)
|
||||
}
|
||||
|
||||
@Test func displayStringUsesSanitizedWords() {
|
||||
#expect(VoiceWakePreferences.displayString(for: ["", " "]) == "clawd, claude")
|
||||
}
|
||||
|
||||
@Test func loadAndSaveTriggerWordsRoundTrip() {
|
||||
let suiteName = "VoiceWakePreferencesTests.\(UUID().uuidString)"
|
||||
let defaults = UserDefaults(suiteName: suiteName)!
|
||||
|
||||
#expect(VoiceWakePreferences.loadTriggerWords(defaults: defaults) == VoiceWakePreferences.defaultTriggerWords)
|
||||
VoiceWakePreferences.saveTriggerWords(["computer"], defaults: defaults)
|
||||
#expect(VoiceWakePreferences.loadTriggerWords(defaults: defaults) == ["computer"])
|
||||
}
|
||||
}
|
||||
21
docker-compose/ez-assistant/apps/ios/fastlane/.env.example
Normal file
@@ -0,0 +1,21 @@
|
||||
# App Store Connect API key (pick one approach)
|
||||
#
|
||||
# Recommended (use the downloaded .p8 directly):
|
||||
# ASC_KEY_ID=XXXXXXXXXX
|
||||
# ASC_ISSUER_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
|
||||
# ASC_KEY_PATH=/absolute/path/to/AuthKey_XXXXXXXXXX.p8
|
||||
#
|
||||
# Or (JSON key file):
|
||||
# APP_STORE_CONNECT_API_KEY_PATH=/absolute/path/to/AuthKey_XXXXXX.json
|
||||
#
|
||||
# Or:
|
||||
# ASC_KEY_ID=XXXXXXXXXX
|
||||
# ASC_ISSUER_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
|
||||
# ASC_KEY_CONTENT=BASE64_P8_CONTENT
|
||||
|
||||
# Code signing
|
||||
# IOS_DEVELOPMENT_TEAM=XXXXXXXXXX
|
||||
|
||||
# Deliver toggles (off by default)
|
||||
# DELIVER_METADATA=1
|
||||
# DELIVER_SCREENSHOTS=1
|
||||
7
docker-compose/ez-assistant/apps/ios/fastlane/Appfile
Normal file
@@ -0,0 +1,7 @@
|
||||
app_identifier("bot.molt.ios")
|
||||
|
||||
# Auth is expected via App Store Connect API key.
|
||||
# Provide either:
|
||||
# - APP_STORE_CONNECT_API_KEY_PATH=/path/to/AuthKey_XXXXXX.p8.json (recommended)
|
||||
# or:
|
||||
# - ASC_KEY_ID, ASC_ISSUER_ID, and ASC_KEY_CONTENT (base64 or raw p8 content)
|
||||
103
docker-compose/ez-assistant/apps/ios/fastlane/Fastfile
Normal file
@@ -0,0 +1,103 @@
|
||||
require "shellwords"
|
||||
|
||||
default_platform(:ios)
|
||||
|
||||
def load_env_file(path)
|
||||
return unless File.exist?(path)
|
||||
|
||||
File.foreach(path) do |line|
|
||||
stripped = line.strip
|
||||
next if stripped.empty? || stripped.start_with?("#")
|
||||
|
||||
key, value = stripped.split("=", 2)
|
||||
next if key.nil? || key.empty? || value.nil?
|
||||
|
||||
ENV[key] = value if ENV[key].nil? || ENV[key].strip.empty?
|
||||
end
|
||||
end
|
||||
|
||||
platform :ios do
|
||||
private_lane :asc_api_key do
|
||||
load_env_file(File.join(__dir__, ".env"))
|
||||
|
||||
api_key = nil
|
||||
|
||||
key_path = ENV["APP_STORE_CONNECT_API_KEY_PATH"]
|
||||
if key_path && !key_path.strip.empty?
|
||||
api_key = app_store_connect_api_key(path: key_path)
|
||||
else
|
||||
p8_path = ENV["ASC_KEY_PATH"]
|
||||
if p8_path && !p8_path.strip.empty?
|
||||
key_id = ENV["ASC_KEY_ID"]
|
||||
issuer_id = ENV["ASC_ISSUER_ID"]
|
||||
UI.user_error!("Missing ASC_KEY_ID or ASC_ISSUER_ID for ASC_KEY_PATH auth.") if [key_id, issuer_id].any? { |v| v.nil? || v.strip.empty? }
|
||||
|
||||
api_key = app_store_connect_api_key(
|
||||
key_id: key_id,
|
||||
issuer_id: issuer_id,
|
||||
key_filepath: p8_path
|
||||
)
|
||||
else
|
||||
key_id = ENV["ASC_KEY_ID"]
|
||||
issuer_id = ENV["ASC_ISSUER_ID"]
|
||||
key_content = ENV["ASC_KEY_CONTENT"]
|
||||
|
||||
UI.user_error!("Missing App Store Connect API key. Set APP_STORE_CONNECT_API_KEY_PATH (json) or ASC_KEY_PATH (p8) or ASC_KEY_ID/ASC_ISSUER_ID/ASC_KEY_CONTENT.") if [key_id, issuer_id, key_content].any? { |v| v.nil? || v.strip.empty? }
|
||||
|
||||
is_base64 = key_content.include?("BEGIN PRIVATE KEY") ? false : true
|
||||
|
||||
api_key = app_store_connect_api_key(
|
||||
key_id: key_id,
|
||||
issuer_id: issuer_id,
|
||||
key_content: key_content,
|
||||
is_key_content_base64: is_base64
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
api_key
|
||||
end
|
||||
|
||||
desc "Build + upload to TestFlight"
|
||||
lane :beta do
|
||||
api_key = asc_api_key
|
||||
|
||||
team_id = ENV["IOS_DEVELOPMENT_TEAM"]
|
||||
if team_id.nil? || team_id.strip.empty?
|
||||
helper_path = File.expand_path("../../scripts/ios-team-id.sh", __dir__)
|
||||
if File.exist?(helper_path)
|
||||
team_id = sh("bash #{helper_path.shellescape}").strip
|
||||
end
|
||||
end
|
||||
UI.user_error!("Missing IOS_DEVELOPMENT_TEAM (Apple Team ID). Add it to fastlane/.env or export it in your shell.") if team_id.nil? || team_id.strip.empty?
|
||||
|
||||
build_app(
|
||||
project: "Moltbot.xcodeproj",
|
||||
scheme: "Moltbot",
|
||||
export_method: "app-store",
|
||||
clean: true,
|
||||
xcargs: "DEVELOPMENT_TEAM=#{team_id} -allowProvisioningUpdates",
|
||||
export_xcargs: "-allowProvisioningUpdates",
|
||||
export_options: {
|
||||
signingStyle: "automatic"
|
||||
}
|
||||
)
|
||||
|
||||
upload_to_testflight(
|
||||
api_key: api_key,
|
||||
skip_waiting_for_build_processing: true
|
||||
)
|
||||
end
|
||||
|
||||
desc "Upload App Store metadata (and optionally screenshots)"
|
||||
lane :metadata do
|
||||
api_key = asc_api_key
|
||||
|
||||
deliver(
|
||||
api_key: api_key,
|
||||
force: true,
|
||||
skip_screenshots: ENV["DELIVER_SCREENSHOTS"] != "1",
|
||||
skip_metadata: ENV["DELIVER_METADATA"] != "1"
|
||||
)
|
||||
end
|
||||
end
|
||||
32
docker-compose/ez-assistant/apps/ios/fastlane/SETUP.md
Normal file
@@ -0,0 +1,32 @@
|
||||
# fastlane setup (Moltbot iOS)
|
||||
|
||||
Install:
|
||||
|
||||
```bash
|
||||
brew install fastlane
|
||||
```
|
||||
|
||||
Create an App Store Connect API key:
|
||||
|
||||
- App Store Connect → Users and Access → Keys → App Store Connect API → Generate API Key
|
||||
- Download the `.p8`, note the **Issuer ID** and **Key ID**
|
||||
|
||||
Create `apps/ios/fastlane/.env` (gitignored):
|
||||
|
||||
```bash
|
||||
ASC_KEY_ID=YOUR_KEY_ID
|
||||
ASC_ISSUER_ID=YOUR_ISSUER_ID
|
||||
ASC_KEY_PATH=/absolute/path/to/AuthKey_XXXXXXXXXX.p8
|
||||
|
||||
# Code signing (Apple Team ID / App ID Prefix)
|
||||
IOS_DEVELOPMENT_TEAM=YOUR_TEAM_ID
|
||||
```
|
||||
|
||||
Tip: run `scripts/ios-team-id.sh` from the repo root to print a Team ID to paste into `.env`. Fastlane falls back to this helper if `IOS_DEVELOPMENT_TEAM` is missing.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cd apps/ios
|
||||
fastlane beta
|
||||
```
|
||||
134
docker-compose/ez-assistant/apps/ios/project.yml
Normal file
@@ -0,0 +1,134 @@
|
||||
name: Moltbot
|
||||
options:
|
||||
bundleIdPrefix: bot.molt
|
||||
deploymentTarget:
|
||||
iOS: "18.0"
|
||||
xcodeVersion: "16.0"
|
||||
|
||||
settings:
|
||||
base:
|
||||
SWIFT_VERSION: "6.0"
|
||||
|
||||
packages:
|
||||
MoltbotKit:
|
||||
path: ../shared/MoltbotKit
|
||||
Swabble:
|
||||
path: ../../Swabble
|
||||
|
||||
schemes:
|
||||
Moltbot:
|
||||
shared: true
|
||||
build:
|
||||
targets:
|
||||
Moltbot: all
|
||||
test:
|
||||
targets:
|
||||
- MoltbotTests
|
||||
|
||||
targets:
|
||||
Moltbot:
|
||||
type: application
|
||||
platform: iOS
|
||||
sources:
|
||||
- path: Sources
|
||||
dependencies:
|
||||
- package: MoltbotKit
|
||||
- package: MoltbotKit
|
||||
product: MoltbotChatUI
|
||||
- package: MoltbotKit
|
||||
product: MoltbotProtocol
|
||||
- package: Swabble
|
||||
product: SwabbleKit
|
||||
- sdk: AppIntents.framework
|
||||
preBuildScripts:
|
||||
- name: SwiftFormat (lint)
|
||||
basedOnDependencyAnalysis: false
|
||||
inputFileLists:
|
||||
- $(SRCROOT)/SwiftSources.input.xcfilelist
|
||||
script: |
|
||||
set -euo pipefail
|
||||
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
|
||||
if ! command -v swiftformat >/dev/null 2>&1; then
|
||||
echo "error: swiftformat not found (brew install swiftformat)" >&2
|
||||
exit 1
|
||||
fi
|
||||
swiftformat --lint --config "$SRCROOT/../../.swiftformat" \
|
||||
--filelist "$SRCROOT/SwiftSources.input.xcfilelist"
|
||||
- name: SwiftLint
|
||||
basedOnDependencyAnalysis: false
|
||||
inputFileLists:
|
||||
- $(SRCROOT)/SwiftSources.input.xcfilelist
|
||||
script: |
|
||||
set -euo pipefail
|
||||
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
|
||||
if ! command -v swiftlint >/dev/null 2>&1; then
|
||||
echo "error: swiftlint not found (brew install swiftlint)" >&2
|
||||
exit 1
|
||||
fi
|
||||
swiftlint lint --config "$SRCROOT/.swiftlint.yml" --use-script-input-file-lists
|
||||
settings:
|
||||
base:
|
||||
CODE_SIGN_IDENTITY: "Apple Development"
|
||||
CODE_SIGN_STYLE: Manual
|
||||
DEVELOPMENT_TEAM: Y5PE65HELJ
|
||||
PRODUCT_BUNDLE_IDENTIFIER: bot.molt.ios
|
||||
PROVISIONING_PROFILE_SPECIFIER: "bot.molt.ios Development"
|
||||
SWIFT_VERSION: "6.0"
|
||||
SWIFT_STRICT_CONCURRENCY: complete
|
||||
ENABLE_APPINTENTS_METADATA: NO
|
||||
info:
|
||||
path: Sources/Info.plist
|
||||
properties:
|
||||
CFBundleDisplayName: Moltbot
|
||||
CFBundleIconName: AppIcon
|
||||
CFBundleShortVersionString: "2026.1.26"
|
||||
CFBundleVersion: "20260126"
|
||||
UILaunchScreen: {}
|
||||
UIApplicationSceneManifest:
|
||||
UIApplicationSupportsMultipleScenes: false
|
||||
UIBackgroundModes:
|
||||
- audio
|
||||
NSLocalNetworkUsageDescription: Moltbot discovers and connects to your Moltbot gateway on the local network.
|
||||
NSAppTransportSecurity:
|
||||
NSAllowsArbitraryLoadsInWebContent: true
|
||||
NSBonjourServices:
|
||||
- _moltbot-gw._tcp
|
||||
NSCameraUsageDescription: Moltbot can capture photos or short video clips when requested via the gateway.
|
||||
NSLocationWhenInUseUsageDescription: Moltbot uses your location when you allow location sharing.
|
||||
NSLocationAlwaysAndWhenInUseUsageDescription: Moltbot can share your location in the background when you enable Always.
|
||||
NSMicrophoneUsageDescription: Moltbot needs microphone access for voice wake.
|
||||
NSSpeechRecognitionUsageDescription: Moltbot uses on-device speech recognition for voice wake.
|
||||
UISupportedInterfaceOrientations:
|
||||
- UIInterfaceOrientationPortrait
|
||||
- UIInterfaceOrientationPortraitUpsideDown
|
||||
- UIInterfaceOrientationLandscapeLeft
|
||||
- UIInterfaceOrientationLandscapeRight
|
||||
UISupportedInterfaceOrientations~ipad:
|
||||
- UIInterfaceOrientationPortrait
|
||||
- UIInterfaceOrientationPortraitUpsideDown
|
||||
- UIInterfaceOrientationLandscapeLeft
|
||||
- UIInterfaceOrientationLandscapeRight
|
||||
|
||||
MoltbotTests:
|
||||
type: bundle.unit-test
|
||||
platform: iOS
|
||||
sources:
|
||||
- path: Tests
|
||||
dependencies:
|
||||
- target: Moltbot
|
||||
- package: Swabble
|
||||
product: SwabbleKit
|
||||
- sdk: AppIntents.framework
|
||||
settings:
|
||||
base:
|
||||
PRODUCT_BUNDLE_IDENTIFIER: bot.molt.ios.tests
|
||||
SWIFT_VERSION: "6.0"
|
||||
SWIFT_STRICT_CONCURRENCY: complete
|
||||
TEST_HOST: "$(BUILT_PRODUCTS_DIR)/Moltbot.app/Moltbot"
|
||||
BUNDLE_LOADER: "$(TEST_HOST)"
|
||||
info:
|
||||
path: Tests/Info.plist
|
||||
properties:
|
||||
CFBundleDisplayName: MoltbotTests
|
||||
CFBundleShortVersionString: "2026.1.26"
|
||||
CFBundleVersion: "20260126"
|
||||