import CoreImage import Foundation import MLXLMCommon import UIKit func log(_ message: String) { print(message) } // MARK: - 模型/推理配置 @Observable class ModelConfig { static let selectedModelDefaultsKey = "PhoneClaw.selectedModelID" var maxTokens = 4000 var topK = 64 var topP = 0.95 var temperature = 1.0 var useGPU = true var selectedModelID = UserDefaults.standard.string(forKey: selectedModelDefaultsKey) ?? MLXLocalLLMService.defaultModel.id /// System prompt — 由 AgentEngine.loadSystemPrompt() 从 SYSPROMPT.md 注入,不在代码里硬编码。 var systemPrompt = "" } // MARK: - SYSPROMPT 默认内容(仅在文件不存在时写入磁盘) private let kDefaultSystemPrompt = """ 你是 PhoneClaw,一个运行在本地设备上的私人 AI 助手。你完全离线运行,不联网,保护用户隐私。 你拥有以下能力(Skill): ___SKILLS___ 只有当用户明确要求执行某项设备内操作时,才调用 load_skill 加载该能力的详细指令。 像"配置""信息""看看""帮我查一下"这类含糊词,不足以单独触发工具调用。 如果用户只是普通聊天、追问上文、让你解释结果,直接回答,不要调用工具。 如果确实需要某个能力,你必须自己调用 load_skill,不要让用户去"使用某个能力"或"打开某个 skill"。 当且仅当确实需要某个能力时,先调用 load_skill: {"name": "load_skill", "arguments": {"skill": "能力名"}} 在已经拿到工具结果后,优先直接给出最终答案,不要无谓追问。 用中文回答,简洁实用。 """ // MARK: - 聊天消息 struct ChatImageAttachment: Identifiable { let id = UUID() let data: Data init?(image: UIImage) { if let jpeg = image.jpegData(compressionQuality: 0.92) { self.data = jpeg } else if let png = image.pngData() { self.data = png } else { return nil } } var uiImage: UIImage? { UIImage(data: data) } var ciImage: CIImage? { if let image = CIImage( data: data, options: [.applyOrientationProperty: true] ) { return image } guard let uiImage else { return nil } if let ciImage = uiImage.ciImage { return ciImage } if let cgImage = uiImage.cgImage { return CIImage(cgImage: cgImage) } return CIImage(image: uiImage) } } struct ChatMessage: Identifiable { let id: UUID var role: Role var content: String var images: [ChatImageAttachment] let timestamp = Date() var skillName: String? = nil init( role: Role, content: String, images: [ChatImageAttachment] = [], skillName: String? = nil ) { self.id = UUID() self.role = role self.content = content self.images = images self.skillName = skillName } mutating func update(content: String) { guard self.content != content else { return } self.content = content } mutating func update(role: Role, content: String, skillName: String? = nil) { self.role = role self.content = content self.skillName = skillName } enum Role { case user, assistant, system, skillResult } } // MARK: - Agent Engine @Observable class AgentEngine { let llm = MLXLocalLLMService() var messages: [ChatMessage] = [] var isProcessing = false var config = ModelConfig() // 文件驱动的 Skill 系统 let skillLoader = SkillLoader() let toolRegistry = ToolRegistry.shared // Skill 条目(给 UI 管理用,可开关) var skillEntries: [SkillEntry] = [] var enabledSkillInfos: [SkillInfo] { skillEntries.filter(\.isEnabled).map { SkillInfo(name: $0.id, description: $0.description, displayName: $0.name, icon: $0.icon, samplePrompt: $0.samplePrompt) } } var availableModels: [BundledModelOption] { MLXLocalLLMService.availableModels } init() { loadSkillEntries() } private func loadSkillEntries() { let definitions = skillLoader.discoverSkills() self.skillEntries = definitions.map { SkillEntry(from: $0, registry: toolRegistry) } } func reloadSkills() { let enabledState = Dictionary(uniqueKeysWithValues: skillEntries.map { ($0.id, $0.isEnabled) }) loadSkillEntries() for i in skillEntries.indices { if let wasEnabled = enabledState[skillEntries[i].id] { skillEntries[i].isEnabled = wasEnabled skillLoader.setEnabled(skillEntries[i].id, enabled: wasEnabled) } } } // MARK: - Skill 查找(文件驱动) private func findSkillId(for name: String) -> String? { if skillLoader.getDefinition(name) != nil { return name } return skillLoader.findSkillId(forTool: name) } private func findDisplayName(for name: String) -> String { if let skillId = findSkillId(for: name), let def = skillLoader.getDefinition(skillId) { return def.metadata.displayName } return name } private func handleLoadSkill(skillName: String) -> String? { guard let entry = skillEntries.first(where: { $0.id == skillName }), entry.isEnabled else { return nil } return skillLoader.loadBody(skillId: skillName) } private func handleToolExecution(toolName: String, args: [String: Any]) async throws -> String { return try await toolRegistry.execute(name: toolName, args: args) } private func autoToolCallForLoadedSkills( skillIds: [String] ) -> (name: String, arguments: [String: Any])? { let uniqueSkillIds = Array(NSOrderedSet(array: skillIds)) as? [String] ?? skillIds guard uniqueSkillIds.count == 1, let skillId = uniqueSkillIds.first, let def = skillLoader.getDefinition(skillId), def.isEnabled else { return nil } let uniqueToolNames = Array(NSOrderedSet(array: def.metadata.allowedTools)) as? [String] ?? def.metadata.allowedTools guard uniqueToolNames.count == 1, let toolName = uniqueToolNames.first, let tool = toolRegistry.find(name: toolName), tool.parameters == "无" else { return nil } return (tool.name, [:]) } private func registeredTools(for skillId: String) -> [RegisteredTool] { if let def = skillLoader.getDefinition(skillId) { let tools = toolRegistry.toolsFor(names: def.metadata.allowedTools) if !tools.isEmpty { return tools } } if let entry = skillEntries.first(where: { $0.id == skillId }) { let tools = entry.tools.compactMap { toolRegistry.find(name: $0.name) } if !tools.isEmpty { return tools } } return [] } private func inferFallbackToolCall( skillIds: [String], userQuestion: String ) -> (name: String, arguments: [String: Any])? { let uniqueSkillIds = Array(NSOrderedSet(array: skillIds)) as? [String] ?? skillIds guard uniqueSkillIds.count == 1, let skillId = uniqueSkillIds.first else { return nil } let tools = registeredTools(for: skillId) guard !tools.isEmpty else { return nil } if tools.count == 1, tools[0].parameters == "无" { return (tools[0].name, [:]) } let normalizedQuestion = userQuestion.lowercased() func has(_ keyword: String) -> Bool { normalizedQuestion.contains(keyword) } let candidateNames: [String] switch skillId { case "device": if has("系统版本") || has("ios 版本") || has("版本号") { candidateNames = ["device-system-version", "device-info"] } else if has("名字") || has("名称") || has("叫什么") { candidateNames = ["device-name", "device-info"] } else if has("型号") || has("机型") { candidateNames = ["device-model", "device-info"] } else if has("内存") || has("ram") { candidateNames = ["device-memory", "device-info"] } else if has("处理器") || has("核心") || has("cpu") { candidateNames = ["device-processor-count", "device-info"] } else { candidateNames = ["device-info"] } default: candidateNames = [] } for name in candidateNames { if let tool = tools.first(where: { $0.name == name }), tool.parameters == "无" { return (tool.name, [:]) } } return nil } private enum SingleToolExtractionOutcome { case toolCall(name: String, arguments: [String: Any]) case needsClarification(String) case failed } private func singleRegisteredToolForLoadedSkills(skillIds: [String]) -> RegisteredTool? { let uniqueSkillIds = Array(NSOrderedSet(array: skillIds)) as? [String] ?? skillIds guard uniqueSkillIds.count == 1, let skillId = uniqueSkillIds.first else { return nil } let tools = registeredTools(for: skillId) guard tools.count == 1 else { return nil } return tools.first } private func parseJSONObject(_ text: String) -> [String: Any]? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return nil } let candidates: [String] = { if trimmed.hasPrefix("```json") || trimmed.hasPrefix("```") { let stripped = trimmed .replacingOccurrences(of: "```json", with: "") .replacingOccurrences(of: "```", with: "") .trimmingCharacters(in: .whitespacesAndNewlines) return [stripped] } if let start = trimmed.firstIndex(of: "{"), let end = trimmed.lastIndex(of: "}"), start <= end { return [trimmed, String(trimmed[start...end])] } return [trimmed] }() for candidate in candidates { guard let data = candidate.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } return object } return nil } private func iso8601StringForModel(from date: Date) -> String { let formatter = ISO8601DateFormatter() formatter.timeZone = .current formatter.formatOptions = [.withInternetDateTime] return formatter.string(from: date) } private func chineseNumberValue(_ token: String) -> Int? { if let value = Int(token) { return value } let digits: [Character: Int] = [ "零": 0, "〇": 0, "一": 1, "二": 2, "两": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9 ] if token == "十" { return 10 } if token.hasPrefix("十"), let last = token.last, let digit = digits[last] { return 10 + digit } if token.hasSuffix("十"), let first = token.first, let digit = digits[first] { return digit * 10 } if token.count == 2 { let chars = Array(token) if let tens = digits[chars[0]], chars[1] == "十" { return tens * 10 } } if token.count == 3 { let chars = Array(token) if let tens = digits[chars[0]], chars[1] == "十", let ones = digits[chars[2]] { return tens * 10 + ones } } return nil } private func parseBasicChineseDate(from text: String) -> Date? { let patterns = [ "(今天|今日|今晚|今夜|明天|明晚|后天)(?:的)?(凌晨|早上|上午|中午|下午|晚上|傍晚)?([零〇一二两三四五六七八九十\\d]{1,3})点(?:(半)|([零〇一二两三四五六七八九十\\d]{1,3})分?)?", "(凌晨|早上|上午|中午|下午|晚上|傍晚)([零〇一二两三四五六七八九十\\d]{1,3})点(?:(半)|([零〇一二两三四五六七八九十\\d]{1,3})分?)?" ] for pattern in patterns { guard let regex = try? NSRegularExpression(pattern: pattern) else { continue } let range = NSRange(text.startIndex..., in: text) guard let match = regex.firstMatch(in: text, range: range) else { continue } var dayToken: String? var periodToken: String? var hourToken: String? var hasHalf = false var minuteToken: String? if match.numberOfRanges >= 6 { if let r = Range(match.range(at: 1), in: text), !r.isEmpty { dayToken = String(text[r]) } if let r = Range(match.range(at: 2), in: text), !r.isEmpty { periodToken = String(text[r]) } if let r = Range(match.range(at: 3), in: text), !r.isEmpty { hourToken = String(text[r]) } if let r = Range(match.range(at: 4), in: text), !r.isEmpty { hasHalf = true } if let r = Range(match.range(at: 5), in: text), !r.isEmpty { minuteToken = String(text[r]) } } if hourToken == nil, match.numberOfRanges >= 5 { if let r = Range(match.range(at: 1), in: text), !r.isEmpty { periodToken = String(text[r]) } if let r = Range(match.range(at: 2), in: text), !r.isEmpty { hourToken = String(text[r]) } if let r = Range(match.range(at: 3), in: text), !r.isEmpty { hasHalf = true } if let r = Range(match.range(at: 4), in: text), !r.isEmpty { minuteToken = String(text[r]) } } guard let hourToken, var hour = chineseNumberValue(hourToken) else { continue } var minute = 0 if hasHalf { minute = 30 } else if let minuteToken, let parsedMinute = chineseNumberValue(minuteToken) { minute = parsedMinute } if let periodToken { switch periodToken { case "下午", "晚上", "傍晚": if hour < 12 { hour += 12 } case "中午": if hour < 11 { hour += 12 } case "凌晨": if hour == 12 { hour = 0 } default: break } } var dayOffset = 0 switch dayToken { case "明天", "明晚": dayOffset = 1 case "后天": dayOffset = 2 default: dayOffset = 0 } let calendar = Calendar.current let baseDate = calendar.startOfDay(for: Date()) guard let day = calendar.date(byAdding: .day, value: dayOffset, to: baseDate), let finalDate = calendar.date( bySettingHour: hour, minute: minute, second: 0, of: day ) else { continue } return finalDate } return nil } private func detectDateInQuestion(_ text: String) -> Date? { if let detector = try? NSDataDetector(types: NSTextCheckingResult.CheckingType.date.rawValue) { let range = NSRange(text.startIndex..., in: text) if let match = detector.firstMatch(in: text, options: [], range: range), let date = match.date { return date } } return parseBasicChineseDate(from: text) } private func heuristicArgumentsForTool( toolName: String, userQuestion: String ) -> [String: Any]? { let text = userQuestion.trimmingCharacters(in: .whitespacesAndNewlines) guard !text.isEmpty else { return nil } switch toolName { case "contacts-upsert": let phone = text.firstMatch(of: /1[3-9]\d{9}/).map { String($0.0) } var name: String? let patterns = [ "(?:帮我)?(?:存(?:一下)?|保存|记一下)(.+?)(?:的)?(?:电话|手机号|号码|联系方式)", "(?:联系人|通讯录)里(?:添加|保存)?(.+?)(?:的)?(?:电话|手机号|号码|联系方式)" ] for pattern in patterns { guard let regex = try? NSRegularExpression(pattern: pattern) else { continue } let range = NSRange(text.startIndex..., in: text) if let match = regex.firstMatch(in: text, range: range), let capture = Range(match.range(at: 1), in: text) { name = String(text[capture]).trimmingCharacters(in: .whitespacesAndNewlines) break } } if name == nil, let phone, let phoneRange = text.range(of: phone) { let prefix = text[.. Bool { switch toolName { case "calendar-create-event": return arguments["title"] is String && arguments["start"] is String case "reminders-create": return arguments["title"] is String case "contacts-upsert": return arguments["name"] is String default: return !arguments.isEmpty } } private func extractSingleToolCallForLoadedSkill( originalPrompt: String, userQuestion: String, skillInstructions: String, skillIds: [String], images: [CIImage] ) async -> SingleToolExtractionOutcome { guard let tool = singleRegisteredToolForLoadedSkills(skillIds: skillIds), tool.parameters != "无" else { return .failed } let extractionPrompt = PromptBuilder.buildSingleToolArgumentsPrompt( originalPrompt: originalPrompt, userQuestion: userQuestion, skillInstructions: skillInstructions, toolName: tool.name, toolParameters: tool.parameters, currentImageCount: images.count ) if let raw = await streamLLM(prompt: extractionPrompt, images: images) { let cleaned = cleanOutput(raw) if let payload = parseJSONObject(cleaned) { if let clarification = payload["_needs_clarification"] as? String, !clarification.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines).isEmpty { return .needsClarification(clarification) } if validateSingleToolArguments(toolName: tool.name, arguments: payload) { return .toolCall(name: tool.name, arguments: payload) } } } if let heuristic = heuristicArgumentsForTool(toolName: tool.name, userQuestion: userQuestion), validateSingleToolArguments(toolName: tool.name, arguments: heuristic) { return .toolCall(name: tool.name, arguments: heuristic) } return .failed } private func markSkillsDone(_ displayNames: [String]) { guard !displayNames.isEmpty else { return } for index in messages.indices { guard messages[index].role == .system, let skillName = messages[index].skillName, displayNames.contains(skillName), messages[index].content == "identified" || messages[index].content == "loaded" else { continue } messages[index].update(role: .system, content: "done", skillName: skillName) } } private func looksLikeStructuredIntermediateOutput(_ text: String) -> Bool { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return false } if trimmed.hasPrefix("```json") || trimmed.hasPrefix("```") { return true } if let regex = try? NSRegularExpression( pattern: "\"[A-Za-z_][A-Za-z0-9_]*\"\\s*:", options: [] ) { let matchCount = regex.numberOfMatches( in: trimmed, range: NSRange(trimmed.startIndex..., in: trimmed) ) if matchCount >= 2 && !trimmed.hasPrefix("{") { return true } } let suspiciousFragments = [ "tool_name\":", "result_for_user_name\":", "text_for_display\":", "tool_operation_success\":", "arguments_for_tool_no_skill\":", "memory_user_power_conversion\":" ] if suspiciousFragments.filter({ trimmed.contains($0) }).count >= 2 { return true } guard let data = trimmed.data(using: .utf8), let json = try? JSONSerialization.jsonObject(with: data) else { return false } if let dict = json as? [String: Any] { if dict["name"] != nil { return false } let suspiciousKeys = [ "final_answer", "tool_call", "arguments", "device_call", "next_action", "action", "tool" ] return suspiciousKeys.contains { dict[$0] != nil } } return false } private func looksLikePromptEcho(_ text: String) -> Bool { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return false } if trimmed.hasPrefix("user\n") || trimmed == "user" { return true } let suspiciousPhrases = [ "根据已加载的 Skill", "不要将任何关于工具、系统或该请求的描述变成 Markdown 代码或 JSON 模板", "如果需要,请直接调用", "package_name", "text_for_user" ] let hitCount = suspiciousPhrases.reduce(into: 0) { count, phrase in if trimmed.contains(phrase) { count += 1 } } return hitCount >= 2 } private func syntheticToolCallText( name: String, arguments: [String: Any] ) -> String { let jsonData = try? JSONSerialization.data(withJSONObject: [ "name": name, "arguments": arguments ]) let jsonString = jsonData.flatMap { String(data: $0, encoding: .utf8) } ?? "{\"name\":\"\(name)\",\"arguments\":{}}" return """ \(jsonString) """ } private func parsedToolPayload(from toolResult: String) -> [String: Any]? { guard let data = toolResult.data(using: .utf8), let payload = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } return payload } private func toolResultSummaryForModel( toolName: String, toolResult: String ) -> String { let trimmed = toolResult.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return "工具 \(toolName) 已执行,但没有返回内容。" } if let payload = parsedToolPayload(from: trimmed) { if let success = payload["success"] as? Bool, !success, let error = payload["error"] as? String, !error.isEmpty { return "工具 \(toolName) 执行失败:\(error)" } if let result = payload["result"] as? String { let summary = result.trimmingCharacters(in: .whitespacesAndNewlines) if !summary.isEmpty { return summary } } } if let rendered = renderToolResultLocally(toolName: toolName, toolResult: trimmed) { return rendered } return trimmed } private func fallbackReplyForEmptyToolFollowUp(toolName: String, toolResult: String) -> String { let trimmed = toolResult.trimmingCharacters(in: .whitespacesAndNewlines) let summary = toolResultSummaryForModel(toolName: toolName, toolResult: trimmed) if !summary.isEmpty, summary != trimmed { return summary } if trimmed.isEmpty { return "工具 \(toolName) 已执行,但没有返回内容。" } return """ 工具 \(toolName) 已执行完成,但模型没有生成最终回答。 工具返回结果: \(trimmed) """ } private func renderToolResultLocally( toolName: String, toolResult: String ) -> String? { guard let payload = parsedToolPayload(from: toolResult), let success = payload["success"] as? Bool, success else { return nil } func string(_ key: String) -> String? { if let value = payload[key] as? String, !value.isEmpty { return value } return nil } func int(_ key: String) -> Int? { if let value = payload[key] as? Int { return value } if let value = payload[key] as? Double { return Int(value) } if let value = payload[key] as? String, let intValue = Int(value) { return intValue } return nil } func double(_ key: String) -> Double? { if let value = payload[key] as? Double { return value } if let value = payload[key] as? Int { return Double(value) } if let value = payload[key] as? String, let doubleValue = Double(value) { return doubleValue } return nil } switch toolName { case "device-info": var lines: [String] = [] if let name = string("name") { lines.append("设备名称:\(name)") } if let localizedModel = string("localized_model") ?? string("model") { lines.append("设备类型:\(localizedModel)") } if let systemName = string("system_name"), let systemVersion = string("system_version") { lines.append("系统版本:\(systemName) \(systemVersion)") } else if let systemVersion = string("system_version") { lines.append("系统版本:\(systemVersion)") } if let memoryGB = double("memory_gb") { lines.append(String(format: "物理内存:%.1f GB", memoryGB)) } if let processorCount = int("processor_count") { lines.append("处理器核心数:\(processorCount)") } return lines.isEmpty ? nil : lines.joined(separator: "\n") case "device-name": if let name = string("name") { return "这台设备的名称是 \(name)。" } case "device-model": if let localizedModel = string("localized_model") ?? string("model") { return "这台设备的官方设备类型是 \(localizedModel)。" } case "device-system-version": if let systemName = string("system_name"), let systemVersion = string("system_version") { return "当前系统版本是 \(systemName) \(systemVersion)。" } case "device-memory": if let memoryGB = double("memory_gb") { return String(format: "这台设备的物理内存约为 %.1f GB。", memoryGB) } case "device-processor-count": if let processorCount = int("processor_count") { return "这台设备的处理器核心数是 \(processorCount)。" } case "device-identifier-for-vendor": if let identifier = string("identifier_for_vendor") { return "当前 App 在这台设备上的 identifierForVendor 是 \(identifier)。" } case "clipboard-read": if let content = string("content") { return "剪贴板当前内容是:\(content)" } case "clipboard-write": if let copiedLength = int("copied_length") { return "已写入剪贴板,共 \(copiedLength) 个字符。" } case "text-reverse": if let reversed = string("reversed") { return "翻转结果:\(reversed)" } case "calculate-hash": if let hash = payload["hash"] { return "哈希值是 \(hash)。" } case "calendar-create-event": if let title = string("title"), let start = string("start") { var parts = ["已创建日历事项“\(title)”", "开始时间是 \(start)"] if let location = string("location") { parts.append("地点是 \(location)") } return parts.joined(separator: ",") + "。" } case "reminders-create": if let title = string("title") { if let due = string("due") { return "已创建提醒事项“\(title)”,提醒时间是 \(due)。" } return "已创建提醒事项“\(title)”。" } case "contacts-upsert": if let name = string("name") { let action = string("action") == "updated" ? "已更新" : "已创建" var parts = ["\(action)联系人“\(name)”"] if let phone = string("phone") { parts.append("手机号是 \(phone)") } if let company = string("company") { parts.append("公司是 \(company)") } return parts.joined(separator: ",") + "。" } default: break } return nil } private func fallbackReplyForEmptySkillFollowUp(skillName: String) -> String { "Skill \(skillName) 已加载,但模型没有继续生成工具调用或最终回答。请重试,或把问题说得更具体一些。" } // MARK: - 初始化 /// ConfigurationsView 的"Restore default"按钮使用。 var defaultSystemPrompt: String { kDefaultSystemPrompt } func setup() { applyModelSelection() loadSystemPrompt() // 从 SYSPROMPT.md 注入 system prompt applySamplingConfig() llm.loadModel() } // MARK: - SYSPROMPT 注入 /// 从 ApplicationSupport/PhoneClaw/SYSPROMPT.md 读取 system prompt。 /// 文件不存在时自动写入 kDefaultSystemPrompt(供用户后续编辑)。 func loadSystemPrompt() { let fm = FileManager.default guard let supportDir = fm.urls(for: .applicationSupportDirectory, in: .userDomainMask).first else { return } let dir = supportDir.appendingPathComponent("PhoneClaw", isDirectory: true) let file = dir.appendingPathComponent("SYSPROMPT.md") if !fm.fileExists(atPath: dir.path) { try? fm.createDirectory(at: dir, withIntermediateDirectories: true) } if fm.fileExists(atPath: file.path), let content = try? String(contentsOf: file, encoding: .utf8), !content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { config.systemPrompt = content print("[Agent] SYSPROMPT loaded (\(content.count) chars)") } else { try? kDefaultSystemPrompt.write(to: file, atomically: true, encoding: .utf8) config.systemPrompt = kDefaultSystemPrompt print("[Agent] SYSPROMPT not found — default written to \(file.path)") } } func applySamplingConfig() { llm.samplingTopK = config.topK llm.samplingTopP = Float(config.topP) llm.samplingTemperature = Float(config.temperature) llm.maxOutputTokens = config.maxTokens } @discardableResult func applyModelSelection() -> Bool { UserDefaults.standard.set( config.selectedModelID, forKey: ModelConfig.selectedModelDefaultsKey ) return llm.selectModel(id: config.selectedModelID) } func reloadModel() { let selectedModelID = config.selectedModelID Task { [weak self] in guard let self else { return } self.isProcessing = false _ = self.llm.selectModel(id: selectedModelID) await self.llm.prepareForReload() self.llm.loadModel() } } func permissionStatuses() -> [AppPermissionKind: AppPermissionStatus] { toolRegistry.allPermissionStatuses() } func requestPermission(_ kind: AppPermissionKind) async -> AppPermissionStatus { do { _ = try await toolRegistry.requestAccess(for: kind) } catch { log("[Permission] \(kind.rawValue) request failed: \(error.localizedDescription)") } return toolRegistry.authorizationStatus(for: kind) } // MARK: - 处理用户输入(MLX 流式输出) func processInput(_ text: String, images: [UIImage] = []) async { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) let normalizedText = trimmed.isEmpty && !images.isEmpty ? "请描述这张图片。" : trimmed guard !normalizedText.isEmpty, !isProcessing else { return } guard llm.isLoaded else { messages.append(ChatMessage(role: .system, content: "⏳ 模型还在加载中...")) return } let attachments = images.compactMap(ChatImageAttachment.init(image:)) messages.append(ChatMessage(role: .user, content: normalizedText, images: attachments)) isProcessing = true applySamplingConfig() let activeSkillInfos = attachments.isEmpty ? enabledSkillInfos : [] let historyDepth = attachments.isEmpty ? llm.safeHistoryDepth : 0 print("[MEM] safeHistoryDepth=\(historyDepth), headroom=\(llm.availableHeadroomMB) MB") let promptImages = promptImages(historyDepth: historyDepth, currentImages: attachments) print("[VLM] userAttachments=\(attachments.count), promptImages=\(promptImages.count)") messages.append(ChatMessage(role: .assistant, content: "▍")) let msgIndex = messages.count - 1 if !attachments.isEmpty { let multimodalChat: [Chat.Message] = [ .system(PromptBuilder.multimodalSystemPrompt), .user( normalizedText, images: promptImages.map { .ciImage($0) } ), ] llm.generateStream(chat: multimodalChat) { [weak self] token in guard let self = self else { return } let updated = self.messages[msgIndex].content == "▍" ? token : self.messages[msgIndex].content.replacingOccurrences(of: "▍", with: "") + token self.messages[msgIndex].update(content: updated + "▍") } onComplete: { [weak self] result in guard let self = self else { return } switch result { case .success(let fullText): log("[Agent] 1st raw: \(fullText.prefix(300))") let cleaned = self.cleanOutput(fullText) self.messages[msgIndex].update( content: cleaned.isEmpty ? "(无回复)" : cleaned ) case .failure(let error): self.messages[msgIndex].update(role: .system, content: "❌ \(error.localizedDescription)") } self.isProcessing = false } return } let prompt = PromptBuilder.build( userMessage: normalizedText, currentImageCount: attachments.count, tools: activeSkillInfos, history: messages, systemPrompt: config.systemPrompt, historyDepth: historyDepth ) var detectedToolCall = false var buffer = "" var bufferFlushed = false llm.generateStream(prompt: prompt, images: promptImages) { [weak self] token in guard let self = self else { return } if detectedToolCall { buffer += token return } buffer += token if buffer.contains("") { detectedToolCall = true return } if !bufferFlushed { let trimmed = buffer.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.isEmpty { return } if "".hasPrefix(trimmed) { return } bufferFlushed = true self.messages[msgIndex].update(content: self.cleanOutputStreaming(buffer)) return } let cleaned = self.cleanOutputStreaming(buffer) if !cleaned.isEmpty { self.messages[msgIndex].update(content: cleaned) } } onComplete: { [weak self] result in guard let self = self else { return } switch result { case .success(let fullText): log("[Agent] 1st raw: \(fullText.prefix(300))") if self.parseToolCall(fullText) != nil { self.messages[msgIndex].update(content: "") Task { await self.executeToolChain( prompt: prompt, fullText: fullText, userQuestion: normalizedText, images: promptImages ) } return } else { let cleaned = self.cleanOutput(fullText) self.messages[msgIndex].update( content: cleaned.isEmpty ? "(无回复)" : cleaned ) } case .failure(let error): self.messages[msgIndex].update(role: .system, content: "❌ \(error.localizedDescription)") } self.isProcessing = false } } // MARK: - Skill 结果后的后续推理(支持多轮工具链) private func streamLLM(prompt: String, images: [CIImage]) async -> String? { return await withCheckedContinuation { (continuation: CheckedContinuation) in llm.generateStream(prompt: prompt, images: images) { _ in } onComplete: { result in switch result { case .success(let text): log("[Agent] LLM raw: \(text.prefix(300))") continuation.resume(returning: text) case .failure(let error): log("[Agent] LLM failed: \(error.localizedDescription)") continuation.resume(returning: nil) } } } } private func streamLLM(prompt: String, msgIndex: Int, images: [CIImage]) async -> String? { var buffer = "" return await withCheckedContinuation { (continuation: CheckedContinuation) in var toolCallDetected = false var bufferFlushed = false llm.generateStream(prompt: prompt, images: images) { [weak self] token in guard let self = self else { return } buffer += token if toolCallDetected { return } if buffer.contains("") { toolCallDetected = true if bufferFlushed && self.messages[msgIndex].role == .assistant { self.messages[msgIndex].update(content: "") } return } if !bufferFlushed { let trimmed = buffer.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.isEmpty { return } if "".hasPrefix(trimmed) { return } bufferFlushed = true } let cleaned = self.cleanOutputStreaming(buffer) if !cleaned.isEmpty && self.messages[msgIndex].role == .assistant { self.messages[msgIndex].update(content: cleaned) } } onComplete: { [weak self] result in guard let self = self else { continuation.resume(returning: nil) return } switch result { case .success(let text): log("[Agent] LLM raw: \(text.prefix(300))") continuation.resume(returning: text) case .failure(let error): self.messages[msgIndex].update(role: .system, content: "❌ \(error.localizedDescription)") continuation.resume(returning: nil) } } } } private func executeToolChain( prompt: String, fullText: String, userQuestion: String, images: [CIImage], round: Int = 1, maxRounds: Int = 10 ) async { guard round <= maxRounds else { log("[Agent] 达到最大工具链轮数 \(maxRounds)") isProcessing = false return } guard let call = parseToolCall(fullText) else { let cleaned = cleanOutput(fullText) if let lastAssistant = messages.lastIndex(where: { $0.role == .assistant }) { messages[lastAssistant].update(content: cleaned.isEmpty ? "(无回复)" : cleaned) } isProcessing = false return } log("[Agent] Round \(round): tool_call name=\(call.name)") // ── load_skill ── if call.name == "load_skill" { let allCalls = parseAllToolCalls(fullText) let loadSkillCalls = allCalls.filter { $0.name == "load_skill" } var allInstructions = "" var loadedDisplayNames: [String] = [] var loadedSkillIds: [String] = [] for lsCall in loadSkillCalls { let skillName = (lsCall.arguments["skill"] as? String) ?? (lsCall.arguments["name"] as? String) ?? "" log("[Agent] load_skill: \(skillName)") let displayName = findDisplayName(for: skillName) loadedDisplayNames.append(displayName) messages.append(ChatMessage(role: .system, content: "identified", skillName: displayName)) let cardIdx = messages.count - 1 guard let instructions = handleLoadSkill(skillName: skillName) else { messages[cardIdx].update(role: .system, content: "done", skillName: displayName) continue } try? await Task.sleep(for: .milliseconds(300)) messages[cardIdx].update(role: .system, content: "loaded", skillName: displayName) messages.append(ChatMessage(role: .skillResult, content: instructions, skillName: skillName)) allInstructions += instructions + "\n\n" loadedSkillIds.append(skillName) } guard !allInstructions.isEmpty else { isProcessing = false return } if let autoCall = autoToolCallForLoadedSkills(skillIds: loadedSkillIds) ?? inferFallbackToolCall(skillIds: loadedSkillIds, userQuestion: userQuestion) { log("[Agent] load_skill 直接执行工具: \(autoCall.name)") let syntheticToolCall = syntheticToolCallText( name: autoCall.name, arguments: autoCall.arguments ) await executeToolChain( prompt: prompt, fullText: syntheticToolCall, userQuestion: userQuestion, images: images, round: round + 1, maxRounds: maxRounds ) return } let singleToolExtraction = await extractSingleToolCallForLoadedSkill( originalPrompt: prompt, userQuestion: userQuestion, skillInstructions: allInstructions, skillIds: loadedSkillIds, images: images ) switch singleToolExtraction { case .toolCall(let name, let arguments): log("[Agent] load_skill 参数提取后执行工具: \(name)") let syntheticToolCall = syntheticToolCallText(name: name, arguments: arguments) await executeToolChain( prompt: prompt, fullText: syntheticToolCall, userQuestion: userQuestion, images: images, round: round + 1, maxRounds: maxRounds ) return case .needsClarification(let clarification): messages.append(ChatMessage(role: .assistant, content: clarification)) markSkillsDone(loadedDisplayNames) isProcessing = false return case .failed: break } let followUpPrompt = PromptBuilder.buildLoadedSkillPrompt( originalPrompt: prompt, userQuestion: userQuestion, skillInstructions: allInstructions, currentImageCount: images.count ) messages.append(ChatMessage(role: .assistant, content: "▍")) let followUpIndex = messages.count - 1 guard let nextText = await streamLLM(prompt: followUpPrompt, msgIndex: followUpIndex, images: images) else { isProcessing = false return } if parseToolCall(nextText) != nil { log("[Agent] load_skill 后检测到 tool 调用 (round \(round + 1))") messages[followUpIndex].update(content: "") await executeToolChain( prompt: followUpPrompt, fullText: nextText, userQuestion: userQuestion, images: images, round: round + 1, maxRounds: maxRounds ) } else { let cleaned = cleanOutput(nextText) if cleaned.isEmpty || looksLikeStructuredIntermediateOutput(cleaned) || looksLikePromptEcho(cleaned) { let retryPrompt = PromptBuilder.buildLoadedSkillPrompt( originalPrompt: prompt, userQuestion: userQuestion, skillInstructions: allInstructions, currentImageCount: images.count, forceResponse: true ) guard let retryText = await streamLLM(prompt: retryPrompt, msgIndex: followUpIndex, images: images) else { isProcessing = false return } if parseToolCall(retryText) != nil { log("[Agent] load_skill 重试后检测到 tool 调用 (round \(round + 1))") messages[followUpIndex].update(content: "") await executeToolChain( prompt: retryPrompt, fullText: retryText, userQuestion: userQuestion, images: images, round: round + 1, maxRounds: maxRounds ) } else { let retryCleaned = cleanOutput(retryText) let loadedSkillName = loadedDisplayNames.joined(separator: ", ").isEmpty ? "已加载的能力" : loadedDisplayNames.joined(separator: ", ") let finalReply = retryCleaned.isEmpty || looksLikeStructuredIntermediateOutput(retryCleaned) || looksLikePromptEcho(retryCleaned) ? fallbackReplyForEmptySkillFollowUp(skillName: loadedSkillName) : retryCleaned messages[followUpIndex].update(content: finalReply) markSkillsDone(loadedDisplayNames) isProcessing = false } } else { messages[followUpIndex].update(content: cleaned) markSkillsDone(loadedDisplayNames) isProcessing = false } } return } // ── 具体 Tool 调用 ── let ownerSkillId = findSkillId(for: call.name) let displayName = findDisplayName(for: call.name) let cardIndex: Int if let idx = messages.lastIndex(where: { $0.role == .system && ($0.skillName == displayName || $0.skillName == call.name) && ($0.content == "identified" || $0.content == "loaded") }) { cardIndex = idx } else { messages.append(ChatMessage(role: .system, content: "identified", skillName: displayName)) cardIndex = messages.count - 1 } guard ownerSkillId != nil else { messages[cardIndex].update(role: .system, content: "done", skillName: displayName) messages.append(ChatMessage(role: .assistant, content: "⚠️ 未知工具: \(call.name)")) isProcessing = false return } let enabledIds = Set(skillEntries.filter(\.isEnabled).map(\.id)) guard enabledIds.contains(ownerSkillId!) else { messages[cardIndex].update(role: .system, content: "done", skillName: displayName) messages.append(ChatMessage(role: .assistant, content: "⚠️ Skill \(displayName) 未启用")) isProcessing = false return } messages[cardIndex].update(role: .system, content: "executing:\(call.name)", skillName: displayName) do { let toolResult = try await handleToolExecution(toolName: call.name, args: call.arguments) let toolResultSummary = toolResultSummaryForModel(toolName: call.name, toolResult: toolResult) messages[cardIndex].update(role: .system, content: "done", skillName: displayName) messages.append(ChatMessage(role: .skillResult, content: toolResultSummary, skillName: call.name)) log("[Agent] Tool \(call.name) round \(round) done") let followUpPrompt = PromptBuilder.buildToolAnswerPrompt( originalPrompt: prompt, toolName: call.name, toolResultSummary: toolResultSummary, userQuestion: userQuestion, currentImageCount: images.count ) messages.append(ChatMessage(role: .assistant, content: "▍")) let followUpIndex = messages.count - 1 guard let nextText = await streamLLM(prompt: followUpPrompt, msgIndex: followUpIndex, images: images) else { isProcessing = false return } if !parseAllToolCalls(nextText).isEmpty { log("[Agent] 检测到第 \(round + 1) 轮工具调用") messages[followUpIndex].update(content: "") await executeToolChain( prompt: followUpPrompt, fullText: nextText, userQuestion: userQuestion, images: images, round: round + 1, maxRounds: maxRounds ) } else { let cleaned = cleanOutput(nextText) if cleaned.isEmpty || looksLikeStructuredIntermediateOutput(cleaned) || looksLikePromptEcho(cleaned) { messages[followUpIndex].update(content: fallbackReplyForEmptyToolFollowUp( toolName: call.name, toolResult: toolResult )) } else { messages[followUpIndex].update(content: cleaned) } isProcessing = false } } catch { messages[cardIndex].update(role: .system, content: "done", skillName: displayName) messages.append(ChatMessage(role: .system, content: "❌ Tool 执行失败: \(error)")) isProcessing = false } } // MARK: - 工具 func clearMessages() { messages.removeAll() } func cancelActiveGeneration() { guard isProcessing || llm.isGenerating else { return } llm.cancel() isProcessing = false if let lastAssistant = messages.lastIndex(where: { $0.role == .assistant }) { let content = messages[lastAssistant].content.replacingOccurrences(of: "▍", with: "") messages[lastAssistant].update(content: content.isEmpty ? "(已中断)" : content) } log("[Agent] Generation cancelled because the app left foreground") } private func promptImages( historyDepth: Int, currentImages: [ChatImageAttachment] ) -> [CIImage] { _ = historyDepth return Array(currentImages.prefix(1).compactMap(\.ciImage)) } func setAllSkills(enabled: Bool) { for i in skillEntries.indices { skillEntries[i].isEnabled = enabled } } // MARK: - 解析 private func parseToolCall(_ text: String) -> (name: String, arguments: [String: Any])? { return parseAllToolCalls(text).first } private func parseAllToolCalls(_ text: String) -> [(name: String, arguments: [String: Any])] { var results: [(name: String, arguments: [String: Any])] = [] let patterns = [ "\\s*(\\{.*?\\})\\s*", "```json\\s*(\\{.*?\\})\\s*```", "\\s*(\\{.*?\\})\\s*" ] for pattern in patterns { guard let regex = try? NSRegularExpression(pattern: pattern, options: .dotMatchesLineSeparators) else { continue } let range = NSRange(text.startIndex..., in: text) let matches = regex.matches(in: text, range: range) for match in matches { if let jsonRange = Range(match.range(at: 1), in: text) { let json = String(text[jsonRange]) if let data = json.data(using: .utf8), let dict = try? JSONSerialization.jsonObject(with: data) as? [String: Any], let name = dict["name"] as? String { results.append((name, dict["arguments"] as? [String: Any] ?? [:])) } } } if !results.isEmpty { break } } return results } private func extractSkillName(from text: String) -> String? { guard let regex = try? NSRegularExpression(pattern: "\"name\"\\s*:\\s*\"([^\"]+)\""), let match = regex.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)), let nameRange = Range(match.range(at: 1), in: text) else { return nil } return String(text[nameRange]) } private func cleanOutputStreaming(_ text: String) -> String { var result = text if let tcRange = result.range(of: "") { result = String(result[result.startIndex..", "", ""] for pat in endPatterns { if let range = result.range(of: pat) { result = String(result[result.startIndex...*?", options: .dotMatchesLineSeparators) { result = regex.stringByReplacingMatches(in: result, range: NSRange(result.startIndex..., in: result), withTemplate: "") } result = result.replacingOccurrences( of: "<\\|?[/a-z_]+\\|?>", with: "", options: .regularExpression ) if result.hasPrefix("model\n") { result = String(result.dropFirst(6)) } else if result == "model" { return "" } else if result.hasPrefix("user\n") { result = String(result.dropFirst(5)) } else if result == "user" { return "" } return String(result.drop(while: { $0.isWhitespace || $0.isNewline })) } private func cleanOutput(_ text: String) -> String { var result = text if let regex = try? NSRegularExpression(pattern: ".*?", options: .dotMatchesLineSeparators) { result = regex.stringByReplacingMatches(in: result, range: NSRange(result.startIndex..., in: result), withTemplate: "") } if let tcRange = result.range(of: "") { result = String(result[result.startIndex..", "", ""] for pat in endPatterns { if let range = result.range(of: pat) { result = String(result[result.startIndex..", with: "", options: .regularExpression ) if let lastOpen = result.lastIndex(of: "<") { let tail = String(result[lastOpen...]) let tailBody = tail.dropFirst() if !tailBody.isEmpty && tailBody.allSatisfy({ $0.isLetter || $0 == "_" || $0 == "/" || $0 == "|" }) { result = String(result[result.startIndex..