From 0d223ae373009da893f83beea6e575f15be04679 Mon Sep 17 00:00:00 2001 From: Kellyvv Date: Tue, 7 Apr 2026 01:50:29 +0800 Subject: [PATCH] feat: add audio input and stabilize gemma 4 inference --- Agent/AgentEngine.swift | 606 +++++++++++-- Agent/AudioCaptureService.swift | 250 ++++++ Info.plist | 2 + LLM/LLMEngine.swift | 13 +- LLM/MLX/Gemma4AudioFeatureExtractor.swift | 182 ++++ LLM/MLX/Gemma4AudioModel.swift | 555 ++++++++++++ LLM/MLX/Gemma4Config.swift | 137 +++ LLM/MLX/Gemma4Model.swift | 79 +- LLM/MLX/Gemma4Processor.swift | 218 ++++- LLM/MLX/Gemma4Registration.swift | 29 +- LLM/MLX/Gemma4VisionModel.swift | 50 +- LLM/MLXLocalLLMService.swift | 539 ++++++++++- LLM/PromptBuilder.swift | 93 +- .../Libraries/MLXLMCommon/Chat.swift | 28 +- .../Libraries/MLXLMCommon/LanguageModel.swift | 39 +- .../Libraries/MLXLMCommon/UserInput.swift | 56 +- PhoneClaw.xcodeproj/project.pbxproj | 34 +- README.md | 6 + Skills/ToolRegistry.swift | 27 + UI/ChatModels.swift | 53 +- UI/ConfigurationsView.swift | 4 + UI/ContentView.swift | 839 +++++++++++++++++- 22 files changed, 3622 insertions(+), 217 deletions(-) create mode 100644 Agent/AudioCaptureService.swift create mode 100644 LLM/MLX/Gemma4AudioFeatureExtractor.swift create mode 100644 LLM/MLX/Gemma4AudioModel.swift diff --git a/Agent/AgentEngine.swift b/Agent/AgentEngine.swift index edf9f6c..8168f6a 100644 --- a/Agent/AgentEngine.swift +++ b/Agent/AgentEngine.swift @@ -7,17 +7,31 @@ func log(_ message: String) { print(message) } +private extension UserInput.Audio { + static func from(snapshot: AudioCaptureSnapshot) -> UserInput.Audio { + .pcm( + .init( + samples: snapshot.pcm, + sampleRate: snapshot.sampleRate, + channelCount: snapshot.channelCount + ) + ) + } +} + // MARK: - 模型/推理配置 @Observable class ModelConfig { static let selectedModelDefaultsKey = "PhoneClaw.selectedModelID" + static let enableThinkingDefaultsKey = "PhoneClaw.enableThinking" var maxTokens = 4000 var topK = 64 var topP = 0.95 var temperature = 1.0 var useGPU = true + var enableThinking = UserDefaults.standard.bool(forKey: enableThinkingDefaultsKey) var selectedModelID = UserDefaults.standard.string(forKey: selectedModelDefaultsKey) ?? MLXLocalLLMService.defaultModel.id /// System prompt — 由 AgentEngine.loadSystemPrompt() 从 SYSPROMPT.md 注入，不在代码里硬编码。 @@ -52,17 +66,42 @@ ___SKILLS___ struct ChatImageAttachment: Identifiable { let id = UUID() let data: Data + private static let storageMaxDimension: CGFloat = 1_024 + private static let compressionQuality: CGFloat = 0.78 init?(image: UIImage) { - if let jpeg = image.jpegData(compressionQuality: 0.92) { + let prepared = Self.preparedImage(image, maxDimension: Self.storageMaxDimension) + if let jpeg = prepared.jpegData(compressionQuality: Self.compressionQuality) { self.data = jpeg - } else if let png = image.pngData() { + } else if let png = prepared.pngData() { self.data = png } else { return nil } } + static func preparedImage(_ image: UIImage, maxDimension: CGFloat = storageMaxDimension) -> UIImage { + let originalSize = image.size + let longestSide = max(originalSize.width, originalSize.height) + guard longestSide > maxDimension, longestSide > 0 else { + return image + } + + let scale = maxDimension / longestSide + let targetSize = CGSize( + width: max(1, floor(originalSize.width * scale)), + height: max(1, floor(originalSize.height * scale)) + ) + + let format = UIGraphicsImageRendererFormat.default() + format.opaque = false + format.scale = 1 + let renderer = UIGraphicsImageRenderer(size: targetSize, format: format) + return renderer.image { _ in + image.draw(in: CGRect(origin: .zero, size: targetSize)) + } + } + var uiImage: UIImage? { UIImage(data: data) } @@ -85,11 +124,113 @@ struct ChatImageAttachment: Identifiable { } } +struct ChatAudioAttachment: Identifiable { + let id = UUID() + let wavData: Data + let duration: TimeInterval + let sampleRate: Double + let waveform: [Float] + + init?(snapshot: AudioCaptureSnapshot) { + guard !snapshot.pcm.isEmpty, snapshot.sampleRate > 0 else { return nil } + self.wavData = Self.makeWAVData( + pcm: snapshot.pcm, + sampleRate: snapshot.sampleRate, + channelCount: 1 + ) + self.duration = snapshot.duration + self.sampleRate = snapshot.sampleRate + self.waveform = Self.makeWaveform(from: snapshot.pcm) + } + + var formattedDuration: String { + let totalSeconds = Int(duration.rounded()) + let minutes = totalSeconds / 60 + let seconds = totalSeconds % 60 + return String(format: "%02d:%02d", minutes, seconds) + } + + private static func makeWaveform(from pcm: [Float], bucketCount: Int = 36) -> [Float] { + guard !pcm.isEmpty else { return Array(repeating: 0.12, count: bucketCount) } + let samplesPerBucket = max(pcm.count / bucketCount, 1) + var levels: [Float] = [] + levels.reserveCapacity(bucketCount) + + var index = 0 + while index < pcm.count { + let end = min(index + samplesPerBucket, pcm.count) + let slice = pcm[index.. bucketCount { + levels = Array(levels.prefix(bucketCount)) + } + + let maxLevel = max(levels.max() ?? 0, 0.001) + return levels.map { max($0 / maxLevel, 0.08) } + } + + private static func makeWAVData( + pcm: [Float], + sampleRate: Double, + channelCount: Int + ) -> Data { + let integerSampleRate = max(Int(sampleRate.rounded()), 1) + let clampedSamples = pcm.map { sample -> Int16 in + let limited = min(max(sample, -1), 1) + return Int16((limited * Float(Int16.max)).rounded()) + } + + let bytesPerSample = MemoryLayout.size + let dataChunkSize = clampedSamples.count * bytesPerSample + let riffChunkSize = 36 + dataChunkSize + let byteRate = integerSampleRate * channelCount * bytesPerSample + let blockAlign = channelCount * bytesPerSample + + var data = Data() + data.reserveCapacity(44 + dataChunkSize) + data.append("RIFF".data(using: .ascii)!) + append(UInt32(riffChunkSize), to: &data) + data.append("WAVE".data(using: .ascii)!) + data.append("fmt ".data(using: .ascii)!) + append(UInt32(16), to: &data) + append(UInt16(1), to: &data) + append(UInt16(channelCount), to: &data) + append(UInt32(integerSampleRate), to: &data) + append(UInt32(byteRate), to: &data) + append(UInt16(blockAlign), to: &data) + append(UInt16(bytesPerSample * 8), to: &data) + data.append("data".data(using: .ascii)!) + append(UInt32(dataChunkSize), to: &data) + + for sample in clampedSamples { + append(sample, to: &data) + } + return data + } + + private static func append(_ value: T, to data: inout Data) { + var littleEndian = value.littleEndian + withUnsafeBytes(of: &littleEndian) { bytes in + data.append(contentsOf: bytes) + } + } +} + struct ChatMessage: Identifiable { let id: UUID var role: Role var content: String var images: [ChatImageAttachment] + var audios: [ChatAudioAttachment] let timestamp = Date() var skillName: String? = nil @@ -97,12 +238,14 @@ struct ChatMessage: Identifiable { role: Role, content: String, images: [ChatImageAttachment] = [], + audios: [ChatAudioAttachment] = [], skillName: String? = nil ) { self.id = UUID() self.role = role self.content = content self.images = images + self.audios = audios self.skillName = skillName } @@ -139,6 +282,9 @@ class AgentEngine { // Skill 条目（给 UI 管理用，可开关） var skillEntries: [SkillEntry] = [] + private let thinkingOpenMarker = "[[PHONECLAW_THINK]]" + private let thinkingCloseMarker = "[[/PHONECLAW_THINK]]" + var enabledSkillInfos: [SkillInfo] { skillEntries.filter(\.isEnabled).map { @@ -347,6 +493,28 @@ class AgentEngine { } else { candidateNames = ["clipboard-read", "clipboard-write"] } + case "calendar": + let actionKeywords = [ + "安排", "创建", "新建", "加入", "加到", "写进", "写入", "记到" + ] + let createIntent = + (has("日历") || has("日程") || has("会议") || has("约会")) + && actionKeywords.contains(where: has) + candidateNames = createIntent ? ["calendar-create-event"] : [] + case "reminders": + let actionKeywords = [ + "提醒", "提醒我", "记得", "待办", "创建", "新建", "添加" + ] + let createIntent = actionKeywords.contains(where: has) + candidateNames = createIntent ? ["reminders-create"] : [] + case "text": + if has("哈希") || has("hash") { + candidateNames = ["calculate-hash"] + } else if has("翻转") || has("反转") { + candidateNames = ["text-reverse"] + } else { + candidateNames = ["calculate-hash", "text-reverse"] + } default: candidateNames = [] } @@ -575,6 +743,91 @@ class AgentEngine { guard !text.isEmpty else { return nil } switch toolName { + case "calculate-hash": + let patterns = [ + "(?:计算|求|生成|算一下|帮我算)(.+?)(?:的)?(?:哈希|hash)(?:值)?", + "(.+?)(?:的)?(?:哈希|hash)(?:值)?(?:是多少|是什么)?" + ] + + var extracted: String? + for pattern in patterns { + guard let regex = try? NSRegularExpression(pattern: pattern, options: [.caseInsensitive]) else { continue } + let range = NSRange(text.startIndex..., in: text) + if let match = regex.firstMatch(in: text, range: range), + let capture = Range(match.range(at: 1), in: text) { + let value = String(text[capture]) + .trimmingCharacters(in: .whitespacesAndNewlines) + .trimmingCharacters(in: CharacterSet(charactersIn: "“”\"' ")) + if !value.isEmpty { + extracted = value + break + } + } + } + + if extracted == nil { + let cleaned = text + .replacingOccurrences(of: "帮我", with: "") + .replacingOccurrences(of: "计算", with: "") + .replacingOccurrences(of: "求", with: "") + .replacingOccurrences(of: "生成", with: "") + .replacingOccurrences(of: "算一下", with: "") + .replacingOccurrences(of: "哈希值", with: "") + .replacingOccurrences(of: "哈希", with: "") + .replacingOccurrences(of: "hash", with: "", options: .caseInsensitive) + .trimmingCharacters(in: .whitespacesAndNewlines) + .trimmingCharacters(in: CharacterSet(charactersIn: "“”\"' ")) + if !cleaned.isEmpty { + extracted = cleaned + } + } + + guard let extracted, !extracted.isEmpty else { return nil } + return ["text": extracted] + + case "text-reverse": + let patterns = [ + "(?:把|将)(.+?)(?:翻转|反转)(?:过来|一下)?", + "(?:翻转|反转)(.+?)(?:过来|一下)?$" + ] + + var extracted: String? + for pattern in patterns { + guard let regex = try? NSRegularExpression(pattern: pattern) else { continue } + let range = NSRange(text.startIndex..., in: text) + if let match = regex.firstMatch(in: text, range: range), + let capture = Range(match.range(at: 1), in: text) { + let value = String(text[capture]) + .trimmingCharacters(in: .whitespacesAndNewlines) + .trimmingCharacters(in: CharacterSet(charactersIn: "“”\"' ")) + if !value.isEmpty { + extracted = value + break + } + } + } + + if extracted == nil { + let cleaned = text + .replacingOccurrences(of: "帮我", with: "") + .replacingOccurrences(of: "把", with: "") + .replacingOccurrences(of: "将", with: "") + .replacingOccurrences(of: "翻转过来", with: "") + .replacingOccurrences(of: "反转过来", with: "") + .replacingOccurrences(of: "翻转一下", with: "") + .replacingOccurrences(of: "反转一下", with: "") + .replacingOccurrences(of: "翻转", with: "") + .replacingOccurrences(of: "反转", with: "") + .trimmingCharacters(in: .whitespacesAndNewlines) + .trimmingCharacters(in: CharacterSet(charactersIn: "“”\"' ")) + if !cleaned.isEmpty { + extracted = cleaned + } + } + + guard let extracted, !extracted.isEmpty else { return nil } + return ["text": extracted] + case "clipboard-write": let patterns = [ "(?:复制|拷贝|写入)(.+?)(?:到|进|到系统)?(?:剪贴板)", @@ -904,6 +1157,8 @@ class AgentEngine { return arguments["title"] is String && arguments["start"] is String case "reminders-create": return arguments["title"] is String + case "calculate-hash", "text-reverse": + return arguments["text"] is String case "contacts-upsert": return arguments["name"] is String case "contacts-search", "contacts-delete": @@ -935,25 +1190,75 @@ class AgentEngine { normalizedQuestion.contains(keyword) } - let mentionsClipboard = has("剪贴板") || has("clipboard") - guard mentionsClipboard else { return nil } - - let readKeywords = ["读取", "读一下", "看看", "看下", "查看", "内容", "是什么"] - let writeKeywords = ["复制", "拷贝", "写入", "放到剪贴板", "存到剪贴板", "复制到剪贴板"] - - if writeKeywords.contains(where: has), - let arguments = heuristicArgumentsForTool(toolName: "clipboard-write", userQuestion: userQuestion), - validateSingleToolArguments(toolName: "clipboard-write", arguments: arguments) { + if (has("哈希") || has("hash")), + let arguments = heuristicArgumentsForTool(toolName: "calculate-hash", userQuestion: userQuestion), + validateSingleToolArguments(toolName: "calculate-hash", arguments: arguments) { return syntheticToolCallText( - name: "clipboard-write", + name: "calculate-hash", arguments: arguments ) } - if readKeywords.contains(where: has) || mentionsClipboard { + if (has("翻转") || has("反转")), + let arguments = heuristicArgumentsForTool(toolName: "text-reverse", userQuestion: userQuestion), + validateSingleToolArguments(toolName: "text-reverse", arguments: arguments) { + return syntheticToolCallText( + name: "text-reverse", + arguments: arguments + ) + } + + let mentionsClipboard = has("剪贴板") || has("clipboard") + if mentionsClipboard { + let readKeywords = ["读取", "读一下", "看看", "看下", "查看", "内容", "是什么"] + let writeKeywords = ["复制", "拷贝", "写入", "放到剪贴板", "存到剪贴板", "复制到剪贴板"] + + if writeKeywords.contains(where: has), + let arguments = heuristicArgumentsForTool(toolName: "clipboard-write", userQuestion: userQuestion), + validateSingleToolArguments(toolName: "clipboard-write", arguments: arguments) { + return syntheticToolCallText( + name: "clipboard-write", + arguments: arguments + ) + } + + if readKeywords.contains(where: has) || mentionsClipboard { + return syntheticToolCallText( + name: "load_skill", + arguments: ["skill": "clipboard"] + ) + } + } + + let calendarIntent = + (has("日历") || has("日程") || has("会议") || has("约会")) + && ["安排", "创建", "新建", "加入", "加到", "记到", "写进", "写入"].contains(where: has) + if calendarIntent { + if let arguments = heuristicArgumentsForTool(toolName: "calendar-create-event", userQuestion: userQuestion), + validateSingleToolArguments(toolName: "calendar-create-event", arguments: arguments) { + return syntheticToolCallText( + name: "calendar-create-event", + arguments: arguments + ) + } return syntheticToolCallText( name: "load_skill", - arguments: ["skill": "clipboard"] + arguments: ["skill": "calendar"] + ) + } + + let reminderIntent = ["提醒我", "提醒", "记得", "待办", "提醒事项"].contains(where: has) + if reminderIntent { + if let arguments = heuristicArgumentsForTool(toolName: "reminders-create", userQuestion: userQuestion), + validateSingleToolArguments(toolName: "reminders-create", arguments: arguments) { + return syntheticToolCallText( + name: "reminders-create", + arguments: arguments + ) + } + return syntheticToolCallText( + name: "load_skill", + arguments: ["skill": "reminders"] ) } @@ -980,6 +1285,12 @@ class AgentEngine { } if tools.count == 1, let tool = tools.first { + if let heuristic = heuristicArgumentsForTool(toolName: tool.name, userQuestion: userQuestion), + validateSingleToolArguments(toolName: tool.name, arguments: heuristic) { + log("[Agent] load_skill heuristic 直接执行工具: \(tool.name)") + return .toolCall(name: tool.name, arguments: heuristic) + } + let extractionPrompt = PromptBuilder.buildSingleToolArgumentsPrompt( originalPrompt: originalPrompt, userQuestion: userQuestion, @@ -1003,10 +1314,6 @@ class AgentEngine { } } - if let heuristic = heuristicArgumentsForTool(toolName: tool.name, userQuestion: userQuestion), - validateSingleToolArguments(toolName: tool.name, arguments: heuristic) { - return .toolCall(name: tool.name, arguments: heuristic) - } return .failed } @@ -1364,6 +1671,51 @@ class AgentEngine { "Skill \(skillName) 已加载，但模型没有继续生成工具调用或最终回答。请重试，或把问题说得更具体一些。" } + private func shouldUseToolingPrompt(for userQuestion: String) -> Bool { + let normalizedQuestion = userQuestion.lowercased() + guard !normalizedQuestion.isEmpty else { return false } + + let domainKeywords = [ + "日历", "提醒", "提醒事项", "通讯录", "联系人", "剪贴板", + "设备", "系统", "相册", "照片", "蓝牙", "wifi", "电量", + "亮度", "音量", "电话", "短信", "邮件" + ] + let actionKeywords = [ + "打开", "查看", "读取", "搜索", "查询", "创建", + "新建", "添加", "保存", "修改", "删除", "复制", + "拨打", "发送", "设置", "调高", "调低" + ] + + let mentionsDomain = domainKeywords.contains { normalizedQuestion.contains($0) } + let mentionsAction = actionKeywords.contains { normalizedQuestion.contains($0) } + if mentionsDomain && mentionsAction { + return true + } + + for entry in skillEntries where entry.isEnabled { + if normalizedQuestion.contains(entry.id.lowercased()) + || normalizedQuestion.contains(entry.name.lowercased()) + { + return true + } + + if entry.tools.contains(where: { normalizedQuestion.contains($0.name.lowercased()) }) { + return true + } + + if let definition = skillLoader.getDefinition(entry.id), + definition.metadata.triggers.contains(where: { trigger in + let normalizedTrigger = trigger.lowercased().trimmingCharacters(in: .whitespacesAndNewlines) + return !normalizedTrigger.isEmpty && normalizedQuestion.contains(normalizedTrigger) + }) + { + return true + } + } + + return mentionsDomain + } + // MARK: - 初始化 /// ConfigurationsView 的"Restore default"按钮使用。 @@ -1409,6 +1761,10 @@ class AgentEngine { llm.samplingTopP = Float(config.topP) llm.samplingTemperature = Float(config.temperature) llm.maxOutputTokens = config.maxTokens + UserDefaults.standard.set( + config.enableThinking, + forKey: ModelConfig.enableThinkingDefaultsKey + ) } @discardableResult @@ -1446,45 +1802,78 @@ class AgentEngine { // MARK: - 处理用户输入（MLX 流式输出） - func processInput(_ text: String, images: [UIImage] = []) async { + func processInput( + _ text: String, + images: [UIImage] = [], + audio: AudioCaptureSnapshot? = nil + ) async { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) - let normalizedText = trimmed.isEmpty && !images.isEmpty ? "请描述这张图片。" : trimmed - guard !normalizedText.isEmpty, !isProcessing else { return } - guard llm.isLoaded else { - messages.append(ChatMessage(role: .system, content: "⏳ 模型还在加载中...")) - return - } - + let displayText = trimmed let attachments = images.compactMap(ChatImageAttachment.init(image:)) - messages.append(ChatMessage(role: .user, content: normalizedText, images: attachments)) + let audioClips = audio.flatMap(ChatAudioAttachment.init(snapshot:)).map { [$0] } ?? [] + let audioAttachment = audio.map(UserInput.Audio.from(snapshot:)) + let normalizedText: String + if trimmed.isEmpty, !images.isEmpty { + normalizedText = "请描述这张图片。" + } else if trimmed.isEmpty, audio != nil { + normalizedText = "请直接转写这段音频内容。" + } else { + normalizedText = trimmed + } + let requiresMultimodal = !attachments.isEmpty || audioAttachment != nil + guard !isProcessing else { return } + guard !normalizedText.isEmpty || !attachments.isEmpty || audioAttachment != nil else { return } + messages.append( + ChatMessage( + role: .user, + content: displayText, + images: attachments, + audios: audioClips + ) + ) isProcessing = true applySamplingConfig() - let activeSkillInfos = attachments.isEmpty ? enabledSkillInfos : [] - let historyDepth = attachments.isEmpty ? llm.safeHistoryDepth : 0 + let preflightToolCall = requiresMultimodal ? nil : preflightSkillLoadCall(for: normalizedText) + let shouldUseFullAgentPrompt = + !requiresMultimodal + && (preflightToolCall != nil || shouldUseToolingPrompt(for: normalizedText)) + let activeSkillInfos = shouldUseFullAgentPrompt ? enabledSkillInfos : [] + let historyDepth = requiresMultimodal ? 0 : llm.safeHistoryDepth print("[MEM] safeHistoryDepth=\(historyDepth), headroom=\(llm.availableHeadroomMB) MB") let promptImages = promptImages(historyDepth: historyDepth, currentImages: attachments) - print("[VLM] userAttachments=\(attachments.count), promptImages=\(promptImages.count)") + print( + "[VLM] userAttachments=\(attachments.count), promptImages=\(promptImages.count), " + + "audio=\(audioAttachment == nil ? 0 : 1)" + ) messages.append(ChatMessage(role: .assistant, content: "▍")) let msgIndex = messages.count - 1 - if !attachments.isEmpty { + if requiresMultimodal { let multimodalChat: [Chat.Message] = [ - .system(PromptBuilder.multimodalSystemPrompt), + .system( + PromptBuilder.multimodalSystemPrompt( + hasImages: !promptImages.isEmpty, + hasAudio: audioAttachment != nil + ) + ), .user( normalizedText, - images: promptImages.map { .ciImage($0) } + images: promptImages.map { .ciImage($0) }, + audios: audioAttachment.map { [$0] } ?? [] ), ] + let multimodalContext: [String: any Sendable]? = + config.enableThinking ? ["enable_thinking": true] : nil + var multimodalBuffer = "" - llm.generateStream(chat: multimodalChat) { [weak self] token in + llm.generateStream(chat: multimodalChat, additionalContext: multimodalContext) { [weak self] token in guard let self = self else { return } - let updated = self.messages[msgIndex].content == "▍" - ? token - : self.messages[msgIndex].content.replacingOccurrences(of: "▍", with: "") + token - self.messages[msgIndex].update(content: updated + "▍") + multimodalBuffer += token + let cleaned = self.cleanOutputStreaming(multimodalBuffer) + self.messages[msgIndex].update(content: (cleaned.isEmpty ? "" : cleaned) + "▍") } onComplete: { [weak self] result in guard let self = self else { return } switch result { @@ -1495,6 +1884,7 @@ class AgentEngine { content: cleaned.isEmpty ? "（无回复）" : cleaned ) case .failure(let error): + log("[Agent] multimodal failed: \(error.localizedDescription)") self.messages[msgIndex].update(role: .system, content: "❌ \(error.localizedDescription)") } self.isProcessing = false @@ -1502,16 +1892,29 @@ class AgentEngine { return } - let prompt = PromptBuilder.build( - userMessage: normalizedText, - currentImageCount: attachments.count, - tools: activeSkillInfos, - history: messages, - systemPrompt: config.systemPrompt, - historyDepth: historyDepth - ) + let prompt: String + if shouldUseFullAgentPrompt { + prompt = PromptBuilder.build( + userMessage: normalizedText, + currentImageCount: attachments.count, + tools: activeSkillInfos, + history: messages, + systemPrompt: config.systemPrompt, + enableThinking: config.enableThinking, + historyDepth: historyDepth + ) + } else { + prompt = PromptBuilder.buildLightweightTextPrompt( + userMessage: normalizedText, + history: messages, + systemPrompt: config.systemPrompt, + enableThinking: config.enableThinking, + historyDepth: historyDepth + ) + } + log("[Agent] text prompt mode=\(shouldUseFullAgentPrompt ? "agent" : "light"), chars=\(prompt.count), skills=\(activeSkillInfos.count)") - if let preflightToolCall = preflightSkillLoadCall(for: normalizedText) { + if let preflightToolCall { log("[Agent] preflight tool path triggered") if messages.indices.contains(msgIndex), messages[msgIndex].role == .assistant, @@ -1531,7 +1934,7 @@ class AgentEngine { var buffer = "" var bufferFlushed = false - llm.generateStream(prompt: prompt, images: promptImages) { [weak self] token in + llm.generateStream(prompt: prompt, images: promptImages, audios: []) { [weak self] token in guard let self = self else { return } if detectedToolCall { @@ -1593,7 +1996,7 @@ class AgentEngine { private func streamLLM(prompt: String, images: [CIImage]) async -> String? { return await withCheckedContinuation { (continuation: CheckedContinuation) in - llm.generateStream(prompt: prompt, images: images) { _ in + llm.generateStream(prompt: prompt, images: images, audios: []) { _ in } onComplete: { result in switch result { case .success(let text): @@ -1612,7 +2015,7 @@ class AgentEngine { return await withCheckedContinuation { (continuation: CheckedContinuation) in var toolCallDetected = false var bufferFlushed = false - llm.generateStream(prompt: prompt, images: images) { [weak self] token in + llm.generateStream(prompt: prompt, images: images, audios: []) { [weak self] token in guard let self = self else { return } buffer += token @@ -2008,7 +2411,7 @@ class AgentEngine { } private func cleanOutputStreaming(_ text: String) -> String { - var result = text + var result = preserveThinkingChannels(in: text) if let tcRange = result.range(of: "") { result = String(result[result.startIndex.. String { - var result = text + var result = preserveThinkingChannels(in: text) if let regex = try? NSRegularExpression(pattern: ".*?", options: .dotMatchesLineSeparators) { result = regex.stringByReplacingMatches(in: result, range: NSRange(result.startIndex..., in: result), withTemplate: "") @@ -2088,6 +2492,100 @@ class AgentEngine { result = "" } - return result.trimmingCharacters(in: .whitespacesAndNewlines) + result = result.trimmingCharacters(in: .whitespacesAndNewlines) + return normalizeSafetyTruncation(in: result) + } + + private func normalizeSafetyTruncation(in text: String) -> String { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard let warningRange = trimmed.range(of: "> ⚠️ ") else { + return trimmed + } + + let body = String(trimmed[.. String { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return trimmed } + + if let paragraphBreak = trimmed.range(of: "\n\n", options: .backwards) { + let tailLength = trimmed.distance(from: paragraphBreak.upperBound, to: trimmed.endIndex) + if tailLength > 0 && tailLength <= 280 { + return String(trimmed[.. 0 && tailLength <= 220 { + return String(trimmed[.. String.Index? { + let sentenceEndings: Set = ["。", "！", "？", ".", "!", "?"] + var index = text.endIndex + while index > text.startIndex { + index = text.index(before: index) + if sentenceEndings.contains(text[index]) { + return text.index(after: index) + } + } + return nil + } + + private func preserveThinkingChannels(in text: String) -> String { + let openTokens = ["<|channel|>thought\n", "<|channel>thought\n"] + let closeToken = "" + + var result = "" + var cursor = text.startIndex + + while cursor < text.endIndex { + let nextOpen = openTokens + .compactMap { token -> (Range, String)? in + guard let range = text.range(of: token, range: cursor.. Bool { + if isCapturing { + stopCapture() + return true + } else { + return await startCapture() + } + } + + @discardableResult + func startCapture() async -> Bool { + refreshPermissionStatus() + if permissionStatus == .notDetermined { + let granted = await requestPermission() + guard granted else { + lastErrorMessage = "麦克风权限未授予，无法开始录音。" + return false + } + } + + guard permissionStatus.isGranted else { + lastErrorMessage = "麦克风权限不可用，请到系统设置中开启。" + return false + } + + guard !isCapturing else { return true } + + resetCaptureState() + lastErrorMessage = nil + statusText = "准备录音..." + + do { + try audioSession.setCategory( + .playAndRecord, + mode: .measurement, + options: [.defaultToSpeaker, .allowBluetooth] + ) + try audioSession.setPreferredSampleRate(Self.preferredSampleRate) + try audioSession.setActive(true) + + let inputNode = audioEngine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + sampleRate = inputFormat.sampleRate + channelCount = Int(inputFormat.channelCount) + + inputNode.removeTap(onBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 1024, format: inputFormat) { [weak self] buffer, _ in + self?.handleIncomingPCM(buffer) + } + + audioEngine.prepare() + try audioEngine.start() + + isCapturing = true + updateStatusText() + return true + } catch { + stopCapture(deactivateSession: false) + lastErrorMessage = "启动录音失败：\(error.localizedDescription)" + statusText = lastErrorMessage ?? "" + return false + } + } + + @discardableResult + func stopCapture(deactivateSession: Bool = true) -> AudioCaptureSnapshot? { + audioEngine.inputNode.removeTap(onBus: 0) + audioEngine.stop() + if deactivateSession { + try? audioSession.setActive(false, options: [.notifyOthersOnDeactivation]) + } + + let snapshot = latestSnapshot() + isCapturing = false + peakLevel = 0 + + if let snapshot, snapshot.duration > 0 { + statusText = String( + format: "已录制 %.1f 秒音频（%.0f Hz，%d 声道），可以直接发送给模型。", + snapshot.duration, + snapshot.sampleRate, + snapshot.channelCount + ) + } else if lastErrorMessage == nil { + statusText = "" + } + + return snapshot + } + + func clearStatus() { + statusText = "" + lastErrorMessage = nil + } + + func consumeLatestSnapshot() -> AudioCaptureSnapshot? { + let snapshot = latestSnapshot() + resetCaptureState() + clearStatus() + return snapshot + } + + func latestSnapshot() -> AudioCaptureSnapshot? { + pcmLock.lock() + let pcm = rollingPCM + pcmLock.unlock() + + guard !pcm.isEmpty, sampleRate > 0 else { return nil } + return AudioCaptureSnapshot( + pcm: pcm, + sampleRate: sampleRate, + channelCount: 1, + duration: Double(pcm.count) / sampleRate + ) + } + + private func requestPermission() async -> Bool { + let granted = await withCheckedContinuation { continuation in + audioSession.requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + refreshPermissionStatus() + return granted + } + + private func resetCaptureState() { + pcmLock.lock() + rollingPCM.removeAll(keepingCapacity: true) + pcmLock.unlock() + + capturedSampleCount = 0 + bufferedSampleCount = 0 + duration = 0 + peakLevel = 0 + sampleRate = 0 + channelCount = 0 + } + + private func handleIncomingPCM(_ buffer: AVAudioPCMBuffer) { + let frameCount = Int(buffer.frameLength) + guard frameCount > 0 else { return } + let formatChannelCount = Int(buffer.format.channelCount) + guard let channelData = buffer.floatChannelData else { return } + + let samples: [Float] + if formatChannelCount <= 1 { + samples = Array(UnsafeBufferPointer(start: channelData[0], count: frameCount)) + } else { + var mixed = Array(repeating: Float.zero, count: frameCount) + let scale = 1.0 / Float(formatChannelCount) + for channel in 0 ..< formatChannelCount { + let channelSamples = UnsafeBufferPointer(start: channelData[channel], count: frameCount) + for index in 0 ..< frameCount { + mixed[index] += channelSamples[index] * scale + } + } + samples = mixed + } + let rms = sqrt(samples.reduce(0) { $0 + ($1 * $1) } / Float(max(frameCount, 1))) + let peak = samples.map { abs($0) }.max() ?? 0 + + pcmLock.lock() + rollingPCM.append(contentsOf: samples) + let maxStoredSamples = Int(max(sampleRate, Self.preferredSampleRate) * Self.maxStoredSeconds) + if rollingPCM.count > maxStoredSamples { + rollingPCM.removeFirst(rollingPCM.count - maxStoredSamples) + } + let bufferedCount = rollingPCM.count + pcmLock.unlock() + + Task { @MainActor [weak self] in + guard let self else { return } + capturedSampleCount += frameCount + bufferedSampleCount = bufferedCount + if sampleRate == 0 { + sampleRate = buffer.format.sampleRate + } + if channelCount == 0 { + channelCount = 1 + } + if sampleRate > 0 { + duration = Double(capturedSampleCount) / sampleRate + } + peakLevel = max(rms, peak) + updateStatusText() + } + } + + private func updateStatusText() { + guard isCapturing else { return } + statusText = String( + format: "录音中 %.1f 秒 · %.0f Hz · 缓冲 %.1f 秒 PCM", + duration, + max(sampleRate, Self.preferredSampleRate), + sampleRate > 0 ? Double(bufferedSampleCount) / sampleRate : 0 + ) + } +} diff --git a/Info.plist b/Info.plist index 476863b..ce6aced 100644 --- a/Info.plist +++ b/Info.plist @@ -42,5 +42,7 @@ PhoneClaw 需要访问提醒事项 NSContactsUsageDescription PhoneClaw 需要存储联系人信息 + NSMicrophoneUsageDescription + PhoneClaw 需要麦克风来录制语音并采集音频输入 diff --git a/LLM/LLMEngine.swift b/LLM/LLMEngine.swift index aecb6c5..9a7f65e 100644 --- a/LLM/LLMEngine.swift +++ b/LLM/LLMEngine.swift @@ -1,5 +1,6 @@ import Foundation import CoreImage +import MLXLMCommon // MARK: - LLM Engine Protocol @@ -7,7 +8,11 @@ import CoreImage public protocol LLMEngine { func load() async throws func warmup() async throws - func generateStream(prompt: String, images: [CIImage]) -> AsyncThrowingStream + func generateStream( + prompt: String, + images: [CIImage], + audios: [UserInput.Audio] + ) -> AsyncThrowingStream func cancel() func unload() var stats: LLMStats { get } @@ -17,7 +22,11 @@ public protocol LLMEngine { public extension LLMEngine { func generateStream(prompt: String) -> AsyncThrowingStream { - generateStream(prompt: prompt, images: []) + generateStream(prompt: prompt, images: [], audios: []) + } + + func generateStream(prompt: String, images: [CIImage]) -> AsyncThrowingStream { + generateStream(prompt: prompt, images: images, audios: []) } } diff --git a/LLM/MLX/Gemma4AudioFeatureExtractor.swift b/LLM/MLX/Gemma4AudioFeatureExtractor.swift new file mode 100644 index 0000000..d2047de --- /dev/null +++ b/LLM/MLX/Gemma4AudioFeatureExtractor.swift @@ -0,0 +1,182 @@ +import Foundation +import MLX +import MLXLMCommon + +struct Gemma4ExtractedAudioFeatures { + let features: MLXArray + let invalidMask: MLXArray + let tokenCount: Int + let sampleCount: Int +} + +struct Gemma4AudioFeatureExtractor { + private static let frameLengthMilliseconds: Double = 20 + private static let maxInputSamples = 480_000 + private static let melFloor: Float = 1e-3 + + private let config: Gemma4ProcessorConfiguration.AudioFeatureExtractor + private let frameLength: Int + private let fftLength: Int + private let window: MLXArray + private let melFilters: MLXArray + + init(config: Gemma4ProcessorConfiguration.AudioFeatureExtractor) { + self.config = config + let frameLength = Int( + round(Double(config.samplingRate) * Self.frameLengthMilliseconds / 1000.0) + ) + self.frameLength = frameLength + self.fftLength = Self.effectiveFFTLength( + configuredFFTLength: config.fftLength, + frameLength: frameLength + ) + + let windowValues: [Float] = (0.. Gemma4ExtractedAudioFeatures { + var waveform = pcm.samples + if Int(round(pcm.sampleRate)) != config.samplingRate { + waveform = resampleLinear( + waveform, + from: pcm.sampleRate, + to: Double(config.samplingRate) + ) + } + + if waveform.count > Self.maxInputSamples { + waveform = Array(waveform.prefix(Self.maxInputSamples)) + } + if waveform.count < frameLength { + waveform.append(contentsOf: repeatElement(0, count: frameLength - waveform.count)) + } + + let frames = buildFrames(from: waveform) + let frameCount = max(frames.count, 1) + + let flattened = frames.flatMap { $0 } + var frameArray = MLXArray(flattened).reshaped(frameCount, frameLength) + frameArray = frameArray * window + + let spectrum = abs(rfft(frameArray, n: fftLength, axis: -1, stream: .cpu)).asType(.float32) + let melSpec = matmul(spectrum, melFilters, stream: .cpu) + let floored = MLX.where( + melSpec .< MLXArray(Self.melFloor), + MLXArray(Self.melFloor), + melSpec + ) + let logMel = log(floored, stream: .cpu).asType(.float32) + + let invalidMask = MLXArray(Array(repeating: false, count: frameCount)) + .expandedDimensions(axis: 0) + let tokenCount = Self.downsampleLength(Self.downsampleLength(frameCount)) + + return Gemma4ExtractedAudioFeatures( + features: logMel.expandedDimensions(axis: 0), + invalidMask: invalidMask, + tokenCount: tokenCount, + sampleCount: waveform.count + ) + } + + private func buildFrames(from waveform: [Float]) -> [[Float]] { + let windowSize = frameLength + let hop = config.hopLength + guard waveform.count >= windowSize else { + return [Array(waveform.prefix(frameLength)) + Array(repeating: 0, count: max(0, frameLength - waveform.count))] + } + + let frameCount = max(1, ((waveform.count - windowSize) / hop) + 1) + return (0.. [Float] { + guard !samples.isEmpty, abs(sourceRate - targetRate) > 1 else { return samples } + let newCount = max(1, Int(round(Double(samples.count) * targetRate / sourceRate))) + if newCount == samples.count { return samples } + + let scale = Double(samples.count - 1) / Double(max(newCount - 1, 1)) + return (0.. Int { + max(1, ((inputLength + 2 - 3) / 2) + 1) + } + + static func effectiveFFTLength( + configuredFFTLength: Int, + frameLength: Int + ) -> Int { + let safeFrameLength = max(frameLength, 1) + let baseFFTLength = 1 << Int(Foundation.ceil(Foundation.log2(Double(safeFrameLength)))) + + // Gemma 4's reference extractor defaults fft_overdrive=True. + // When processor_config only stores the base FFT length (512 for 20 ms @ 16 kHz), + // we need to double it here to match the reference pipeline. + if configuredFFTLength > baseFFTLength { + return configuredFFTLength + } + return baseFFTLength * 2 + } + + private static func makeMelFilterBank( + fftLength: Int, + samplingRate: Int, + numMelFilters: Int + ) -> [[Float]] { + let numFrequencyBins = (fftLength / 2) + 1 + let melMin = hzToMel(0) + let melMax = hzToMel(Float(samplingRate) / 2) + let melPoints: [Float] = (0..<(numMelFilters + 2)).map { index in + let ratio = Float(index) / Float(numMelFilters + 1) + return melMin + (melMax - melMin) * ratio + } + let freqPoints = melPoints.map(melToHz) + let allFreqs: [Float] = (0.. Float { + 2595 * Foundation.log10(1 + frequency / 700) + } + + private static func melToHz(_ mel: Float) -> Float { + 700 * (pow(10, mel / 2595) - 1) + } +} diff --git a/LLM/MLX/Gemma4AudioModel.swift b/LLM/MLX/Gemma4AudioModel.swift new file mode 100644 index 0000000..85a60bf --- /dev/null +++ b/LLM/MLX/Gemma4AudioModel.swift @@ -0,0 +1,555 @@ +import Foundation +import MLX +import MLXFast +import MLXNN + +private func gemma4AudioSigmoid(_ x: MLXArray) -> MLXArray { + MLXArray(1) / (MLXArray(1) + exp(-x)) +} + +private func gemma4AudioSilu(_ x: MLXArray) -> MLXArray { + x * gemma4AudioSigmoid(x) +} + +private func gemma4AudioSoftplus(_ x: MLXArray) -> MLXArray { + log(MLXArray(1) + exp(x)) +} + +private func gemma4AudioRelu(_ x: MLXArray) -> MLXArray { + maximum(x, MLXArray(0)) +} + +private func gemma4AudioPaddingWidths( + ndim: Int, + updates: [(Int, Int, Int)] +) -> [IntOrPair] { + var widths = Array(repeating: IntOrPair(0), count: ndim) + for (axis, before, after) in updates { + widths[axis] = IntOrPair((before, after)) + } + return widths +} + +final class Gemma4AudioRMSNorm: Module { + @ParameterInfo(key: "weight") var weight: MLXArray + let eps: Float + + init(dim: Int, eps: Float = 1e-6) { + self.eps = eps + self._weight.wrappedValue = MLXArray.ones([dim]) + } + + func callAsFunction(_ x: MLXArray) -> MLXArray { + MLXFast.rmsNorm(x, weight: weight, eps: eps) + } +} + +final class Gemma4AudioConvBlock: Module { + @ModuleInfo(key: "conv") var conv: Conv2d + @ModuleInfo(key: "norm") var norm: LayerNorm + + let timeStride = 2 + + init(config: Gemma4AudioConfiguration, index: Int) { + let inputChannels = index == 0 ? 1 : config.subsamplingConvChannels[index - 1] + let outputChannels = config.subsamplingConvChannels[index] + self._conv.wrappedValue = Conv2d( + inputChannels: inputChannels, + outputChannels: outputChannels, + kernelSize: [3, 3], + stride: [2, 2], + padding: 0, + bias: false + ) + self._norm.wrappedValue = LayerNorm( + dimensions: outputChannels, + eps: config.rmsNormEps, + affine: true, + bias: false + ) + } + + func callAsFunction(_ x: MLXArray, invalidMask: MLXArray) -> (MLXArray, MLXArray) { + let expandedMask = invalidMask.expandedDimensions(axis: -1).expandedDimensions(axis: -1) + var hiddenStates = MLX.where(expandedMask, MLXArray(0, dtype: x.dtype), x) + hiddenStates = padded( + hiddenStates, + widths: gemma4AudioPaddingWidths( + ndim: hiddenStates.ndim, + updates: [(1, 1, 1), (2, 1, 1)] + ), + value: MLXArray(0, dtype: hiddenStates.dtype) + ) + hiddenStates = gemma4AudioRelu(norm(conv(hiddenStates))) + + let strideIndices = MLXArray( + Array(stride(from: 0, to: invalidMask.dim(1), by: timeStride)).map(Int32.init) + ) + let downsampledMask = take(invalidMask, strideIndices, axis: 1) + let trimmedMask = downsampledMask[0..., .. (MLXArray, MLXArray) { + var hiddenStates = audioMel.expandedDimensions(axis: -1) + var currentMask = invalidMask + + (hiddenStates, currentMask) = layer0(hiddenStates, invalidMask: currentMask) + (hiddenStates, currentMask) = layer1(hiddenStates, invalidMask: currentMask) + + let flattened = hiddenStates.reshaped( + hiddenStates.dim(0), + hiddenStates.dim(1), + hiddenStates.dim(2) * hiddenStates.dim(3) + ) + return (inputProjLinear(flattened), currentMask) + } +} + +final class Gemma4AudioRelativePositionEmbedding { + let numHeads: Int + let channels: Int + let headDim: Int + let maxBackward: Int + let maxForward: Int + let invTimescales: MLXArray + let relativeKProj: Linear + + init(config: Gemma4AudioConfiguration, relativeKProj: Linear) { + self.numHeads = config.numAttentionHeads + self.channels = config.hiddenSize + self.headDim = config.hiddenSize / config.numAttentionHeads + self.maxBackward = max(0, config.attentionContextLeft - 1) + self.maxForward = config.attentionContextRight + self.relativeKProj = relativeKProj + + let numTimescales = max(channels / 2, 1) + let minTimescale: Float = 1 + let maxTimescale: Float = 10_000 + let logTimescaleIncrement = Float( + Foundation.log(Double(maxTimescale / minTimescale)) + ) / Float(max(numTimescales - 1, 1)) + let scales: [Float] = (0.. MLXArray { + let batchSize = queries.dim(0) + let numBlocks = queries.dim(1) + let blockSize = queries.dim(2) + let contextSize = keys.dim(2) + let maxSpanPlusOne = maxBackward + maxForward + 1 + + let positionValues = Array(stride(from: maxBackward, through: -maxForward, by: -1)).map(Float.init) + let positionIndices = MLXArray(positionValues).reshaped(1, maxSpanPlusOne) + let sinusoid = positionIndices.expandedDimensions(axis: -1).asType(.float32) * invTimescales + let timingSignal = concatenated([sin(sinusoid), cos(sinusoid)], axis: -1) + let projected = relativeKProj(timingSignal.asType(relativeKProj.weight.dtype)) + .reshaped(maxSpanPlusOne, numHeads, headDim) + .asType(queries.dtype) + + let queriesP = queries.transposed(0, 3, 1, 2, 4) + let keysP = keys.transposed(0, 3, 1, 4, 2) + let termAC = matmul(queriesP, keysP) + + let sinEmbT = projected.transposed(1, 2, 0) + let reshapedQueries = queriesP.reshaped(batchSize, numHeads, numBlocks * blockSize, headDim) + var termBD = matmul(reshapedQueries, sinEmbT) + .reshaped(batchSize, numHeads, numBlocks, blockSize, maxSpanPlusOne) + + let padAmount = max((contextSize + 1) - maxSpanPlusOne, 0) + if padAmount > 0 { + termBD = padded( + termBD, + widths: gemma4AudioPaddingWidths( + ndim: termBD.ndim, + updates: [(4, 0, padAmount)] + ) + ) + } + termBD = termBD.reshaped(batchSize, numHeads, numBlocks, blockSize * (contextSize + 1)) + termBD = termBD[0..., 0..., 0..., ..<(blockSize * contextSize)] + termBD = termBD.reshaped(batchSize, numHeads, numBlocks, blockSize, contextSize) + + return termAC + termBD + } +} + +final class Gemma4AudioAttention: Module { + @ModuleInfo(key: "relative_k_proj") var relativeKProj: Linear + @ParameterInfo(key: "per_dim_scale") var perDimScale: MLXArray + @ModuleInfo(key: "q_proj") var qProj: Gemma4ClippableLinear + @ModuleInfo(key: "k_proj") var kProj: Gemma4ClippableLinear + @ModuleInfo(key: "v_proj") var vProj: Gemma4ClippableLinear + @ModuleInfo(key: "post") var post: Gemma4ClippableLinear + + let numHeads: Int + let hiddenSize: Int + let headDim: Int + let chunkSize: Int + let maxFutureHorizon: Int + let maxPastHorizon: Int + let contextSize: Int + let invalidLogitsValue: Float + let softcap: Float + let qScale: Float + let kScale: Float + let relativePositionEmbedding: Gemma4AudioRelativePositionEmbedding + + init(config: Gemma4AudioConfiguration) { + self.numHeads = config.numAttentionHeads + self.hiddenSize = config.hiddenSize + self.headDim = config.hiddenSize / config.numAttentionHeads + self.chunkSize = config.attentionChunkSize + self.maxFutureHorizon = config.attentionContextRight + self.maxPastHorizon = max(0, config.attentionContextLeft - 1) + self.contextSize = chunkSize + maxPastHorizon + maxFutureHorizon + self.invalidLogitsValue = config.attentionInvalidLogitsValue + self.softcap = config.attentionLogitCap + self.qScale = Float(Foundation.pow(Double(headDim), -0.5) / Foundation.log(2.0)) + self.kScale = Float(Foundation.log(1.0 + Foundation.exp(1.0)) / Foundation.log(2.0)) + + self._relativeKProj.wrappedValue = Linear( + config.hiddenSize, + config.numAttentionHeads * (config.hiddenSize / config.numAttentionHeads), + bias: false + ) + self._perDimScale.wrappedValue = MLXArray.zeros([headDim]) + self._qProj.wrappedValue = Gemma4ClippableLinear( + inFeatures: config.hiddenSize, + outFeatures: config.numAttentionHeads * headDim, + useClipping: config.useClippedLinears + ) + self._kProj.wrappedValue = Gemma4ClippableLinear( + inFeatures: config.hiddenSize, + outFeatures: config.numAttentionHeads * headDim, + useClipping: config.useClippedLinears + ) + self._vProj.wrappedValue = Gemma4ClippableLinear( + inFeatures: config.hiddenSize, + outFeatures: config.numAttentionHeads * headDim, + useClipping: config.useClippedLinears + ) + self._post.wrappedValue = Gemma4ClippableLinear( + inFeatures: config.hiddenSize, + outFeatures: config.hiddenSize, + useClipping: config.useClippedLinears + ) + self.relativePositionEmbedding = Gemma4AudioRelativePositionEmbedding( + config: config, + relativeKProj: _relativeKProj.wrappedValue + ) + } + + func callAsFunction( + _ hiddenStates: MLXArray, + invalidMask: MLXArray, + causalValidMask: MLXArray + ) -> MLXArray { + let batch = hiddenStates.dim(0) + let length = hiddenStates.dim(1) + let qkvShape = [batch, length, numHeads, headDim] + + var queries = qProj(hiddenStates).asType(.float32).reshaped(qkvShape) + var keys = kProj(hiddenStates).asType(.float32).reshaped(qkvShape) + let values = vProj(hiddenStates).asType(.float32).reshaped(qkvShape) + + let scaledPerDim = gemma4AudioSoftplus(perDimScale) + queries = queries * MLXArray(qScale) * scaledPerDim + keys = keys * MLXArray(kScale) + + let queryBlocks = convertToBlock(queries) + let keyBlocks = extractBlockContext(keys) + let valueBlocks = extractBlockContext(values) + + let validMask = logicalNot(invalidMask) + let extractedValid = extractBlockContext(validMask) + let condition = + extractedValid.expandedDimensions(axis: 1).expandedDimensions(axis: 3) + .&& causalValidMask.expandedDimensions(axis: 0).expandedDimensions(axis: 0).expandedDimensions(axis: 0) + + var logits = relativePositionEmbedding(queryBlocks, keyBlocks) + logits = tanh(logits / MLXArray(softcap)) * MLXArray(softcap) + logits = MLX.where(condition, logits, MLXArray(invalidLogitsValue).asType(logits.dtype)) + + let probabilities = softmax(logits, axis: -1) + let valueBlocksPrepared = valueBlocks.transposed(0, 3, 1, 2, 4) + var context = matmul(probabilities, valueBlocksPrepared) + context = context.transposed(0, 2, 3, 1, 4) + context = context.reshaped(batch, queryBlocks.dim(1) * chunkSize, numHeads, headDim) + context = context[0..., .. MLXArray { + let batch = x.dim(0) + let length = x.dim(1) + let remaining = Array(x.shape.dropFirst(2)) + let numBlocks = (length + chunkSize - 1) / chunkSize + let padLength = (numBlocks * chunkSize) - length + let paddedInput: MLXArray + if padLength > 0 { + paddedInput = padded( + x, + widths: gemma4AudioPaddingWidths( + ndim: x.ndim, + updates: [(1, 0, padLength)] + ) + ) + } else { + paddedInput = x + } + return paddedInput.reshaped([batch, numBlocks, chunkSize] + remaining) + } + + private func extractBlockContext(_ x: MLXArray) -> MLXArray { + let paddedInput = padded( + x, + widths: gemma4AudioPaddingWidths( + ndim: x.ndim, + updates: [(1, maxPastHorizon, maxFutureHorizon + chunkSize - 1)] + ) + ) + let totalLength = paddedInput.dim(1) + let numBlocks = ((totalLength - contextSize) / chunkSize) + 1 + let slices: [MLXArray] = (0.. MLXArray { + let residual = x + var hiddenStates = clip(x, min: MLXArray(-gradientClipping), max: MLXArray(gradientClipping)) + hiddenStates = preLayerNorm(hiddenStates) + hiddenStates = ffwLayer1(hiddenStates) + hiddenStates = gemma4AudioSilu(hiddenStates) + hiddenStates = ffwLayer2(hiddenStates) + hiddenStates = clip(hiddenStates, min: MLXArray(-gradientClipping), max: MLXArray(gradientClipping)) + hiddenStates = postLayerNorm(hiddenStates) + return residual + hiddenStates * MLXArray(residualWeight) + } +} + +final class Gemma4AudioLightConv1d: Module { + @ModuleInfo(key: "pre_layer_norm") var preLayerNorm: Gemma4AudioRMSNorm + @ModuleInfo(key: "linear_start") var linearStart: Gemma4ClippableLinear + @ModuleInfo(key: "depthwise_conv1d") var depthwiseConv1d: Conv1d + @ModuleInfo(key: "conv_norm") var convNorm: Gemma4AudioRMSNorm + @ModuleInfo(key: "linear_end") var linearEnd: Gemma4ClippableLinear + + let gradientClipping: Float + let causalPadding: Int + + init(config: Gemma4AudioConfiguration) { + self.gradientClipping = config.gradientClipping + self.causalPadding = config.convKernelSize - 1 + self._preLayerNorm.wrappedValue = Gemma4AudioRMSNorm(dim: config.hiddenSize, eps: config.rmsNormEps) + self._linearStart.wrappedValue = Gemma4ClippableLinear( + inFeatures: config.hiddenSize, + outFeatures: config.hiddenSize * 2, + useClipping: config.useClippedLinears + ) + self._depthwiseConv1d.wrappedValue = Conv1d( + inputChannels: config.hiddenSize, + outputChannels: config.hiddenSize, + kernelSize: config.convKernelSize, + stride: 1, + padding: 0, + groups: config.hiddenSize, + bias: false + ) + self._convNorm.wrappedValue = Gemma4AudioRMSNorm(dim: config.hiddenSize, eps: config.rmsNormEps) + self._linearEnd.wrappedValue = Gemma4ClippableLinear( + inFeatures: config.hiddenSize, + outFeatures: config.hiddenSize, + useClipping: config.useClippedLinears + ) + } + + func callAsFunction(_ x: MLXArray) -> MLXArray { + let residual = x + var hiddenStates = preLayerNorm(x) + hiddenStates = linearStart(hiddenStates) + + let half = hiddenStates.dim(-1) / 2 + let x1 = hiddenStates[0..., 0..., .. MLXArray { + var hiddenStates = feedForward1(x) + + let residual = hiddenStates + hiddenStates = clip(hiddenStates, min: MLXArray(-gradientClipping), max: MLXArray(gradientClipping)) + hiddenStates = normPreAttn(hiddenStates) + hiddenStates = selfAttn(hiddenStates, invalidMask: invalidMask, causalValidMask: causalValidMask) + hiddenStates = clip(hiddenStates, min: MLXArray(-gradientClipping), max: MLXArray(gradientClipping)) + hiddenStates = residual + normPostAttn(hiddenStates) + + let validityMask = logicalNot(invalidMask).expandedDimensions(axis: -1).asType(hiddenStates.dtype) + hiddenStates = hiddenStates * validityMask + hiddenStates = lconv1d(hiddenStates) + hiddenStates = feedForward2(hiddenStates) + hiddenStates = clip(hiddenStates, min: MLXArray(-gradientClipping), max: MLXArray(gradientClipping)) + return normOut(hiddenStates) + } +} + +final class Gemma4AudioEncoder: Module { + @ModuleInfo(key: "subsample_conv_projection") var subsampleConvProjection: Gemma4AudioSubsampleProjection + @ModuleInfo var layers: [Gemma4AudioConformerBlock] + @ModuleInfo(key: "output_proj") var outputProj: Linear + + let config: Gemma4AudioConfiguration + let hasOutputProjection: Bool + + init(config: Gemma4AudioConfiguration) { + self.config = config + self.hasOutputProjection = config.outputProjDims != nil + self._subsampleConvProjection.wrappedValue = Gemma4AudioSubsampleProjection(config: config) + self._layers.wrappedValue = (0.. MLXArray { + let chunkSize = config.attentionChunkSize + let maxPastHorizon = max(0, config.attentionContextLeft - 1) + let maxFutureHorizon = config.attentionContextRight + let upperDiagonal = maxPastHorizon + maxFutureHorizon + let contextSize = chunkSize + maxPastHorizon + maxFutureHorizon + let lowerCausal = tril(MLXArray.ones([contextSize, chunkSize])).transposed() + let upperCausal = tril(MLXArray.ones([chunkSize, contextSize]), k: upperDiagonal) + return (lowerCausal * upperCausal).asType(.bool) + } + + func callAsFunction(_ audioMel: MLXArray, invalidMask: MLXArray) -> (MLXArray, MLXArray) { + var (audioEncodings, currentMask) = subsampleConvProjection(audioMel, invalidMask: invalidMask) + let causalValidMask = buildCausalValidMask() + for layer in layers { + audioEncodings = layer(audioEncodings, invalidMask: currentMask, causalValidMask: causalValidMask) + } + + if hasOutputProjection { + audioEncodings = outputProj(audioEncodings) + } + + if currentMask.dim(1) != audioEncodings.dim(1) { + currentMask = currentMask[0..., .. Gemma4ModelConfiguration { + Gemma4ModelConfiguration( + textConfig: textConfig, + visionConfig: visionConfig, + audioConfig: enabled ? audioConfig : nil, + modelType: modelType, + quantization: quantization, + tieWordEmbeddings: tieWordEmbeddings, + visionSoftTokensPerImage: visionSoftTokensPerImage, + imageTokenId: imageTokenId, + audioTokenId: audioTokenId + ) + } } // MARK: - Gemma 4 Vision Configuration @@ -196,6 +232,16 @@ public struct Gemma4AudioConfiguration: Codable, Sendable { public let hiddenSize: Int public let numHiddenLayers: Int public let numAttentionHeads: Int + public let subsamplingConvChannels: [Int] + public let convKernelSize: Int + public let residualWeight: Float + public let attentionChunkSize: Int + public let attentionContextLeft: Int + public let attentionContextRight: Int + public let attentionLogitCap: Float + public let attentionInvalidLogitsValue: Float + public let useClippedLinears: Bool + public let gradientClipping: Float public let outputProjDims: Int? public let rmsNormEps: Float @@ -204,9 +250,78 @@ public struct Gemma4AudioConfiguration: Codable, Sendable { case hiddenSize = "hidden_size" case numHiddenLayers = "num_hidden_layers" case numAttentionHeads = "num_attention_heads" + case subsamplingConvChannels = "subsampling_conv_channels" + case convKernelSize = "conv_kernel_size" + case residualWeight = "residual_weight" + case attentionChunkSize = "attention_chunk_size" + case attentionContextLeft = "attention_context_left" + case attentionContextRight = "attention_context_right" + case attentionLogitCap = "attention_logit_cap" + case attentionInvalidLogitsValue = "attention_invalid_logits_value" + case useClippedLinears = "use_clipped_linears" + case gradientClipping = "gradient_clipping" case outputProjDims = "output_proj_dims" case rmsNormEps = "rms_norm_eps" } + + public init( + modelType: String, + hiddenSize: Int, + numHiddenLayers: Int, + numAttentionHeads: Int, + subsamplingConvChannels: [Int], + convKernelSize: Int, + residualWeight: Float, + attentionChunkSize: Int, + attentionContextLeft: Int, + attentionContextRight: Int, + attentionLogitCap: Float, + attentionInvalidLogitsValue: Float, + useClippedLinears: Bool, + gradientClipping: Float, + outputProjDims: Int?, + rmsNormEps: Float + ) { + self.modelType = modelType + self.hiddenSize = hiddenSize + self.numHiddenLayers = numHiddenLayers + self.numAttentionHeads = numAttentionHeads + self.subsamplingConvChannels = subsamplingConvChannels + self.convKernelSize = convKernelSize + self.residualWeight = residualWeight + self.attentionChunkSize = attentionChunkSize + self.attentionContextLeft = attentionContextLeft + self.attentionContextRight = attentionContextRight + self.attentionLogitCap = attentionLogitCap + self.attentionInvalidLogitsValue = attentionInvalidLogitsValue + self.useClippedLinears = useClippedLinears + self.gradientClipping = gradientClipping + self.outputProjDims = outputProjDims + self.rmsNormEps = rmsNormEps + } + + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + modelType = try c.decodeIfPresent(String.self, forKey: .modelType) ?? "gemma4_audio" + hiddenSize = try c.decodeIfPresent(Int.self, forKey: .hiddenSize) ?? 1024 + numHiddenLayers = try c.decodeIfPresent(Int.self, forKey: .numHiddenLayers) ?? 12 + numAttentionHeads = try c.decodeIfPresent(Int.self, forKey: .numAttentionHeads) ?? 8 + subsamplingConvChannels = try c.decodeIfPresent([Int].self, forKey: .subsamplingConvChannels) + ?? [128, 32] + convKernelSize = try c.decodeIfPresent(Int.self, forKey: .convKernelSize) ?? 5 + residualWeight = try c.decodeIfPresent(Float.self, forKey: .residualWeight) ?? 0.5 + attentionChunkSize = try c.decodeIfPresent(Int.self, forKey: .attentionChunkSize) ?? 12 + attentionContextLeft = try c.decodeIfPresent(Int.self, forKey: .attentionContextLeft) ?? 13 + attentionContextRight = try c.decodeIfPresent(Int.self, forKey: .attentionContextRight) ?? 0 + attentionLogitCap = try c.decodeIfPresent(Float.self, forKey: .attentionLogitCap) ?? 50 + attentionInvalidLogitsValue = + try c.decodeIfPresent(Float.self, forKey: .attentionInvalidLogitsValue) ?? -1_000_000_000 + useClippedLinears = try c.decodeIfPresent(Bool.self, forKey: .useClippedLinears) ?? true + gradientClipping = + try c.decodeIfPresent(Float.self, forKey: .gradientClipping) ?? 10_000_000_000 + outputProjDims = try c.decodeIfPresent(Int.self, forKey: .outputProjDims) + rmsNormEps = try c.decodeIfPresent(Float.self, forKey: .rmsNormEps) ?? 1e-6 + } } // MARK: - Gemma 4 Processor Configuration @@ -217,6 +332,27 @@ public struct Gemma4ProcessorConfiguration: Codable, Sendable { public let audioSeqLength: Int public let audioMsPerToken: Int? public let imageProcessor: ImageProcessor + public let featureExtractor: AudioFeatureExtractor? + + public struct AudioFeatureExtractor: Codable, Sendable { + public let featureExtractorType: String + public let samplingRate: Int + public let numMelFilters: Int + public let fftLength: Int + public let hopLength: Int + public let chunkDuration: Double + public let overlapDuration: Double + + enum CodingKeys: String, CodingKey { + case featureExtractorType = "feature_extractor_type" + case samplingRate = "sampling_rate" + case numMelFilters = "num_mel_filters" + case fftLength = "fft_length" + case hopLength = "hop_length" + case chunkDuration = "chunk_duration" + case overlapDuration = "overlap_duration" + } + } public struct ImageProcessor: Codable, Sendable { public let imageProcessorType: String @@ -275,5 +411,6 @@ public struct Gemma4ProcessorConfiguration: Codable, Sendable { case audioSeqLength = "audio_seq_length" case audioMsPerToken = "audio_ms_per_token" case imageProcessor = "image_processor" + case featureExtractor = "feature_extractor" } } diff --git a/LLM/MLX/Gemma4Model.swift b/LLM/MLX/Gemma4Model.swift index 26c2fa5..807311e 100644 --- a/LLM/MLX/Gemma4Model.swift +++ b/LLM/MLX/Gemma4Model.swift @@ -11,8 +11,11 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { @ModuleInfo(key: "language_model") var languageModel: Gemma4LanguageModel @ModuleInfo(key: "vision_tower") var visionTower: Gemma4VisionModel @ModuleInfo(key: "embed_vision") var embedVision: Gemma4MultimodalProjector + @ModuleInfo(key: "audio_tower") var audioTower: Gemma4AudioEncoder? + @ModuleInfo(key: "embed_audio") var embedAudio: Gemma4MultimodalProjector? public let config: Gemma4ModelConfiguration + private let supportsAudio: Bool public var kvHeads: [Int] { languageModel.kvHeads } @@ -47,11 +50,27 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { outputDim: config.textConfig.hiddenSize, eps: visionConfig.rmsNormEps ) + + self.supportsAudio = config.audioConfig != nil + if let audioConfig = config.audioConfig { + self._audioTower.wrappedValue = Gemma4AudioEncoder(config: audioConfig) + self._embedAudio.wrappedValue = Gemma4MultimodalProjector( + inputDim: audioConfig.outputProjDims ?? audioConfig.hiddenSize, + outputDim: config.textConfig.hiddenSize, + eps: audioConfig.rmsNormEps + ) + } else { + self._audioTower.wrappedValue = nil + self._embedAudio.wrappedValue = nil + } } private func getInputEmbeddings( inputIds: MLXArray, - pixelValues: MLXArray? + pixelValues: MLXArray?, + imageSoftTokenCount: Int?, + audioFeatures: MLXArray?, + audioInvalidMask: MLXArray? ) -> (inputsEmbeds: MLXArray, perLayerInputs: MLXArray?) { let batchedIds = inputIds.ndim == 1 ? inputIds.expandedDimensions(axis: 0) : inputIds var inputsEmbeds = languageModel.model.embedTokens(batchedIds) @@ -69,10 +88,20 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { } if let pixelValues { - var imageFeatures = visionTower(pixelValues) + var imageFeatures = visionTower(pixelValues, outputLength: imageSoftTokenCount) imageFeatures = embedVision(imageFeatures).asType(inputsEmbeds.dtype) let imageMask = batchedIds .== MLXArray(config.imageTokenId ?? 258880) + let imageTokenPositions = imageMask.asArray(Bool.self).filter { $0 }.count + if imageTokenPositions == 0 { + print("[VLM] warning — prompt 中没有图片 soft token，当前图片 embedding 不会被注入。") + } else if imageTokenPositions != imageFeatures.dim(0) * imageFeatures.dim(1) { + print( + "[VLM] warning — 图片 token 数与编码输出长度不一致。" + + " positions=\(imageTokenPositions), " + + "encodings=\(imageFeatures.dim(0) * imageFeatures.dim(1))" + ) + } let embedDim = inputsEmbeds.dim(-1) var imageMaskExpanded = expandedDimensions(imageMask, axis: -1) imageMaskExpanded = repeated(imageMaskExpanded, count: embedDim, axis: -1) @@ -84,6 +113,41 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { ) } + if supportsAudio, let audioFeatures, let audioTower, let embedAudio { + let invalidMask = audioInvalidMask ?? MLXArray(Array(repeating: false, count: audioFeatures.dim(1))) + .expandedDimensions(axis: 0) + var audioEncodings = audioTower(audioFeatures, invalidMask: invalidMask).0 + audioEncodings = embedAudio(audioEncodings).asType(inputsEmbeds.dtype) + + let audioMask = batchedIds .== MLXArray(config.audioTokenId ?? 258881) + let audioTokenPositions = audioMask.asArray(Bool.self).filter { $0 }.count + print( + "[AUDIO] encoder output — " + + "features=\(audioFeatures.shape), " + + "invalidMask=\(invalidMask.shape), " + + "encodings=\(audioEncodings.shape), " + + "tokenPositions=\(audioTokenPositions)" + ) + if audioTokenPositions == 0 { + print("[AUDIO] warning — prompt 中没有音频 soft token，当前音频 embedding 不会被注入。") + } else if audioTokenPositions != audioEncodings.dim(0) * audioEncodings.dim(1) { + print( + "[AUDIO] warning — 音频 token 数与编码输出长度不一致。" + + " positions=\(audioTokenPositions), " + + "encodings=\(audioEncodings.dim(0) * audioEncodings.dim(1))" + ) + } + let embedDim = inputsEmbeds.dim(-1) + var audioMaskExpanded = expandedDimensions(audioMask, axis: -1) + audioMaskExpanded = repeated(audioMaskExpanded, count: embedDim, axis: -1) + + inputsEmbeds = gemma4MaskedScatter( + finalEmbedding: inputsEmbeds, + maskExpanded: audioMaskExpanded, + source: audioEncodings + ) + } + return (inputsEmbeds, perLayerInputs) } @@ -94,7 +158,7 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { ) throws -> PrepareResult { let convertedCache = cache.compactMap { $0 as KVCache } - guard let imagePixels = input.image?.pixels else { + guard input.image?.pixels != nil || input.audio?.features != nil else { let result = languageModel( input.text.tokens, cache: convertedCache, @@ -106,7 +170,10 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { let inputEmbeddings = getInputEmbeddings( inputIds: input.text.tokens, - pixelValues: imagePixels + pixelValues: input.image?.pixels, + imageSoftTokenCount: input.image?.softTokenCount, + audioFeatures: input.audio?.features, + audioInvalidMask: input.audio?.invalidMask ) let result = languageModel( @@ -137,7 +204,9 @@ public class Gemma4Model: Module, VLMModel, KVCacheDimensionProvider { sanitized.reserveCapacity(weights.count) for (key, value) in weights { - if key.hasPrefix("audio_tower.") || key.hasPrefix("embed_audio.") { + if !supportsAudio, + (key.hasPrefix("audio_tower.") || key.hasPrefix("embed_audio.")) + { continue } if key.contains("rotary_emb") { diff --git a/LLM/MLX/Gemma4Processor.swift b/LLM/MLX/Gemma4Processor.swift index a9a4d1b..be22c06 100644 --- a/LLM/MLX/Gemma4Processor.swift +++ b/LLM/MLX/Gemma4Processor.swift @@ -4,10 +4,26 @@ import MLX import MLXLMCommon import MLXVLM +private enum Gemma4AudioProcessingError: LocalizedError { + case urlInputNotSupported(URL) + case multipleAudioInputs(Int) + + var errorDescription: String? { + switch self { + case .urlInputNotSupported(let url): + return "暂不支持从文件 URL 直接读取音频：\(url.lastPathComponent)" + case .multipleAudioInputs(let count): + return "当前版本一次只支持 1 段音频，收到 \(count) 段。" + } + } +} + public struct Gemma4Processor: UserInputProcessor { - // Gemma 4 E2B has enough headroom on-device to use the processor's - // native visual length instead of an aggressively compressed budget. - private static let mobileSoftTokenCap = 280 + // E4B runs close to the jetsam limit on-device; keep the visual token + // budget conservative so single-image turns stay stable. + private static let defaultMobileSoftTokenCap = 160 + private static let runtimeBudgetLock = NSLock() + private static var runtimeImageSoftTokenCapOverride: Int? private let config: Gemma4ProcessorConfiguration private let tokenizer: any Tokenizer @@ -15,12 +31,92 @@ public struct Gemma4Processor: UserInputProcessor { private let imageToken = "<|image|>" private let boiToken = "<|image>" private let eoiToken = "" + private let audioToken = "<|audio|>" + private let boaToken = "<|audio>" + private let eoaToken = "" public init(_ config: Gemma4ProcessorConfiguration, tokenizer: any Tokenizer) { self.config = config self.tokenizer = tokenizer } + public static func setRuntimeImageSoftTokenCap(_ cap: Int?) { + runtimeBudgetLock.lock() + runtimeImageSoftTokenCapOverride = cap + runtimeBudgetLock.unlock() + } + + private static func currentImageSoftTokenCap( + config: Gemma4ProcessorConfiguration + ) -> Int { + runtimeBudgetLock.lock() + let override = runtimeImageSoftTokenCapOverride + runtimeBudgetLock.unlock() + + let configuredCap = min( + config.imageProcessor.maxSoftTokens, + config.imageSeqLength, + defaultMobileSoftTokenCap + ) + guard let override else { return configuredCap } + return max(32, min(configuredCap, override)) + } + + private func preprocessAudio(_ audio: UserInput.Audio) throws -> LMInput.ProcessedAudio { + let pcm: UserInput.Audio.PCM + switch audio { + case .pcm(let value): + pcm = value + case .url(let url): + throw Gemma4AudioProcessingError.urlInputNotSupported(url) + } + + guard let featureConfig = config.featureExtractor else { + let sampleCount = pcm.samples.count + let tokenCount = min( + config.audioSeqLength, + max( + 1, + Int(Foundation.ceil((pcm.duration * 1000.0) / Double(config.audioMsPerToken ?? 40))) + ) + ) + let features = MLXArray(pcm.samples).expandedDimensions(axis: 0).expandedDimensions(axis: -1) + let invalidMask = MLXArray(Array(repeating: false, count: sampleCount)) + .expandedDimensions(axis: 0) + return LMInput.ProcessedAudio( + features: features, + invalidMask: invalidMask, + sampleRate: pcm.sampleRate, + channelCount: pcm.channelCount, + duration: pcm.duration, + sampleCount: sampleCount, + tokenCount: tokenCount + ) + } + + let extracted = Gemma4AudioFeatureExtractor(config: featureConfig).extract(from: pcm) + let tokenCount = computeAudioTokenCount( + sampleCount: extracted.sampleCount, + sampleRate: featureConfig.samplingRate + ) + return LMInput.ProcessedAudio( + features: extracted.features, + invalidMask: extracted.invalidMask, + sampleRate: Double(featureConfig.samplingRate), + channelCount: pcm.channelCount, + duration: Double(extracted.sampleCount) / Double(featureConfig.samplingRate), + sampleCount: extracted.sampleCount, + tokenCount: tokenCount + ) + } + + private func computeAudioTokenCount(sampleCount: Int, sampleRate: Int) -> Int { + let msPerToken = max(config.audioMsPerToken ?? 40, 1) + let durationMs = (Double(sampleCount) / Double(max(sampleRate, 1))) * 1000.0 + let rawCount = Int(Foundation.ceil(durationMs / Double(msPerToken))) + return min(config.audioSeqLength, max(1, rawCount)) + } + private func promptText(from input: UserInput) -> String { switch input.prompt { case .text(let text): @@ -37,7 +133,7 @@ public struct Gemma4Processor: UserInputProcessor { ) -> CIImage { let imageProcessor = config.imageProcessor let patchSize = imageProcessor.patchSize - let maxSoftTokens = min(imageProcessor.maxSoftTokens, Self.mobileSoftTokenCap) + let maxSoftTokens = Self.currentImageSoftTokenCap(config: config) let poolingKernelSize = imageProcessor.poolingKernelSize let maxPatches = maxSoftTokens * poolingKernelSize * poolingKernelSize @@ -105,14 +201,12 @@ public struct Gemma4Processor: UserInputProcessor { } let pixelHeight = pixelValues.dim(2) let pixelWidth = pixelValues.dim(3) - let patchSize = config.imageProcessor.patchSize - let poolingKernel = config.imageProcessor.poolingKernelSize - let numPatches = (pixelHeight / patchSize) * (pixelWidth / patchSize) - let numSoftTokens = min(config.imageProcessor.imageSeqLength, Self.mobileSoftTokenCap) + let numSoftTokens = Self.currentImageSoftTokenCap(config: config) let processedImage = LMInput.ProcessedImage( pixels: pixelValues, - frames: [THW(1, pixelHeight, pixelWidth)] + frames: [THW(1, pixelHeight, pixelWidth)], + softTokenCount: numSoftTokens ) return (processedImage, numSoftTokens) } @@ -152,12 +246,60 @@ public struct Gemma4Processor: UserInputProcessor { return expanded } + private func expandAudioTokens(in prompt: String, audioSoftTokenCount: Int) -> String { + guard audioSoftTokenCount > 0 else { return prompt } + let expanded = boaToken + String(repeating: audioToken, count: audioSoftTokenCount) + eoaToken + if prompt.contains(audioToken) { + return prompt.replacingOccurrences(of: audioToken, with: expanded) + } + return prompt + "\n" + expanded + } + + private func expandAudioTokens(in promptTokens: [Int], audioSoftTokenCount: Int) -> [Int] { + guard audioSoftTokenCount > 0 else { return promptTokens } + + let audioTokenId = tokenizer.encode(text: audioToken, addSpecialTokens: false).first + let boaTokenId = tokenizer.encode(text: boaToken, addSpecialTokens: false).first + let eoaTokenId = tokenizer.encode(text: eoaToken, addSpecialTokens: false).first + + guard let audioTokenId, let boaTokenId, let eoaTokenId else { + return promptTokens + [boaTokenId, eoaTokenId].compactMap { $0 } + } + + let expandedSequence = + [boaTokenId] + + Array(repeating: audioTokenId, count: audioSoftTokenCount) + + [eoaTokenId] + + var expanded: [Int] = [] + expanded.reserveCapacity(promptTokens.count + audioSoftTokenCount + 2) + var replacedExistingPlaceholder = false + + for token in promptTokens { + if token == audioTokenId { + expanded.append(contentsOf: expandedSequence) + replacedExistingPlaceholder = true + } else { + expanded.append(token) + } + } + + if !replacedExistingPlaceholder { + expanded.append(contentsOf: expandedSequence) + } + return expanded + } + public func prepare(input: UserInput) async throws -> LMInput { if input.images.count > 1 { throw VLMError.singleImageAllowed } + if input.audios.count > 1 { + throw Gemma4AudioProcessingError.multipleAudioInputs(input.audios.count) + } var processedImage: LMInput.ProcessedImage? + var processedAudio: LMInput.ProcessedAudio? var softTokenCount = 0 if let image = input.images.first { @@ -170,23 +312,74 @@ public struct Gemma4Processor: UserInputProcessor { softTokenCount = derivedSoftTokenCount } + if let audio = input.audios.first { + processedAudio = try preprocessAudio(audio) + if let featureExtractor = config.featureExtractor { + let frameLength = Int( + round(Double(featureExtractor.samplingRate) * 20 / 1000.0) + ) + print( + "[AUDIO] prompt prepared — samples=\(processedAudio!.sampleCount), " + + "sampleRate=\(featureExtractor.samplingRate), " + + "mel=\(featureExtractor.numMelFilters), " + + "fft=\(Gemma4AudioFeatureExtractor.effectiveFFTLength(configuredFFTLength: featureExtractor.fftLength, frameLength: frameLength)), " + + "hop=\(featureExtractor.hopLength), " + + "chunk=\(featureExtractor.chunkDuration)s" + ) + } else { + print( + "[AUDIO] prompt prepared — samples=\(processedAudio!.sampleCount), " + + "sampleRate=\(Int(processedAudio!.sampleRate))" + ) + } + } + let promptTokens: [Int] - if !input.images.isEmpty, case .chat(let chatMessages) = input.prompt { + if case .chat(let chatMessages) = input.prompt { let messages = Qwen2VLMessageGenerator().generate(messages: chatMessages) let templatedTokens = try tokenizer.applyChatTemplate( messages: messages, tools: input.tools, additionalContext: input.additionalContext ) - promptTokens = expandImageTokens(in: templatedTokens, imageSoftTokenCount: softTokenCount) + var expandedTokens = templatedTokens + if softTokenCount > 0 { + expandedTokens = expandImageTokens(in: expandedTokens, imageSoftTokenCount: softTokenCount) + } + if let processedAudio { + expandedTokens = expandAudioTokens(in: expandedTokens, audioSoftTokenCount: processedAudio.tokenCount) + } + promptTokens = expandedTokens } else { var prompt = promptText(from: input) if processedImage != nil { prompt = expandImageTokens(in: prompt, imageSoftTokenCount: softTokenCount) } + if let processedAudio { + prompt = expandAudioTokens(in: prompt, audioSoftTokenCount: processedAudio.tokenCount) + } promptTokens = tokenizer.encode(text: prompt, addSpecialTokens: false) } + if let processedAudio { + let audioTokenId = tokenizer.encode(text: audioToken, addSpecialTokens: false).first + let boaTokenId = tokenizer.encode(text: boaToken, addSpecialTokens: false).first + let eoaTokenId = tokenizer.encode(text: eoaToken, addSpecialTokens: false).first + let actualAudioTokenCount = audioTokenId.map { id in + promptTokens.reduce(into: 0) { count, token in + if token == id { count += 1 } + } + } ?? 0 + print( + "[AUDIO] token expansion — " + + "boa=\(boaTokenId.map(String.init) ?? "nil"), " + + "audio=\(audioTokenId.map(String.init) ?? "nil"), " + + "eoa=\(eoaTokenId.map(String.init) ?? "nil"), " + + "planned=\(processedAudio.tokenCount), " + + "actual=\(actualAudioTokenCount)" + ) + } + if processedImage != nil { let imageTokenId = tokenizer.encode(text: imageToken, addSpecialTokens: false).first let boiTokenId = tokenizer.encode(text: boiToken, addSpecialTokens: false).first @@ -210,7 +403,8 @@ public struct Gemma4Processor: UserInputProcessor { return LMInput( text: .init(tokens: promptArray, mask: mask), - image: processedImage + image: processedImage, + audio: processedAudio ) } } diff --git a/LLM/MLX/Gemma4Registration.swift b/LLM/MLX/Gemma4Registration.swift index afd04d0..fe169da 100644 --- a/LLM/MLX/Gemma4Registration.swift +++ b/LLM/MLX/Gemma4Registration.swift @@ -13,17 +13,38 @@ import MLXVLM /// await Gemma4Registration.register() /// ``` public enum Gemma4Registration { + private static let runtimeOptionsLock = NSLock() + private static var audioCapabilityEnabled = false + + public static func setAudioCapabilityEnabled(_ enabled: Bool) { + runtimeOptionsLock.lock() + audioCapabilityEnabled = enabled + runtimeOptionsLock.unlock() + } + + private static func currentAudioCapabilityEnabled() -> Bool { + runtimeOptionsLock.lock() + let enabled = audioCapabilityEnabled + runtimeOptionsLock.unlock() + return enabled + } + + private static func makeRuntimeConfiguration(from data: Data) throws -> Gemma4ModelConfiguration { + let configuration = try JSONDecoder.json5().decode( + Gemma4ModelConfiguration.self, + from: data + ) + return configuration.withAudioCapability(enabled: currentAudioCapabilityEnabled()) + } public static func register() async { await LLMTypeRegistry.shared.registerModelType("gemma4") { data in - let configuration = try JSONDecoder.json5().decode( - Gemma4ModelConfiguration.self, from: data) + let configuration = try makeRuntimeConfiguration(from: data) return Gemma4Model(configuration) } await VLMTypeRegistry.shared.registerModelType("gemma4") { data in - let configuration = try JSONDecoder.json5().decode( - Gemma4ModelConfiguration.self, from: data) + let configuration = try makeRuntimeConfiguration(from: data) return Gemma4Model(configuration) } diff --git a/LLM/MLX/Gemma4VisionModel.swift b/LLM/MLX/Gemma4VisionModel.swift index dd511fb..8a19611 100644 --- a/LLM/MLX/Gemma4VisionModel.swift +++ b/LLM/MLX/Gemma4VisionModel.swift @@ -26,16 +26,22 @@ func gemma4MaskedScatter( } let scatterIndices = MLXArray(indices) - guard sourceFlattened.shape[0] == scatterIndices.shape[0] else { - fatalError( - """ - Gemma4 maskedScatter shape mismatch. - source: \(sourceFlattened.shape[0]) - positions: \(scatterIndices.shape[0]) - """) + let sourceSize = Int(sourceFlattened.shape[0]) + guard sourceSize > 0 else { + return finalEmbedding } - finalFlattened[scatterIndices] = sourceFlattened + if sourceSize != scatterIndices.shape[0] { + print( + "[Gemma4] maskedScatter 自动对齐：source=\(sourceSize), positions=\(scatterIndices.shape[0])" + ) + } + + let wrappedSourceIndices = MLXArray( + (0.. MLXArray { + func callAsFunction(_ pixelValues: MLXArray, outputLength: Int? = nil) -> MLXArray { let (_, _, patchPositions, paddingPositions, numRealPatches) = patchGrid(pixelValues) let realPositions = patchPositions[0..., .. 0 { - let padEmbeds = MLXArray.zeros( - [pixelValues.dim(0), numPadding, hiddenStates.dim(-1)] - ).asType(hiddenStates.dtype) - hiddenStates = concatenated([hiddenStates, padEmbeds], axis: 1) - } - let validMask = paddingPositions .== MLXArray(false) let attentionAllowed = expandedDimensions(validMask, axis: 1) .&& expandedDimensions(validMask, axis: 2) @@ -537,7 +527,7 @@ final class Gemma4VisionModel: Module { ) hiddenStates = encoder(hiddenStates, positions: patchPositions, mask: attentionMask) - let pooledOutputLength = config.defaultOutputLength + let pooledOutputLength = outputLength ?? config.defaultOutputLength let (pooled, poolMask) = pooler( hiddenStates, patchPositions: patchPositions, diff --git a/LLM/MLXLocalLLMService.swift b/LLM/MLXLocalLLMService.swift index 55cf5e7..55ebcbd 100644 --- a/LLM/MLXLocalLLMService.swift +++ b/LLM/MLXLocalLLMService.swift @@ -66,7 +66,28 @@ public class MLXLocalLLMService: LLMEngine { ) ] static let defaultModel = availableModels[0] - private static let multimodalMaxOutputTokens = 4000 + private static let multimodalMaxOutputTokens = 512 + private static let e4bThinkingMaxOutputTokens = 1_024 + private static let e2bThinkingMaxOutputTokens = 768 + private static let e4bModelID = "gemma-4-e4b-it-4bit" + private static let e2bModelID = "gemma-4-e2b-it-4bit" + private static let e4bMultimodalCriticalHeadroomMB = 320 + + private struct MultimodalRuntimeBudget { + let imageSoftTokenCap: Int? + let maxOutputTokens: Int + let headroomMB: Int + } + + private struct ThinkingRuntimeBudget { + let maxOutputTokens: Int + let headroomMB: Int + } + + private struct TextRuntimeBudget { + let maxOutputTokens: Int + let headroomMB: Int + } // MARK: - State @@ -98,6 +119,7 @@ public class MLXLocalLLMService: LLMEngine { private let foregroundStateLock = NSLock() private var foregroundGPUAllowed = true private var lifecycleObserverTokens: [NSObjectProtocol] = [] + private var audioCapabilityEnabled = false /// Local path to the model directory private var modelPath: URL { @@ -360,7 +382,7 @@ public class MLXLocalLLMService: LLMEngine { self.isLoaded = false self.loadedModel = nil self.refreshModelInstallStates() - print("[MLX] Load failed: \(error)") + print("[MLX] Load failed: \(error.localizedDescription)") } } } @@ -368,6 +390,7 @@ public class MLXLocalLLMService: LLMEngine { func generateStream( prompt: String, images: [CIImage] = [], + audios: [UserInput.Audio] = [], onToken: @escaping (String) -> Void, onComplete: @escaping (Result) -> Void ) { @@ -375,7 +398,7 @@ public class MLXLocalLLMService: LLMEngine { var fullResponse = "" do { - for try await token in generateStream(prompt: prompt, images: images) { + for try await token in generateStream(prompt: prompt, images: images, audios: audios) { fullResponse += token await MainActor.run { onToken(token) @@ -420,6 +443,34 @@ public class MLXLocalLLMService: LLMEngine { } } + func generateStream( + chat: [Chat.Message], + additionalContext: [String: any Sendable]?, + onToken: @escaping (String) -> Void, + onComplete: @escaping (Result) -> Void + ) { + Task { + var fullResponse = "" + + do { + for try await token in generateStream(chat: chat, additionalContext: additionalContext) { + fullResponse += token + await MainActor.run { + onToken(token) + } + } + + await MainActor.run { + onComplete(.success(fullResponse)) + } + } catch { + await MainActor.run { + onComplete(.failure(error)) + } + } + } + } + // MARK: - LLMEngine Protocol public func load() async throws { @@ -433,14 +484,16 @@ public class MLXLocalLLMService: LLMEngine { isLoading = false } statusMessage = "正在初始化模型..." + await Gemma4Registration.setAudioCapabilityEnabled(audioCapabilityEnabled) await Gemma4Registration.register() guard Self.hasRequiredFiles(for: model, at: path) else { - throw MLXError.modelDirectoryMissing(path.path) + throw MLXError.modelDirectoryMissing(model.displayName) } statusMessage = "正在加载 \(model.displayName)..." let loadStart = CFAbsoluteTimeGetCurrent() + print("[MLX] load capability — audio=\(audioCapabilityEnabled ? 1 : 0)") // ── Memory diagnostics (read before load) ────────────────────────────── let physMB = Double(ProcessInfo.processInfo.physicalMemory) / 1_048_576 @@ -471,6 +524,21 @@ public class MLXLocalLLMService: LLMEngine { print("[MLX] Model loaded in \(Int(elapsed))ms — backend: mlx-gpu — model: \(model.displayName)") } + private func ensureAudioCapability(hasAudio: Bool) async throws { + guard hasAudio != audioCapabilityEnabled || !isLoaded || modelContainer == nil else { + return + } + + audioCapabilityEnabled = hasAudio + print("[MLX] capability switch requested — audio=\(hasAudio ? 1 : 0)") + + if isLoaded || modelContainer != nil { + await prepareForReload(cancelCurrentGeneration: false, cancelCurrentLoad: false) + } + + try await load() + } + /// Returns (footprint MB, jetsam limit MB) via task_info. private func appMemoryFootprintMB() -> (Double, Double) { var info = task_vm_info_data_t() @@ -493,16 +561,23 @@ public class MLXLocalLLMService: LLMEngine { } /// 根据当前剩余内存推荐安全的 history 深度（消息条数）。 - /// Gemma E2B 每 ~200 token history ≈ ~200 MB 推理峰值，保守估算： - /// headroom > 1500 MB → suffix(4) 最近 2 轮 - /// headroom > 900 MB → suffix(2) 最近 1 轮 - /// headroom ≤ 900 MB → suffix(0) 无历史（临界状态） + /// E4B 在 1 GB 左右 headroom 时对文本 KV cache 非常敏感，因此需要更激进地收紧历史。 public var safeHistoryDepth: Int { let h = availableHeadroomMB - switch h { - case 1500...: return 4 - case 900..<1500: return 2 - default: return 0 + let model = loadedModel ?? Self.availableModels.first(where: { $0.id == selectedModel.id }) + + if model?.id == Self.e4bModelID { + switch h { + case 1_700...: return 4 + case 1_100..<1_700: return 2 + default: return 0 + } + } else { + switch h { + case 1_500...: return 4 + case 900..<1_500: return 2 + default: return 0 + } } } @@ -522,27 +597,334 @@ public class MLXLocalLLMService: LLMEngine { statusMessage = "模型已就绪 ✅" } - public func generateStream(prompt: String, images: [CIImage]) -> AsyncThrowingStream { + public func generateStream( + prompt: String, + images: [CIImage], + audios: [UserInput.Audio] + ) -> AsyncThrowingStream { let input: UserInput - if images.isEmpty { + if images.isEmpty, audios.isEmpty { input = UserInput(prompt: prompt) } else { input = UserInput( chat: [ .user( prompt, - images: images.map { .ciImage($0) } + images: images.map { .ciImage($0) }, + audios: audios ) ] ) } - return generateStream(input: input, isMultimodal: !images.isEmpty) + return generateStream(input: input, isMultimodal: !images.isEmpty || !audios.isEmpty) } public func generateStream(chat: [Chat.Message]) -> AsyncThrowingStream { - let hasImages = chat.contains { !$0.images.isEmpty } - let input = UserInput(chat: chat) - return generateStream(input: input, isMultimodal: hasImages) + generateStream(chat: chat, additionalContext: nil) + } + + public func generateStream( + chat: [Chat.Message], + additionalContext: [String: any Sendable]? + ) -> AsyncThrowingStream { + let input = UserInput(chat: chat, additionalContext: additionalContext) + let hasMedia = !input.images.isEmpty || !input.audios.isEmpty + return generateStream(input: input, isMultimodal: hasMedia) + } + + private func dynamicMultimodalBudget( + hasImages: Bool, + hasAudio: Bool + ) throws -> MultimodalRuntimeBudget? { + guard hasImages || hasAudio else { return nil } + guard let model = loadedModel ?? Self.availableModels.first(where: { $0.id == selectedModel.id }) else { + return nil + } + let headroom = availableHeadroomMB + + guard model.id == Self.e4bModelID else { + return MultimodalRuntimeBudget( + imageSoftTokenCap: hasImages ? 160 : nil, + maxOutputTokens: Self.multimodalMaxOutputTokens, + headroomMB: headroom + ) + } + + guard headroom > Self.e4bMultimodalCriticalHeadroomMB else { + let recommendation = "请关闭后台应用后重试，或减少附件数量。\(currentMultimodalFallbackRecommendation())" + throw MLXError.multimodalMemoryRisk( + model: model.displayName, + headroomMB: headroom, + recommendation: recommendation + ) + } + + let imageSoftTokenCap: Int? + let maxOutputTokens: Int + + // Budget calibration (E4B, measured on-device): + // Actual activation cost ≈ 1 MB/output token + // (observed: 160 tokens consumed 167 MB headroom = 5311-5144 MB footprint delta) + // Safety margin formula: headroom − (imageSoftTokens × 2 MB) − (maxOut × 1 MB) > 300 MB + switch headroom { + case ..<500: + imageSoftTokenCap = hasImages ? 48 : nil + maxOutputTokens = 120 // was 64 — 48×2 + 120×1 + 300 = 516, fits <500 boundary + case ..<700: + imageSoftTokenCap = hasImages ? 64 : nil + maxOutputTokens = 200 // was 128 — 64×2 + 200 + 300 = 628 < 700 ✓ + case ..<900: + imageSoftTokenCap = hasImages ? 80 : nil + maxOutputTokens = 340 // was 192 — 80×2 + 340 + 300 = 800 < 900 ✓ + case ..<1_100: + imageSoftTokenCap = hasImages ? 96 : nil + maxOutputTokens = Self.multimodalMaxOutputTokens // 512: 96×2 + 512 + 300 = 1004 < 1100 ✓ + case ..<1_300: + imageSoftTokenCap = hasImages ? 128 : nil + maxOutputTokens = Self.multimodalMaxOutputTokens + default: + imageSoftTokenCap = hasImages ? 160 : nil + maxOutputTokens = Self.multimodalMaxOutputTokens + } + + print( + "[MEM] multimodal runtime budget — model=\(model.displayName), " + + "headroom=\(headroom) MB, " + + "imageSoftTokenCap=\(imageSoftTokenCap.map(String.init) ?? "n/a"), " + + "maxOutputTokens=\(maxOutputTokens), " + + "audio=\(hasAudio ? 1 : 0)" + ) + + return MultimodalRuntimeBudget( + imageSoftTokenCap: imageSoftTokenCap, + maxOutputTokens: maxOutputTokens, + headroomMB: headroom + ) + } + + private func isThinkingEnabled(for input: UserInput) -> Bool { + if let enabled = input.additionalContext?["enable_thinking"] as? Bool, enabled { + return true + } + + switch input.prompt { + case .text(let text): + return text.contains("<|think|>") + case .chat(let messages): + return messages.contains { $0.content.contains("<|think|>") } + case .messages(let messages): + return messages.contains { message in + if let content = message["content"] as? String { + return content.contains("<|think|>") + } + if let content = message["content"] as? [[String: any Sendable]] { + return content.contains { item in + (item["text"] as? String)?.contains("<|think|>") == true + } + } + return false + } + } + } + + private func dynamicThinkingBudget( + enabled: Bool + ) -> ThinkingRuntimeBudget? { + guard enabled else { return nil } + guard let model = loadedModel ?? Self.availableModels.first(where: { $0.id == selectedModel.id }) else { + return nil + } + + let headroom = availableHeadroomMB + let maxOutputTokens: Int + + if model.id == Self.e4bModelID { + switch headroom { + case ..<500: + maxOutputTokens = 128 + case ..<700: + maxOutputTokens = 192 + case ..<900: + maxOutputTokens = 256 + case ..<1_100: + maxOutputTokens = 384 + case ..<1_300: + maxOutputTokens = 512 + default: + maxOutputTokens = Self.e4bThinkingMaxOutputTokens + } + } else { + switch headroom { + case ..<500: + maxOutputTokens = 192 + case ..<800: + maxOutputTokens = 384 + case ..<1_200: + maxOutputTokens = 512 + default: + maxOutputTokens = Self.e2bThinkingMaxOutputTokens + } + } + + print( + "[MEM] thinking runtime budget — model=\(model.displayName), " + + "headroom=\(headroom) MB, " + + "maxOutputTokens=\(maxOutputTokens)" + ) + + return ThinkingRuntimeBudget( + maxOutputTokens: maxOutputTokens, + headroomMB: headroom + ) + } + + private func dynamicTextBudget( + enabled: Bool + ) -> TextRuntimeBudget? { + guard enabled else { return nil } + guard let model = loadedModel ?? Self.availableModels.first(where: { $0.id == selectedModel.id }) else { + return nil + } + + let headroom = availableHeadroomMB + let maxOutputTokens: Int + + if model.id == Self.e4bModelID { + switch headroom { + case ..<500: + maxOutputTokens = 256 + case ..<700: + maxOutputTokens = 384 + case ..<900: + maxOutputTokens = 512 + case ..<1_100: + maxOutputTokens = 768 + case ..<1_300: + maxOutputTokens = 1_024 + default: + maxOutputTokens = 1_280 + } + } else { + switch headroom { + case ..<500: + maxOutputTokens = 384 + case ..<800: + maxOutputTokens = 768 + case ..<1_200: + maxOutputTokens = 1_024 + default: + maxOutputTokens = 1_536 + } + } + + print( + "[MEM] text runtime budget — model=\(model.displayName), " + + "headroom=\(headroom) MB, " + + "maxOutputTokens=\(maxOutputTokens)" + ) + + return TextRuntimeBudget( + maxOutputTokens: maxOutputTokens, + headroomMB: headroom + ) + } + + private func adjustedTextOutputTokens( + baseMaxOutputTokens: Int, + preparedSequenceLength: Int, + thinkingEnabled: Bool + ) -> Int? { + guard let model = loadedModel ?? Self.availableModels.first(where: { $0.id == selectedModel.id }) else { + return nil + } + + let headroom = availableHeadroomMB + let totalSequenceBudget: Int + + if model.id == Self.e4bModelID { + if thinkingEnabled { + switch headroom { + case ..<500: + totalSequenceBudget = 320 + case ..<700: + totalSequenceBudget = 448 + case ..<900: + totalSequenceBudget = 576 + case ..<1_100: + totalSequenceBudget = 896 + case ..<1_300: + totalSequenceBudget = 1_152 + default: + totalSequenceBudget = 1_280 + } + } else { + switch headroom { + case ..<500: + totalSequenceBudget = 384 + case ..<700: + totalSequenceBudget = 512 + case ..<900: + totalSequenceBudget = 640 + case ..<1_100: + totalSequenceBudget = 768 + case ..<1_300: + totalSequenceBudget = 896 + default: + totalSequenceBudget = 1_024 + } + } + } else { + if thinkingEnabled { + switch headroom { + case ..<500: + totalSequenceBudget = 448 + case ..<800: + totalSequenceBudget = 640 + case ..<1_200: + totalSequenceBudget = 896 + default: + totalSequenceBudget = 1_152 + } + } else { + switch headroom { + case ..<500: + totalSequenceBudget = 512 + case ..<800: + totalSequenceBudget = 768 + case ..<1_200: + totalSequenceBudget = 1_024 + default: + totalSequenceBudget = 1_280 + } + } + } + + let minimumOutputTokens = thinkingEnabled ? 96 : 128 + let adjustedMaxOutputTokens = min( + baseMaxOutputTokens, + max(minimumOutputTokens, totalSequenceBudget - preparedSequenceLength) + ) + + print( + "[MEM] text sequence budget — model=\(model.displayName), " + + "headroom=\(headroom) MB, " + + "preparedTokens=\(preparedSequenceLength), " + + "totalSequenceBudget=\(totalSequenceBudget), " + + "adjustedMaxOutputTokens=\(adjustedMaxOutputTokens)" + ) + + return adjustedMaxOutputTokens + } + + private func currentMultimodalFallbackRecommendation() -> String { + if let e2b = Self.availableModels.first(where: { $0.id == Self.e2bModelID }) { + if isModelAvailable(e2b) { + return "如仍失败，可手动切换到 \(e2b.displayName) 处理图片或音频。" + } + return "如仍失败，可先下载并手动切换到 \(e2b.displayName)。" + } else { + return "如仍失败，请改用更轻量的模型处理图片或音频。" + } } private func ensureForegroundGPUExecution() async throws { @@ -631,6 +1013,13 @@ public class MLXLocalLLMService: LLMEngine { continuation.finish(throwing: MLXError.modelNotLoaded) return } + do { + try await self.ensureAudioCapability(hasAudio: !input.audios.isEmpty) + } catch { + continuation.finish(throwing: error) + return + } + guard let container = modelContainer else { continuation.finish(throwing: MLXError.modelNotLoaded) return @@ -643,11 +1032,39 @@ public class MLXLocalLLMService: LLMEngine { // exceed the 6GB jetsam limit on iPhone. MLX.GPU.clearCache() + let thinkingEnabled = self.isThinkingEnabled(for: input) + let textBudget = self.dynamicTextBudget(enabled: !isMultimodal) + let runtimeBudget: MultimodalRuntimeBudget? + do { + runtimeBudget = try self.dynamicMultimodalBudget( + hasImages: !input.images.isEmpty, + hasAudio: !input.audios.isEmpty + ) + } catch { + continuation.finish(throwing: error) + return + } + let thinkingBudget = self.dynamicThinkingBudget(enabled: thinkingEnabled) + Gemma4Processor.setRuntimeImageSoftTokenCap(runtimeBudget?.imageSoftTokenCap) + defer { + Gemma4Processor.setRuntimeImageSoftTokenCap(nil) + } + let effectiveMaxOutputTokens: Int = { + let multimodalCap = isMultimodal + ? runtimeBudget?.maxOutputTokens ?? Self.multimodalMaxOutputTokens + : maxOutputTokens + let thinkingCap = thinkingBudget?.maxOutputTokens ?? maxOutputTokens + let textCap = textBudget?.maxOutputTokens ?? maxOutputTokens + return min(maxOutputTokens, multimodalCap, thinkingCap, textCap) + }() + var resolvedMaxOutputTokens = effectiveMaxOutputTokens + self.isGenerating = true self.cancelled = false let genStart = CFAbsoluteTimeGetCurrent() var firstTokenTime: Double? = nil var tokenCount = 0 + var hitTokenCap = false let (fp, _) = appMemoryFootprintMB() print("[MEM] generateStream start — footprint: \(Int(fp)) MB, MLX active: \(MLX.GPU.activeMemory / 1_048_576) MB") @@ -656,21 +1073,35 @@ public class MLXLocalLLMService: LLMEngine { try await self.ensureForegroundGPUExecution() _ = try await container.perform { context in try await self.ensureForegroundGPUExecution() - let effectiveMaxOutputTokens = - isMultimodal ? min(maxOutputTokens, Self.multimodalMaxOutputTokens) : maxOutputTokens if isMultimodal { - print("[VLM] multimodal budget — maxOutputTokens=\(effectiveMaxOutputTokens)") + print("[VLM] multimodal budget — maxOutputTokens=\(resolvedMaxOutputTokens)") + } else if thinkingEnabled { + print("[LLM] thinking budget — baseMaxOutputTokens=\(resolvedMaxOutputTokens)") } - let input = try await context.processor.prepare(input: input) + let preparedInput = try await context.processor.prepare(input: input) if isMultimodal { - print("[VLM] prepared sequence length=\(input.text.tokens.dim(1))") + print("[VLM] prepared sequence length=\(preparedInput.text.tokens.dim(1))") + } else { + let preparedSequenceLength = preparedInput.text.tokens.dim(1) + print("[LLM] prepared sequence length=\(preparedSequenceLength)") + resolvedMaxOutputTokens = + self.adjustedTextOutputTokens( + baseMaxOutputTokens: resolvedMaxOutputTokens, + preparedSequenceLength: preparedSequenceLength, + thinkingEnabled: thinkingEnabled + ) ?? resolvedMaxOutputTokens + if textBudget != nil { + print("[LLM] text budget — maxOutputTokens=\(resolvedMaxOutputTokens)") + } else if thinkingEnabled { + print("[LLM] thinking budget — maxOutputTokens=\(resolvedMaxOutputTokens)") + } } try await self.ensureForegroundGPUExecution() return try MLXLMCommon.generate( - input: input, + input: preparedInput, parameters: .init( - maxTokens: effectiveMaxOutputTokens, + maxTokens: resolvedMaxOutputTokens, temperature: samplingTemperature, topP: samplingTopP, topK: samplingTopK @@ -693,7 +1124,12 @@ public class MLXLocalLLMService: LLMEngine { } // Multimodal path uses a tighter generation budget on iPhone. - return tokens.count >= effectiveMaxOutputTokens ? .stop : .more + // If we hit the cap, signal truncation so the caller can append a notice. + if tokens.count >= resolvedMaxOutputTokens { + hitTokenCap = true + return .stop + } + return .more } } @@ -716,6 +1152,15 @@ public class MLXLocalLLMService: LLMEngine { let (fpEnd, _) = appMemoryFootprintMB() print("[MEM] generateStream end — footprint: \(Int(fpEnd)) MB, headroom: \(self.availableHeadroomMB) MB") + // If we hit the token cap mid-sentence, append a visible notice. + // This makes truncation explicit rather than silently dropping content. + if hitTokenCap { + let isChinese = Locale.preferredLanguages.contains { $0.hasPrefix("zh") } + let modeLabel = isChinese + ? (thinkingEnabled ? "思考" : "输出") + : (thinkingEnabled ? "Thinking" : "Output") + continuation.yield("\n\n> ⚠️ \(modeLabel)已达内存安全上限（\(resolvedMaxOutputTokens) tokens），内容可能不完整。") + } continuation.finish() } catch { continuation.finish(throwing: error) @@ -741,21 +1186,44 @@ public class MLXLocalLLMService: LLMEngine { currentLoadTask?.cancel() } - public func prepareForReload() async { - cancel() + public func prepareForReload( + cancelCurrentGeneration: Bool = true, + cancelCurrentLoad: Bool = true + ) async { + cancelled = true + if cancelCurrentGeneration { + currentGenerationTask?.cancel() + } + if cancelCurrentLoad { + currentLoadTask?.cancel() + } while isGenerating || isLoading { try? await Task.sleep(nanoseconds: 50_000_000) } - unload() + unload( + cancelCurrentGeneration: cancelCurrentGeneration, + cancelCurrentLoad: cancelCurrentLoad + ) MLX.GPU.clearCache() try? await Task.sleep(nanoseconds: 150_000_000) } public func unload() { - currentGenerationTask?.cancel() - currentLoadTask?.cancel() + unload(cancelCurrentGeneration: true, cancelCurrentLoad: true) + } + + public func unload( + cancelCurrentGeneration: Bool = true, + cancelCurrentLoad: Bool = true + ) { + if cancelCurrentGeneration { + currentGenerationTask?.cancel() + } + if cancelCurrentLoad { + currentLoadTask?.cancel() + } modelContainer = nil isLoaded = false isLoading = false @@ -776,15 +1244,18 @@ enum MLXError: LocalizedError { case modelNotLoaded case modelDirectoryMissing(String) case gpuExecutionRequiresForeground + case multimodalMemoryRisk(model: String, headroomMB: Int, recommendation: String) var errorDescription: String? { switch self { case .modelNotLoaded: return "MLX model not loaded. Call load() first." - case .modelDirectoryMissing(let path): - return "MLX 模型目录不存在: \(path)" + case .modelDirectoryMissing(let modelName): + return "\(modelName) 模型文件不存在，请先在配置页下载或重新安装。" case .gpuExecutionRequiresForeground: return "应用进入后台时，iPhone 不允许继续提交 GPU 推理任务。" + case .multimodalMemoryRisk(let model, let headroomMB, let recommendation): + return "\(model) 当前剩余内存仅约 \(headroomMB) MB，继续处理图片/音频很可能被系统直接杀掉。\(recommendation)" } } } diff --git a/LLM/PromptBuilder.swift b/LLM/PromptBuilder.swift index eac753c..f2d1abb 100644 --- a/LLM/PromptBuilder.swift +++ b/LLM/PromptBuilder.swift @@ -10,7 +10,22 @@ import Foundation struct PromptBuilder { static let defaultSystemPrompt = "你是 PhoneClaw，一个运行在本地的私人 AI 助手。你完全运行在设备上，不联网。" - static let multimodalSystemPrompt = "你是 PhoneClaw，一个运行在本地设备上的视觉助手。请仅根据图片和用户问题作答，优先识别图中的主要物体、用途、场景和可读文本；如果看不清或不确定，请直接说明，不要编造。用简体中文回答。这是纯图文问答，不要调用任何工具或技能。" + private static let thinkingOpenMarker = "[[PHONECLAW_THINK]]" + private static let thinkingCloseMarker = "[[/PHONECLAW_THINK]]" + private static let thinkingLanguageInstruction = "如果启用了思考模式，思考通道和最终回答都必须使用简体中文，不要使用英文。" + + static func multimodalSystemPrompt(hasImages: Bool, hasAudio: Bool, enableThinking: Bool = false) -> String { + let base: String + if hasAudio && !hasImages { + base = "你是 PhoneClaw，一个运行在本地设备上的音频助手。请把用户提供的音频视为需要分析的素材，而不是用户此刻正在对你说的话。请根据音频和文本任务直接作答，不要擅自改写用户任务，也不要额外追加不存在的意图。听不清或不确定时请明确说明，不要编造。如果用户是在询问音频里说了什么，或明确要求转写、识别、逐字写出，请直接给出识别结果，不要复述用户问题，不要寒暄。如果用户明确要求逐字转写，尽量保留原话，不要改写、总结、润色，也不要把音频内容当成需要你回应的对话。用简体中文回答。这是纯音频问答，不要调用任何工具或技能。" + } else if hasImages && hasAudio { + base = "你是 PhoneClaw，一个运行在本地设备上的多模态助手。请把用户提供的音频视为需要分析的素材，而不是用户此刻正在对你说的话。请根据用户提供的图片、音频和文本直接作答，不要擅自改写用户任务，也不要额外追加不存在的意图。看不清、听不清或不确定时请直接说明，不要编造。如果用户是在询问音频里说了什么，或明确要求转写、识别、逐字写出，请直接给出识别结果，不要复述用户问题，不要寒暄。如果用户明确要求逐字转写，尽量保留原话，不要改写、总结、润色，也不要把音频内容当成需要你回应的对话。用简体中文回答。这是纯多模态问答，不要调用任何工具或技能。" + } else { + base = "你是 PhoneClaw，一个运行在本地设备上的视觉助手。请仅根据图片和用户问题直接作答，并严格遵守以下规则：1. 默认先直接给结论，控制在1到2句内；2. 除非用户明确要求详细说明，否则禁止分点、禁止长篇分析、禁止列举多种可能性；3. 不要写“根据您提供的图片”“从画面中可以看到”等铺垫；4. 如果看不清或不确定，只需简短说明“看不清，像……”，不要编造。优先识别图中的主要物体、用途、场景和可读文本。用简体中文回答。这是纯图文问答，不要调用任何工具或技能。" + } + + return enableThinking ? base + "\n" + thinkingLanguageInstruction : base + } private static func imagePromptSuffix(count: Int) -> String { guard count > 0 else { return "" } @@ -39,6 +54,32 @@ struct PromptBuilder { return head + "\n\n" + trimmedExtra + "\n\n" } + private static func sanitizedAssistantHistoryContent(_ text: String) -> String { + var result = text + + while let openRange = result.range(of: thinkingOpenMarker) { + if let closeRange = result.range(of: thinkingCloseMarker, range: openRange.upperBound.. String { + let rawBase = (systemPrompt ?? defaultSystemPrompt).trimmingCharacters(in: .whitespacesAndNewlines) + let firstParagraph = rawBase + .components(separatedBy: "\n\n") + .first? + .trimmingCharacters(in: .whitespacesAndNewlines) + + let base = (firstParagraph?.isEmpty == false ? firstParagraph! : defaultSystemPrompt) + return base + "\n\n当前这轮只是普通文字对话，不需要调用任何设备技能或工具。请直接回答用户，避免提到 Skill、load_skill、tool_call 或设备操作流程。用简体中文回答，默认简洁。" + } + /// 构造完整 Prompt（包含工具定义 + 对话历史） static func build( userMessage: String, @@ -46,15 +87,19 @@ struct PromptBuilder { tools: [SkillInfo], history: [ChatMessage] = [], systemPrompt: String? = nil, + enableThinking: Bool = false, historyDepth: Int = 4 // 动态传入，根据当前内存 headroom 估算 ) -> String { let isMultimodalTurn = currentImageCount > 0 var prompt = "<|turn>system\n" + if enableThinking { + prompt += "<|think|>" + } // ★ 使用自定义 system prompt（如果有），否则用默认 let basePrompt = isMultimodalTurn - ? multimodalSystemPrompt + ? multimodalSystemPrompt(hasImages: currentImageCount > 0, hasAudio: false, enableThinking: enableThinking) : (systemPrompt ?? defaultSystemPrompt) // 构建 Skill 概要列表（只列名称 + 一句话描述，不暴露 Tool） @@ -77,6 +122,10 @@ struct PromptBuilder { } } + if enableThinking && !isMultimodalTurn { + prompt += "\n\n" + thinkingLanguageInstruction + } + prompt += "\n\n" // 对话历史（动态深度，由 llm.safeHistoryDepth 控制） @@ -95,7 +144,8 @@ struct PromptBuilder { // image placeholders for the current turn and its tool follow-ups. prompt += "<|turn>user\n\(msg.content)\n" case .assistant: - prompt += "<|turn>model\n\(msg.content)\n" + let assistantContent = sanitizedAssistantHistoryContent(msg.content) + prompt += "<|turn>model\n\(assistantContent)\n" case .system: if let skillName = msg.skillName { prompt += "<|turn>model\n\n{\"name\": \"\(skillName)\", \"arguments\": {}}\n\n" @@ -113,6 +163,43 @@ struct PromptBuilder { return prompt } + static func buildLightweightTextPrompt( + userMessage: String, + history: [ChatMessage] = [], + systemPrompt: String? = nil, + enableThinking: Bool = false, + historyDepth: Int = 2 + ) -> String { + var prompt = "<|turn>system\n" + if enableThinking { + prompt += "<|think|>" + } + prompt += lightweightTextSystemPrompt(systemPrompt: systemPrompt) + if enableThinking { + prompt += "\n\n" + thinkingLanguageInstruction + } + prompt += "\n\n" + + let recentHistory = history.suffix(historyDepth) + for msg in recentHistory { + if msg.role == .user && msg.id == recentHistory.last?.id { continue } + switch msg.role { + case .user: + prompt += "<|turn>user\n\(msg.content)\n" + case .assistant: + let assistantContent = sanitizedAssistantHistoryContent(msg.content) + guard !assistantContent.isEmpty else { continue } + prompt += "<|turn>model\n\(assistantContent)\n" + case .system, .skillResult: + continue + } + } + + prompt += "<|turn>user\n\(userMessage)\n" + prompt += "<|turn>model\n" + return prompt + } + /// `load_skill` 之后重新推理： /// 直接把已加载的 Skill 指令注入 system turn，再重新回答原问题。 /// 这样比“把 tool_call + skill body + retry 指令继续拼接”更稳定，也更省 prefill。 diff --git a/Packages/InferenceKit/Libraries/MLXLMCommon/Chat.swift b/Packages/InferenceKit/Libraries/MLXLMCommon/Chat.swift index 17ebde1..19d5c20 100644 --- a/Packages/InferenceKit/Libraries/MLXLMCommon/Chat.swift +++ b/Packages/InferenceKit/Libraries/MLXLMCommon/Chat.swift @@ -14,32 +14,46 @@ public enum Chat { /// Array of video data associated with the message. public var videos: [UserInput.Video] + /// Array of audio data associated with the message. + public var audios: [UserInput.Audio] + public init( role: Role, content: String, images: [UserInput.Image] = [], - videos: [UserInput.Video] = [] + videos: [UserInput.Video] = [], + audios: [UserInput.Audio] = [] ) { self.role = role self.content = content self.images = images self.videos = videos + self.audios = audios } public static func system( - _ content: String, images: [UserInput.Image] = [], videos: [UserInput.Video] = [] + _ content: String, + images: [UserInput.Image] = [], + videos: [UserInput.Video] = [], + audios: [UserInput.Audio] = [] ) -> Self { - Self(role: .system, content: content, images: images, videos: videos) + Self(role: .system, content: content, images: images, videos: videos, audios: audios) } public static func assistant( - _ content: String, images: [UserInput.Image] = [], videos: [UserInput.Video] = [] + _ content: String, + images: [UserInput.Image] = [], + videos: [UserInput.Video] = [], + audios: [UserInput.Audio] = [] ) -> Self { - Self(role: .assistant, content: content, images: images, videos: videos) + Self(role: .assistant, content: content, images: images, videos: videos, audios: audios) } public static func user( - _ content: String, images: [UserInput.Image] = [], videos: [UserInput.Video] = [] + _ content: String, + images: [UserInput.Image] = [], + videos: [UserInput.Video] = [], + audios: [UserInput.Audio] = [] ) -> Self { - Self(role: .user, content: content, images: images, videos: videos) + Self(role: .user, content: content, images: images, videos: videos, audios: audios) } public static func tool(_ content: String) -> Self { diff --git a/Packages/InferenceKit/Libraries/MLXLMCommon/LanguageModel.swift b/Packages/InferenceKit/Libraries/MLXLMCommon/LanguageModel.swift index 5142ccb..ca0a847 100644 --- a/Packages/InferenceKit/Libraries/MLXLMCommon/LanguageModel.swift +++ b/Packages/InferenceKit/Libraries/MLXLMCommon/LanguageModel.swift @@ -36,6 +36,7 @@ public struct LMInput { public let text: Text public let image: ProcessedImage? public let video: ProcessedVideo? + public let audio: ProcessedAudio? /// Representation of tokenized input text. public struct Text { @@ -71,12 +72,15 @@ public struct LMInput { public let pixels: MLXArray /// Time, height, and width of the images public let frames: [THW]? + /// Desired number of soft tokens after visual pooling/projecting. + public let softTokenCount: Int? public init( - pixels: MLXArray, frames: [THW]? = nil + pixels: MLXArray, frames: [THW]? = nil, softTokenCount: Int? = nil ) { self.pixels = pixels self.frames = frames + self.softTokenCount = softTokenCount } } @@ -95,17 +99,48 @@ public struct LMInput { } } + /// Representation of prepared input audio. + public struct ProcessedAudio { + public let features: MLXArray + public let invalidMask: MLXArray? + public let sampleRate: Double + public let channelCount: Int + public let duration: TimeInterval + public let sampleCount: Int + public let tokenCount: Int + + public init( + features: MLXArray, + invalidMask: MLXArray? = nil, + sampleRate: Double, + channelCount: Int, + duration: TimeInterval, + sampleCount: Int, + tokenCount: Int + ) { + self.features = features + self.invalidMask = invalidMask + self.sampleRate = sampleRate + self.channelCount = channelCount + self.duration = duration + self.sampleCount = sampleCount + self.tokenCount = tokenCount + } + } + public init(tokens: MLXArray, mask: MLXArray? = nil) { self.init(text: .init(tokens: tokens, mask: mask)) } public init( text: LMInput.Text, image: LMInput.ProcessedImage? = nil, - video: LMInput.ProcessedVideo? = nil + video: LMInput.ProcessedVideo? = nil, + audio: LMInput.ProcessedAudio? = nil ) { self.text = text self.image = image self.video = video + self.audio = audio } } diff --git a/Packages/InferenceKit/Libraries/MLXLMCommon/UserInput.swift b/Packages/InferenceKit/Libraries/MLXLMCommon/UserInput.swift index 0a2e63d..40a139e 100644 --- a/Packages/InferenceKit/Libraries/MLXLMCommon/UserInput.swift +++ b/Packages/InferenceKit/Libraries/MLXLMCommon/UserInput.swift @@ -49,6 +49,29 @@ public struct UserInput { } } + /// Representation of raw audio input. + public enum Audio { + case pcm(PCM) + case url(URL) + + public struct PCM: Sendable { + public let samples: [Float] + public let sampleRate: Double + public let channelCount: Int + + public init(samples: [Float], sampleRate: Double, channelCount: Int) { + self.samples = samples + self.sampleRate = sampleRate + self.channelCount = channelCount + } + + public var duration: TimeInterval { + guard sampleRate > 0 else { return 0 } + return Double(samples.count) / sampleRate + } + } + } + /// Representation of a video resource. public enum Video { case avAsset(AVAsset) @@ -161,6 +184,9 @@ public struct UserInput { self.videos = messages.reduce(into: []) { result, message in result.append(contentsOf: message.videos) } + self.audios = messages.reduce(into: []) { result, message in + result.append(contentsOf: message.audios) + } } } } @@ -177,6 +203,12 @@ public struct UserInput { /// collect the videos from the chat messages, otherwise these are the stored videos with the ``UserInput``. public var videos = [Video]() + /// The audio clips associated with the `UserInput`. + /// + /// If the ``prompt-swift.property`` is a ``Prompt-swift.enum/chat(_:)`` this will + /// collect the audios from the chat messages, otherwise these are the stored audios with the ``UserInput``. + public var audios = [Audio]() + public var tools: [ToolSpec]? /// Additional values provided for the chat template rendering context @@ -195,12 +227,15 @@ public struct UserInput { /// - ``Prompt-swift.enum/text(_:)`` /// - ``init(chat:tools:additionalContext:)`` public init( - prompt: String, images: [Image] = [Image](), videos: [Video] = [Video](), + prompt: String, + images: [Image] = [Image](), + videos: [Video] = [Video](), + audios: [Audio] = [Audio](), tools: [ToolSpec]? = nil, additionalContext: [String: any Sendable]? = nil ) { self.prompt = .chat([ - .user(prompt, images: images, videos: videos) + .user(prompt, images: images, videos: videos, audios: audios) ]) self.tools = tools self.additionalContext = additionalContext @@ -240,13 +275,17 @@ public struct UserInput { /// - ``Prompt-swift.enum/text(_:)`` /// - ``init(chat:tools:additionalContext:)`` public init( - messages: [Message], images: [Image] = [Image](), videos: [Video] = [Video](), + messages: [Message], + images: [Image] = [Image](), + videos: [Video] = [Video](), + audios: [Audio] = [Audio](), tools: [ToolSpec]? = nil, additionalContext: [String: any Sendable]? = nil ) { self.prompt = .messages(messages) self.images = images self.videos = videos + self.audios = audios self.tools = tools self.additionalContext = additionalContext } @@ -289,6 +328,9 @@ public struct UserInput { self.videos = chat.reduce(into: []) { result, message in result.append(contentsOf: message.videos) } + self.audios = chat.reduce(into: []) { result, message in + result.append(contentsOf: message.audios) + } self.processing = processing self.tools = tools @@ -313,6 +355,7 @@ public struct UserInput { prompt: Prompt, images: [Image] = [Image](), videos: [Video] = [Video](), + audios: [Audio] = [Audio](), processing: Processing = .init(), tools: [ToolSpec]? = nil, additionalContext: [String: any Sendable]? = nil ) { @@ -321,6 +364,7 @@ public struct UserInput { case .text, .messages: self.images = images self.videos = videos + self.audios = audios case .chat: break } @@ -341,6 +385,8 @@ internal enum UserInputError: LocalizedError { case notImplemented case unableToLoad(URL) case arrayError(String) + case audioLoadingNotImplemented(URL) + case audioCountExceeded(Int) var errorDescription: String? { switch self { @@ -350,6 +396,10 @@ internal enum UserInputError: LocalizedError { return String(localized: "Unable to load image from URL: \(url.path).") case .arrayError(let message): return String(localized: "Error processing image array: \(message).") + case .audioLoadingNotImplemented(let url): + return String(localized: "Audio loading from URL is not implemented yet: \(url.path).") + case .audioCountExceeded(let count): + return String(localized: "Only a single audio clip is supported right now, received \(count).") } } } diff --git a/PhoneClaw.xcodeproj/project.pbxproj b/PhoneClaw.xcodeproj/project.pbxproj index 614ae45..f250256 100644 --- a/PhoneClaw.xcodeproj/project.pbxproj +++ b/PhoneClaw.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ 20B8C55E032BA07B3E640E80 /* Skills.swift in Sources */ = {isa = PBXBuildFile; fileRef = 571908036E6DCED1D58C1E9E /* Skills.swift */; }; + 2F4A6C8E0A1B2C3D4E5F6071 /* AudioCaptureService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4A6C8E0A1B2C3D4E5F6070 /* AudioCaptureService.swift */; }; 45B7FEC6C4D055F44A87BFA9 /* PhoneClawApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4DD9808FE93325C3CE4CFA00 /* PhoneClawApp.swift */; }; 5084E1CD2E831BC05F5E4374 /* PromptBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E7C06DA449D1395217A42D5 /* PromptBuilder.swift */; }; 9412304167DACB86E5D60D49 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F5ED928BF14864713A9A9D16 /* ContentView.swift */; }; @@ -30,6 +31,8 @@ AA1000000000000000000017 /* MLXLLM in Frameworks */ = {isa = PBXBuildFile; productRef = AA5000000000000000000007 /* MLXLLM */; }; AA1000000000000000000018 /* Gemma4VisionModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA2000000000000000000018 /* Gemma4VisionModel.swift */; }; AA1000000000000000000019 /* Gemma4Processor.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA2000000000000000000019 /* Gemma4Processor.swift */; }; + AA1000000000000000000020 /* Gemma4AudioFeatureExtractor.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA2000000000000000000020 /* Gemma4AudioFeatureExtractor.swift */; }; + AA1000000000000000000021 /* Gemma4AudioModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA2000000000000000000021 /* Gemma4AudioModel.swift */; }; B881A2E7F9483D5ED2AC206B /* SkillsManagerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D269F8BFCAB68D82440F704B /* SkillsManagerView.swift */; }; B8A84E4D0DDDE011B5929DA2 /* AgentEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = E18A47114A10132E25E220F8 /* AgentEngine.swift */; }; BD75E7E5620A8FF0A868E9F0 /* libPods-PhoneClaw.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 4F4ACB3F4531FF61E0AADA07 /* libPods-PhoneClaw.a */; }; @@ -37,12 +40,14 @@ C2B3C4D5E6F718293041B2C3 /* ToolRegistry.swift in Sources */ = {isa = PBXBuildFile; fileRef = C2B3C4D5E6F718293041B2C4 /* ToolRegistry.swift */; }; C3C4D5E6F7081A2B3C52D3E4 /* ChatModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3C4D5E6F7081A2B3C52D3E5 /* ChatModels.swift */; }; C4D5E6F70819A2B3C463E4F5 /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = C4D5E6F70819A2B3C463E4F6 /* Theme.swift */; }; + D5E6F7081A2B3C4D5E6F7087 /* gemma-4-e4b-it-4bit in Resources */ = {isa = PBXBuildFile; fileRef = D5E6F7081A2B3C4D5E6F7086 /* gemma-4-e4b-it-4bit */; }; ICONASSETS0000000000002 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = ICONASSETS0000000000001 /* Assets.xcassets */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ 098B30935B91CD0BEF3A5056 /* Pods-PhoneClaw.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-PhoneClaw.release.xcconfig"; path = "Target Support Files/Pods-PhoneClaw/Pods-PhoneClaw.release.xcconfig"; sourceTree = ""; }; 2E7C06DA449D1395217A42D5 /* PromptBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PromptBuilder.swift; sourceTree = ""; }; + 2F4A6C8E0A1B2C3D4E5F6070 /* AudioCaptureService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCaptureService.swift; sourceTree = ""; }; 38FC1001EB728212335AC141 /* PhoneClaw.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = PhoneClaw.entitlements; sourceTree = ""; }; 408F2AE98EA54717A74FA4B1 /* PhoneClaw.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = PhoneClaw.app; sourceTree = BUILT_PRODUCTS_DIR; }; 4DD9808FE93325C3CE4CFA00 /* PhoneClawApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PhoneClawApp.swift; sourceTree = ""; }; @@ -61,6 +66,8 @@ AA2000000000000000000009 /* MLXTokenizersLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MLXTokenizersLoader.swift; sourceTree = ""; }; AA2000000000000000000018 /* Gemma4VisionModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Gemma4VisionModel.swift; sourceTree = ""; }; AA2000000000000000000019 /* Gemma4Processor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Gemma4Processor.swift; sourceTree = ""; }; + AA2000000000000000000020 /* Gemma4AudioFeatureExtractor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Gemma4AudioFeatureExtractor.swift; sourceTree = ""; }; + AA2000000000000000000021 /* Gemma4AudioModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Gemma4AudioModel.swift; sourceTree = ""; }; C1A2B3D4E5F607182930A1B3 /* SkillLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SkillLoader.swift; sourceTree = ""; }; C2B3C4D5E6F718293041B2C4 /* ToolRegistry.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ToolRegistry.swift; sourceTree = ""; }; C3C4D5E6F7081A2B3C52D3E5 /* ChatModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatModels.swift; sourceTree = ""; }; @@ -137,6 +144,8 @@ children = ( AA2000000000000000000003 /* Gemma4Config.swift */, AA2000000000000000000004 /* Gemma4LanguageModel.swift */, + AA2000000000000000000020 /* Gemma4AudioFeatureExtractor.swift */, + AA2000000000000000000021 /* Gemma4AudioModel.swift */, AA2000000000000000000005 /* Gemma4Model.swift */, AA2000000000000000000006 /* Gemma4Registration.swift */, AA2000000000000000000007 /* Gemma4RoPE.swift */, @@ -185,6 +194,7 @@ E0660A5AE3C683B7B20600EE /* Agent */ = { isa = PBXGroup; children = ( + 2F4A6C8E0A1B2C3D4E5F6070 /* AudioCaptureService.swift */, E18A47114A10132E25E220F8 /* AgentEngine.swift */, ); path = Agent; @@ -274,14 +284,15 @@ /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ - E6F7081A2B3C4D5E6F708100 /* Resources */ = { - isa = PBXResourcesBuildPhase; - buildActionMask = 2147483647; - files = ( - ICONASSETS0000000000002 /* Assets.xcassets in Resources */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; + E6F7081A2B3C4D5E6F708100 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D5E6F7081A2B3C4D5E6F7087 /* gemma-4-e4b-it-4bit in Resources */, + ICONASSETS0000000000002 /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ @@ -314,12 +325,15 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 2F4A6C8E0A1B2C3D4E5F6071 /* AudioCaptureService.swift in Sources */, B8A84E4D0DDDE011B5929DA2 /* AgentEngine.swift in Sources */, C3C4D5E6F7081A2B3C52D3E4 /* ChatModels.swift in Sources */, 97E96A2FBFFAF609F276A896 /* ConfigurationsView.swift in Sources */, 9412304167DACB86E5D60D49 /* ContentView.swift in Sources */, AA1000000000000000000003 /* Gemma4Config.swift in Sources */, AA1000000000000000000004 /* Gemma4LanguageModel.swift in Sources */, + AA1000000000000000000020 /* Gemma4AudioFeatureExtractor.swift in Sources */, + AA1000000000000000000021 /* Gemma4AudioModel.swift in Sources */, AA1000000000000000000005 /* Gemma4Model.swift in Sources */, AA1000000000000000000006 /* Gemma4Registration.swift in Sources */, AA1000000000000000000018 /* Gemma4VisionModel.swift in Sources */, @@ -469,9 +483,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_ENTITLEMENTS = PhoneClaw.entitlements; - CODE_SIGN_IDENTITY = "iPhone Developer"; CURRENT_PROJECT_VERSION = 1; - INFOPLIST_FILE = Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", @@ -490,9 +502,7 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_ENTITLEMENTS = PhoneClaw.entitlements; - CODE_SIGN_IDENTITY = "iPhone Developer"; CURRENT_PROJECT_VERSION = 1; - INFOPLIST_FILE = Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", diff --git a/README.md b/README.md index 03a7386..282a34c 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,12 @@ PhoneClaw 是一个运行在 iPhone 上的本地 AI Agent。它使用 Gemma 4 在设备端完成推理，不依赖云端，不上传聊天内容。 +## 2026-04-07 更新 + +- 新增语音功能，支持录音发送，并可对语音内容进行分析和识别 +- 新增思考模式，可在聊天页右上角按需开启 +- 优化了内存管理与推理预算，长回答、多模态和模型切换场景更稳定 + ## 2026-04-06 更新 - 默认安装方式已恢复为空壳安装，模型改为在手机端按需下载 diff --git a/Skills/ToolRegistry.swift b/Skills/ToolRegistry.swift index bdb082d..cd99e17 100644 --- a/Skills/ToolRegistry.swift +++ b/Skills/ToolRegistry.swift @@ -1,3 +1,4 @@ +import AVFoundation import Contacts import EventKit import Foundation @@ -9,6 +10,7 @@ import UIKit // SKILL.md 通过 allowed-tools 字段引用工具名。 enum AppPermissionKind: String, CaseIterable, Identifiable { + case microphone case calendar case reminders case contacts @@ -17,6 +19,7 @@ enum AppPermissionKind: String, CaseIterable, Identifiable { var title: String { switch self { + case .microphone: return "麦克风" case .calendar: return "日历" case .reminders: return "提醒事项" case .contacts: return "通讯录" @@ -25,6 +28,7 @@ enum AppPermissionKind: String, CaseIterable, Identifiable { var description: String { switch self { + case .microphone: return "允许录音并采集实时音频输入" case .calendar: return "允许创建和写入日历事项" case .reminders: return "允许创建提醒和待办" case .contacts: return "允许查询、保存和删除联系人" @@ -33,6 +37,7 @@ enum AppPermissionKind: String, CaseIterable, Identifiable { var icon: String { switch self { + case .microphone: return "mic" case .calendar: return "calendar" case .reminders: return "bell" case .contacts: return "person.crop.circle" @@ -122,6 +127,18 @@ class ToolRegistry { func authorizationStatus(for kind: AppPermissionKind) -> AppPermissionStatus { switch kind { + case .microphone: + switch AVAudioSession.sharedInstance().recordPermission { + case .granted: + return .granted + case .denied: + return .denied + case .undetermined: + return .notDetermined + @unknown default: + return .restricted + } + case .calendar: let status = EKEventStore.authorizationStatus(for: .event) switch status { @@ -179,6 +196,8 @@ class ToolRegistry { func requestAccess(for kind: AppPermissionKind) async throws -> Bool { switch kind { + case .microphone: + return try await requestMicrophoneAccess() case .calendar: return try await requestCalendarWriteAccess() case .reminders: @@ -188,6 +207,14 @@ class ToolRegistry { } } + private func requestMicrophoneAccess() async throws -> Bool { + await withCheckedContinuation { continuation in + AVAudioSession.sharedInstance().requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + } + private func requestCalendarWriteAccess() async throws -> Bool { try await withCheckedThrowingContinuation { continuation in eventStore.requestWriteOnlyAccessToEvents { granted, error in diff --git a/UI/ChatModels.swift b/UI/ChatModels.swift index f10af00..0fceb1a 100644 --- a/UI/ChatModels.swift +++ b/UI/ChatModels.swift @@ -14,6 +14,7 @@ struct SkillCard: Identifiable, Equatable { struct ResponseBlock: Identifiable, Equatable { let id: UUID var skills: [SkillCard] = [] + var thinkingText: String? var responseText: String? var isThinking: Bool } @@ -35,6 +36,41 @@ enum DisplayItem: Identifiable { /// 用于纯思考占位的稳定 ID private let thinkingPlaceholderID = UUID() +private let thinkingOpenMarker = "[[PHONECLAW_THINK]]" +private let thinkingCloseMarker = "[[/PHONECLAW_THINK]]" + +private func splitThinkingAndResponse(from text: String) -> (thinking: String?, response: String?) { + guard !text.isEmpty else { return (nil, nil) } + + var remaining = text + var thinkingChunks: [String] = [] + + while let openRange = remaining.range(of: thinkingOpenMarker) { + let thoughtStart = openRange.upperBound + if let closeRange = remaining.range(of: thinkingCloseMarker, range: thoughtStart.. [DisplayItem] { var items: [DisplayItem] = [] @@ -72,18 +108,29 @@ func buildDisplayItems(from messages: [ChatMessage], isProcessing: Bool) -> [Dis } block?.skills.append(card) } + } else if !msg.content.isEmpty { + if block == nil { block = ResponseBlock(id: msg.id, isThinking: false) } + let parsed = splitThinkingAndResponse(from: msg.content) + if let thinking = parsed.thinking { + block?.thinkingText = thinking + } + block?.responseText = parsed.response } case .skillResult: break case .assistant: if block == nil { block = ResponseBlock(id: msg.id, isThinking: false) } if msg.content != "▍" && !msg.content.isEmpty { - block?.responseText = msg.content + let parsed = splitThinkingAndResponse(from: msg.content) + if let thinking = parsed.thinking { + block?.thinkingText = thinking + } + block?.responseText = parsed.response } } } - if var b = block, !b.skills.isEmpty || b.responseText != nil { + if var b = block, !b.skills.isEmpty || b.thinkingText != nil || b.responseText != nil { if isProcessing && !b.skills.isEmpty && b.responseText == nil { b.isThinking = true } @@ -91,7 +138,7 @@ func buildDisplayItems(from messages: [ChatMessage], isProcessing: Bool) -> [Dis } if isProcessing { - let hasAI = block.map { !$0.skills.isEmpty || $0.responseText != nil } ?? false + let hasAI = block.map { !$0.skills.isEmpty || $0.thinkingText != nil || $0.responseText != nil } ?? false if !hasAI { items.append(.response(ResponseBlock(id: thinkingPlaceholderID, isThinking: true))) } diff --git a/UI/ConfigurationsView.swift b/UI/ConfigurationsView.swift index e300b48..ddd0596 100644 --- a/UI/ConfigurationsView.swift +++ b/UI/ConfigurationsView.swift @@ -457,6 +457,8 @@ struct ConfigurationsView: View { private func permissionTitle(_ kind: AppPermissionKind) -> String { switch kind { + case .microphone: + return localized("麦克风", "Microphone") case .calendar: return localized("日历", "Calendar") case .reminders: @@ -468,6 +470,8 @@ struct ConfigurationsView: View { private func permissionDescription(_ kind: AppPermissionKind) -> String { switch kind { + case .microphone: + return localized("允许录音并采集实时音频输入", "Allow recording and capturing realtime audio input") case .calendar: return localized("允许创建和写入日历事项", "Allow creating and writing calendar events") case .reminders: diff --git a/UI/ContentView.swift b/UI/ContentView.swift index 2a5a2e5..5e88c77 100644 --- a/UI/ContentView.swift +++ b/UI/ContentView.swift @@ -1,4 +1,7 @@ import SwiftUI +#if canImport(AVFoundation) +import AVFoundation +#endif #if canImport(PhotosUI) import PhotosUI #endif @@ -6,11 +9,16 @@ import PhotosUI import UIKit #endif +private func localizedThinkingText(_ zh: String, _ en: String) -> String { + Locale.preferredLanguages.contains { $0.hasPrefix("zh") } ? zh : en +} + // MARK: - 主入口 struct ContentView: View { @Environment(\.scenePhase) private var scenePhase @State private var engine = AgentEngine() + @State private var audioCapture = AudioCaptureService() @State private var inputText = "" @State private var selectedImages: [UIImage] = [] @State private var selectedPhotoItem: PhotosPickerItem? @@ -18,12 +26,49 @@ struct ContentView: View { @State private var showSkillsManager = false /// 记录每个 skill 卡片的展开状态（key = SkillCard.id） @State private var expandedSkills: Set = [] + /// 记录每个 THINK 卡片的展开状态（key = ResponseBlock.id） + @State private var expandedThoughts: Set = [] @FocusState private var isInputFocused: Bool private var displayItems: [DisplayItem] { buildDisplayItems(from: engine.messages, isProcessing: engine.isProcessing) } + private var scrollAnchorState: String { + guard let last = displayItems.last else { + return "empty:\(engine.isProcessing)" + } + + switch last { + case .user(let msg): + return [ + "user", + msg.id.uuidString, + String(msg.content.count), + String(msg.images.count), + String(msg.audios.count) + ].joined(separator: ":") + case .response(let block): + let skillSignature = block.skills.map { + [ + $0.id.uuidString, + $0.skillName, + $0.skillStatus ?? "", + $0.toolName ?? "" + ].joined(separator: "|") + }.joined(separator: "||") + + return [ + "response", + block.id.uuidString, + block.thinkingText ?? "", + block.responseText ?? "", + block.isThinking ? "1" : "0", + skillSignature + ].joined(separator: ":") + } + } + var body: some View { ZStack { Theme.bg.ignoresSafeArea() @@ -37,6 +82,8 @@ struct ContentView: View { chatList } + composerAttachmentsPanel + if engine.messages.isEmpty { skillChips.padding(.bottom, 8) } @@ -50,8 +97,12 @@ struct ContentView: View { await loadSelectedPhoto() } .onChange(of: scenePhase) { _, newPhase in - guard newPhase != .active else { return } + if newPhase == .active { + audioCapture.refreshPermissionStatus() + return + } engine.cancelActiveGeneration() + _ = audioCapture.stopCapture() } .sheet(isPresented: $showConfigurations) { ConfigurationsView(engine: engine) @@ -70,12 +121,18 @@ struct ContentView: View { ForEach(displayItems) { item in switch item { case .user(let msg): - UserBubble(text: msg.content, images: msg.images.compactMap(\.uiImage)) + UserBubble( + text: msg.content, + images: msg.images.compactMap(\.uiImage), + audios: msg.audios + ) case .response(let block): AIResponseView( block: block, expandedSkills: expandedSkills, - onToggle: { toggleExpand($0) } + isThinkingExpanded: expandedThoughts.contains(block.id), + onToggle: { toggleExpand($0) }, + onToggleThinking: { toggleThinking(block.id) } ) } } @@ -84,15 +141,24 @@ struct ContentView: View { .padding(.vertical, 20) } .scrollIndicators(.hidden) + .onAppear { scrollTo(proxy, animated: false) } .onChange(of: engine.messages.count) { scrollTo(proxy) } .onChange(of: engine.isProcessing) { scrollTo(proxy) } + .onChange(of: scrollAnchorState) { scrollTo(proxy) } } } - private func scrollTo(_ proxy: ScrollViewProxy) { + private func scrollTo(_ proxy: ScrollViewProxy, animated: Bool = true) { guard let last = displayItems.last else { return } - withAnimation(.easeOut(duration: 0.3)) { - proxy.scrollTo(last.id, anchor: .bottom) + let lastID = last.id + DispatchQueue.main.async { + if animated { + withAnimation(.easeOut(duration: 0.2)) { + proxy.scrollTo(lastID, anchor: .bottom) + } + } else { + proxy.scrollTo(lastID, anchor: .bottom) + } } } @@ -106,6 +172,21 @@ struct ContentView: View { } } + private func toggleThinking(_ id: UUID) { + withAnimation(.easeInOut(duration: 0.25)) { + if expandedThoughts.contains(id) { + expandedThoughts.remove(id) + } else { + expandedThoughts.insert(id) + } + } + } + + private func toggleThinkingMode() { + engine.config.enableThinking.toggle() + engine.applySamplingConfig() + } + // MARK: - 顶部栏 private var topBar: some View { @@ -136,6 +217,23 @@ struct ContentView: View { // 右：Skills + 设置 HStack(spacing: 6) { + Button(action: toggleThinkingMode) { + HStack(spacing: 6) { + Image(systemName: engine.config.enableThinking ? "sparkles" : "sparkles") + .font(.system(size: 11, weight: .semibold)) + Text(localizedThinkingText("思考", "Think")) + .font(.system(size: 11, weight: .semibold, design: .rounded)) + } + .foregroundStyle(engine.config.enableThinking ? Theme.bg : Theme.textSecondary) + .padding(.horizontal, 10) + .frame(height: 34) + .background( + engine.config.enableThinking ? Theme.accent : Theme.bgElevated, + in: RoundedRectangle(cornerRadius: 9) + ) + } + .buttonStyle(.plain) + Button(action: { showSkillsManager = true }) { Image(systemName: "puzzlepiece.extension") .font(.system(size: 14, weight: .medium)) @@ -215,36 +313,6 @@ struct ContentView: View { private var inputBar: some View { VStack(spacing: 10) { - if !selectedImages.isEmpty { - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: 10) { - ForEach(Array(selectedImages.enumerated()), id: \.offset) { index, image in - ZStack(alignment: .topTrailing) { - Image(uiImage: image) - .resizable() - .scaledToFill() - .frame(width: 72, height: 72) - .clipShape(RoundedRectangle(cornerRadius: 14)) - .overlay( - RoundedRectangle(cornerRadius: 14) - .strokeBorder(Theme.border, lineWidth: 1) - ) - - Button { - selectedImages.remove(at: index) - } label: { - Image(systemName: "xmark.circle.fill") - .font(.system(size: 18)) - .foregroundStyle(.white, Color.black.opacity(0.65)) - } - .offset(x: 6, y: -6) - } - } - } - .padding(.horizontal, Theme.inputPadH) - } - } - HStack(spacing: 10) { #if canImport(PhotosUI) PhotosPicker(selection: $selectedPhotoItem, matching: .images) { @@ -257,6 +325,22 @@ struct ContentView: View { .buttonStyle(.plain) #endif + Button { + Task { + _ = await audioCapture.toggleCapture() + } + } label: { + Image(systemName: audioCapture.isCapturing ? "stop.fill" : "mic.fill") + .font(.system(size: 15, weight: .medium)) + .foregroundStyle(audioCapture.isCapturing ? Theme.bg : Theme.textSecondary) + .frame(width: 34, height: 34) + .background( + audioCapture.isCapturing ? Theme.accent : Theme.bgElevated, + in: RoundedRectangle(cornerRadius: 9) + ) + } + .buttonStyle(.plain) + #if os(macOS) TextField("Message…", text: $inputText) .textFieldStyle(.plain) @@ -298,19 +382,106 @@ struct ContentView: View { } } + @ViewBuilder + private var composerAttachmentsPanel: some View { + if audioCapture.isCapturing + || audioCapture.latestSnapshot() != nil + || audioCapture.lastErrorMessage != nil + || !selectedImages.isEmpty { + VStack(spacing: 10) { + audioComposerPanel + + if !selectedImages.isEmpty { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 10) { + ForEach(Array(selectedImages.enumerated()), id: \.offset) { index, image in + ZStack(alignment: .topTrailing) { + Image(uiImage: image) + .resizable() + .scaledToFill() + .frame(width: 72, height: 72) + .clipShape(RoundedRectangle(cornerRadius: 14)) + .overlay( + RoundedRectangle(cornerRadius: 14) + .strokeBorder(Theme.border, lineWidth: 1) + ) + + Button { + selectedImages.remove(at: index) + } label: { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 18)) + .foregroundStyle(.white, Color.black.opacity(0.65)) + } + .offset(x: 6, y: -6) + } + } + } + .padding(.horizontal, Theme.inputPadH) + } + } + } + .padding(.bottom, engine.messages.isEmpty ? 8 : 0) + } + } + + @ViewBuilder + private var audioComposerPanel: some View { + if audioCapture.isCapturing { + RecordingStatusCard( + duration: audioCapture.duration, + sampleRate: max(audioCapture.sampleRate, 16_000), + peakLevel: audioCapture.peakLevel, + onStop: { + _ = audioCapture.stopCapture() + }, + onDiscard: { + _ = audioCapture.stopCapture() + _ = audioCapture.consumeLatestSnapshot() + } + ) + .padding(.horizontal, Theme.inputPadH) + } else if let draft = audioCapture.latestSnapshot(), + let attachment = ChatAudioAttachment(snapshot: draft) { + ComposerAudioDraftCard( + attachment: attachment, + onDiscard: { + _ = audioCapture.consumeLatestSnapshot() + } + ) + .padding(.horizontal, Theme.inputPadH) + } else if let error = audioCapture.lastErrorMessage { + AudioErrorBanner( + message: error, + onDismiss: { + audioCapture.clearStatus() + } + ) + .padding(.horizontal, Theme.inputPadH) + } + } + private var canSend: Bool { - (!inputText.trimmingCharacters(in: .whitespaces).isEmpty || !selectedImages.isEmpty) + ( + !inputText.trimmingCharacters(in: .whitespaces).isEmpty + || !selectedImages.isEmpty + || audioCapture.bufferedSampleCount > 0 + ) && !engine.isProcessing && engine.llm.isLoaded } private func send() async { let text = inputText let images = selectedImages + if audioCapture.isCapturing { + _ = audioCapture.stopCapture() + } + let audioSnapshot = audioCapture.consumeLatestSnapshot() inputText = "" selectedImages = [] selectedPhotoItem = nil isInputFocused = false - await engine.processInput(text, images: images) + await engine.processInput(text, images: images, audio: audioSnapshot) } @MainActor @@ -320,7 +491,7 @@ struct ContentView: View { do { if let data = try await selectedPhotoItem.loadTransferable(type: Data.self), let image = UIImage(data: data) { - selectedImages = [image] + selectedImages = [ChatImageAttachment.preparedImage(image)] } } catch { print("[UI] Failed to load selected photo: \(error)") @@ -334,10 +505,14 @@ struct ContentView: View { struct UserBubble: View { let text: String let images: [UIImage] + let audios: [ChatAudioAttachment] var body: some View { HStack { Spacer(minLength: Theme.bubbleMinSpacer) VStack(alignment: .trailing, spacing: 8) { + ForEach(audios) { audio in + AudioAttachmentBubble(attachment: audio) + } ForEach(Array(images.enumerated()), id: \.offset) { _, image in Image(uiImage: image) .resizable() @@ -345,17 +520,497 @@ struct UserBubble: View { .frame(width: 180, height: 180) .clipShape(RoundedRectangle(cornerRadius: 18)) } - Text(text) - .font(.system(size: 15)) - .foregroundStyle(Theme.userText) - .padding(.horizontal, 16) - .padding(.vertical, 10) - .background(Theme.userBubble, in: UserBubbleShape()) + if !text.isEmpty { + Text(text) + .font(.system(size: 15)) + .foregroundStyle(Theme.userText) + .padding(.horizontal, 16) + .padding(.vertical, 10) + .background(Theme.userBubble, in: UserBubbleShape()) + } } } } } +struct AudioAttachmentBubble: View { + let attachment: ChatAudioAttachment + @StateObject private var player = AudioAttachmentPlayer() + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(spacing: 8) { + Label("语音消息", systemImage: "waveform.badge.mic") + .font(.system(size: 12, weight: .semibold, design: .rounded)) + .foregroundStyle(Theme.textPrimary) + + Spacer(minLength: 8) + + Text(attachment.formattedDuration) + .font(.system(size: 11, weight: .semibold, design: .monospaced)) + .foregroundStyle(Theme.textSecondary) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(Theme.bg.opacity(0.35), in: Capsule()) + } + + HStack(spacing: 12) { + AudioPlaybackActionButton( + isPlaying: player.isPlaying, + action: { player.togglePlayback(data: attachment.wavData) } + ) + + VStack(alignment: .leading, spacing: 6) { + AudioWaveformView( + levels: attachment.waveform, + progress: player.progress, + isPlaying: player.isPlaying, + activeColor: Theme.accent, + inactiveColor: Theme.textTertiary.opacity(0.45), + barWidth: 4, + minHeight: 8, + maxExtraHeight: 18 + ) + .frame(height: 30) + + HStack(spacing: 8) { + Text(player.isPlaying ? "播放中" : "点击播放") + .font(.system(size: 11)) + .foregroundStyle(player.isPlaying ? Theme.accent : Theme.textSecondary) + + Spacer(minLength: 8) + + Text(player.secondaryStatusText(totalDuration: attachment.duration)) + .font(.system(size: 11, weight: .medium, design: .monospaced)) + .foregroundStyle(Theme.textTertiary) + } + } + } + } + .frame(minWidth: 230, maxWidth: 272, alignment: .leading) + .padding(.horizontal, 14) + .padding(.vertical, 12) + .background( + LinearGradient( + colors: [Theme.bgElevated, Theme.bgHover.opacity(0.94)], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + in: RoundedRectangle(cornerRadius: 20, style: .continuous) + ) + .overlay( + RoundedRectangle(cornerRadius: 20, style: .continuous) + .strokeBorder(Theme.borderSubtle, lineWidth: 1) + ) + .shadow(color: Color.black.opacity(0.12), radius: 10, y: 4) + } +} + +struct AudioWaveformView: View { + let levels: [Float] + var progress: Double = 0 + var isPlaying: Bool = false + var activeColor: Color = Theme.accent + var inactiveColor: Color = Theme.textTertiary + var barWidth: CGFloat = 3 + var minHeight: CGFloat = 6 + var maxExtraHeight: CGFloat = 18 + + var body: some View { + HStack(alignment: .center, spacing: 2) { + ForEach(Array(levels.enumerated()), id: \.offset) { index, level in + let threshold = Double(index + 1) / Double(max(levels.count, 1)) + let isActive = progress > 0 ? threshold <= progress : isPlaying + Capsule() + .fill(isActive ? activeColor : inactiveColor) + .frame( + width: barWidth, + height: minHeight + CGFloat(level) * maxExtraHeight + ) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .animation(.easeInOut(duration: 0.18), value: progress) + .animation(.easeInOut(duration: 0.18), value: isPlaying) + } +} + +@MainActor +final class AudioAttachmentPlayer: NSObject, ObservableObject, @preconcurrency AVAudioPlayerDelegate { + @Published private(set) var isPlaying = false + @Published private(set) var progress: Double = 0 + @Published private(set) var currentTime: TimeInterval = 0 + + private var player: AVAudioPlayer? + private var timer: Timer? + private var loadedClipSignature: Int? + private let audioSession = AVAudioSession.sharedInstance() + + func togglePlayback(data: Data) { + if isPlaying { + pause() + } else if loadedClipSignature == data.hashValue, player != nil { + resume() + } else { + play(data: data) + } + } + + func stop() { + invalidateTimer() + player?.stop() + player = nil + loadedClipSignature = nil + isPlaying = false + progress = 0 + currentTime = 0 + try? audioSession.setActive(false, options: [.notifyOthersOnDeactivation]) + } + + func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + invalidateTimer() + self.player = nil + loadedClipSignature = nil + isPlaying = false + progress = 0 + currentTime = 0 + try? audioSession.setActive(false, options: [.notifyOthersOnDeactivation]) + } + + func secondaryStatusText(totalDuration: TimeInterval) -> String { + guard totalDuration > 0 else { return "--:--" } + if isPlaying || currentTime > 0 { + return "\(formatTime(currentTime)) / \(formatTime(totalDuration))" + } + return formatTime(totalDuration) + } + + private func pause() { + player?.pause() + invalidateTimer() + isPlaying = false + syncProgress() + } + + private func resume() { + guard let player else { return } + guard player.play() else { return } + isPlaying = true + startProgressUpdates() + } + + private func invalidateTimer() { + timer?.invalidate() + timer = nil + } + + private func startProgressUpdates() { + invalidateTimer() + timer = Timer.scheduledTimer(withTimeInterval: 0.05, repeats: true) { [weak self] _ in + Task { @MainActor in + self?.syncProgress() + } + } + RunLoop.main.add(timer!, forMode: .common) + } + + private func syncProgress() { + guard let player else { + progress = 0 + currentTime = 0 + return + } + currentTime = player.currentTime + progress = player.duration > 0 ? player.currentTime / player.duration : 0 + } + + private func formatTime(_ time: TimeInterval) -> String { + let totalSeconds = max(Int(time.rounded()), 0) + return String(format: "%02d:%02d", totalSeconds / 60, totalSeconds % 60) + } + + private func play(data: Data) { + stop() + do { + try audioSession.setCategory(.playback, mode: .default, options: []) + try audioSession.setActive(true) + let player = try AVAudioPlayer(data: data) + player.delegate = self + player.prepareToPlay() + if !player.play() { + print("[AudioUI] playback failed: AVAudioPlayer returned false") + isPlaying = false + return + } + self.player = player + loadedClipSignature = data.hashValue + isPlaying = true + progress = 0 + currentTime = 0 + startProgressUpdates() + } catch { + print("[AudioUI] playback failed: \(error.localizedDescription)") + isPlaying = false + } + } +} + +struct AudioPlaybackActionButton: View { + let isPlaying: Bool + var symbolName: String? = nil + let action: () -> Void + + var body: some View { + Button(action: action) { + Image(systemName: symbolName ?? (isPlaying ? "pause.fill" : "play.fill")) + .font(.system(size: 15, weight: .bold)) + .foregroundStyle(Theme.bg) + .frame(width: 38, height: 38) + .background( + LinearGradient( + colors: [Theme.accent, Theme.accent.opacity(0.82)], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + in: Circle() + ) + .shadow(color: Theme.accent.opacity(0.22), radius: 10, y: 4) + } + .buttonStyle(.plain) + } +} + +struct RecordingStatusCard: View { + let duration: TimeInterval + let sampleRate: Double + let peakLevel: Float + let onStop: () -> Void + let onDiscard: () -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack(spacing: 10) { + HStack(spacing: 8) { + Circle() + .fill(Color.red.opacity(0.92)) + .frame(width: 8, height: 8) + Text("录音中") + .font(.system(size: 14, weight: .semibold, design: .rounded)) + .foregroundStyle(Theme.textPrimary) + } + + Spacer(minLength: 8) + + audioMetaChip(text: formattedDuration, emphasized: true) + audioMetaChip(text: sampleRateText) + } + + HStack(spacing: 12) { + AudioPlaybackActionButton( + isPlaying: false, + symbolName: "stop.fill", + action: onStop + ) + .overlay( + Circle() + .strokeBorder(Color.red.opacity(0.25), lineWidth: 8) + .scaleEffect(1.08) + ) + + VStack(alignment: .leading, spacing: 8) { + RecordingLevelBars(level: peakLevel) + .frame(height: 28) + + Text("点左侧按钮结束录音，发送时会作为音频附件一并发出。") + .font(.system(size: 11)) + .foregroundStyle(Theme.textSecondary) + .lineLimit(2) + } + + Button(action: onDiscard) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .bold)) + .foregroundStyle(Theme.textSecondary) + .frame(width: 30, height: 30) + .background(Theme.bg.opacity(0.42), in: Circle()) + } + .buttonStyle(.plain) + } + } + .padding(.horizontal, 14) + .padding(.vertical, 14) + .background( + LinearGradient( + colors: [Theme.bgElevated, Theme.bgHover.opacity(0.98)], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + in: RoundedRectangle(cornerRadius: 22, style: .continuous) + ) + .overlay( + RoundedRectangle(cornerRadius: 22, style: .continuous) + .strokeBorder(Theme.border, lineWidth: 1) + ) + } + + private var formattedDuration: String { + let totalSeconds = max(Int(duration.rounded()), 0) + return String(format: "%02d:%02d", totalSeconds / 60, totalSeconds % 60) + } + + private var sampleRateText: String { + String(format: "%.0f kHz", max(sampleRate, 16_000) / 1000) + } +} + +struct ComposerAudioDraftCard: View { + let attachment: ChatAudioAttachment + let onDiscard: () -> Void + + @StateObject private var player = AudioAttachmentPlayer() + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack(spacing: 8) { + Label("语音草稿", systemImage: "paperplane.circle.fill") + .font(.system(size: 14, weight: .semibold, design: .rounded)) + .foregroundStyle(Theme.textPrimary) + + Spacer(minLength: 8) + + audioMetaChip(text: attachment.formattedDuration, emphasized: true) + + Button(action: onDiscard) { + Image(systemName: "xmark") + .font(.system(size: 12, weight: .bold)) + .foregroundStyle(Theme.textSecondary) + .frame(width: 30, height: 30) + .background(Theme.bg.opacity(0.4), in: Circle()) + } + .buttonStyle(.plain) + } + + HStack(spacing: 12) { + AudioPlaybackActionButton( + isPlaying: player.isPlaying, + action: { player.togglePlayback(data: attachment.wavData) } + ) + + VStack(alignment: .leading, spacing: 8) { + AudioWaveformView( + levels: attachment.waveform, + progress: player.progress, + isPlaying: player.isPlaying, + activeColor: Theme.accent, + inactiveColor: Theme.textTertiary.opacity(0.45), + barWidth: 4, + minHeight: 8, + maxExtraHeight: 18 + ) + .frame(height: 30) + + HStack(spacing: 8) { + Text(player.isPlaying ? "预览播放中" : "可直接发送，也可以先试听") + .font(.system(size: 11)) + .foregroundStyle(player.isPlaying ? Theme.accent : Theme.textSecondary) + + Spacer(minLength: 8) + + audioMetaChip(text: String(format: "%.0f kHz", attachment.sampleRate / 1000)) + } + } + } + } + .padding(.horizontal, 14) + .padding(.vertical, 14) + .background( + LinearGradient( + colors: [Theme.accentSubtle, Theme.bgElevated], + startPoint: .topLeading, + endPoint: .bottomTrailing + ), + in: RoundedRectangle(cornerRadius: 22, style: .continuous) + ) + .overlay( + RoundedRectangle(cornerRadius: 22, style: .continuous) + .strokeBorder(Theme.accent.opacity(0.2), lineWidth: 1) + ) + } +} + +struct AudioErrorBanner: View { + let message: String + let onDismiss: () -> Void + + var body: some View { + HStack(spacing: 12) { + Image(systemName: "exclamationmark.triangle.fill") + .font(.system(size: 14)) + .foregroundStyle(Color.orange) + + Text(message) + .font(.system(size: 12)) + .foregroundStyle(Theme.textPrimary) + .frame(maxWidth: .infinity, alignment: .leading) + + Button(action: onDismiss) { + Image(systemName: "xmark") + .font(.system(size: 11, weight: .bold)) + .foregroundStyle(Theme.textSecondary) + .frame(width: 26, height: 26) + .background(Theme.bg.opacity(0.4), in: Circle()) + } + .buttonStyle(.plain) + } + .padding(.horizontal, 14) + .padding(.vertical, 12) + .background( + RoundedRectangle(cornerRadius: 18, style: .continuous) + .fill(Color.orange.opacity(0.12)) + ) + .overlay( + RoundedRectangle(cornerRadius: 18, style: .continuous) + .strokeBorder(Color.orange.opacity(0.25), lineWidth: 1) + ) + } +} + +struct RecordingLevelBars: View { + let level: Float + + var body: some View { + HStack(alignment: .center, spacing: 3) { + ForEach(0..<24, id: \.self) { index in + let seed = abs(sin(Double(index) * 0.55)) + let intensity = max(CGFloat(level), 0.08) + Capsule() + .fill(index < highlightedBarCount ? Theme.accent : Theme.textTertiary.opacity(0.35)) + .frame(width: 4, height: 8 + CGFloat(seed) * (8 + intensity * 18)) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .animation(.easeInOut(duration: 0.16), value: highlightedBarCount) + } + + private var highlightedBarCount: Int { + let normalized = min(max(level, 0), 1) + return max(2, Int((normalized * 24).rounded(.up))) + } +} + +@ViewBuilder +private func audioMetaChip(text: String, emphasized: Bool = false) -> some View { + Text(text) + .font(.system(size: 11, weight: emphasized ? .semibold : .medium, design: .monospaced)) + .foregroundStyle(emphasized ? Theme.textPrimary : Theme.textSecondary) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background( + emphasized ? Theme.bg.opacity(0.42) : Theme.bg.opacity(0.3), + in: Capsule() + ) +} + struct UserBubbleShape: Shape { func path(in rect: CGRect) -> Path { let r: CGFloat = 18, sr: CGFloat = 4 @@ -382,10 +1037,18 @@ struct UserBubbleShape: Shape { struct AIResponseView: View { let block: ResponseBlock let expandedSkills: Set + let isThinkingExpanded: Bool let onToggle: (UUID) -> Void + let onToggleThinking: () -> Void private var hasSkill: Bool { !block.skills.isEmpty } - private var isPureThinking: Bool { !hasSkill && block.responseText == nil && block.isThinking } + private var hasThinkingText: Bool { + guard let thinking = block.thinkingText else { return false } + return !thinking.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + private var isPureThinking: Bool { + !hasSkill && !hasThinkingText && block.responseText == nil && block.isThinking + } var body: some View { HStack { @@ -407,6 +1070,15 @@ struct AIResponseView: View { .transition(.opacity.combined(with: .scale(scale: 0.95))) } + if let thinking = block.thinkingText, !thinking.isEmpty { + ThinkingCardView( + text: thinking, + isExpanded: isThinkingExpanded, + onToggle: onToggleThinking + ) + .transition(.opacity.combined(with: .scale(scale: 0.98))) + } + // Skill 完成后等待 follow-up 文字 if hasSkill && block.isThinking && block.responseText == nil { ThinkingIndicator() @@ -433,6 +1105,81 @@ struct AIResponseView: View { } } +struct ThinkingCardView: View { + let text: String + let isExpanded: Bool + let onToggle: () -> Void + + private var lineCount: Int { + max(1, text.components(separatedBy: .newlines).count) + } + + private var previewText: String { + let compact = text + .replacingOccurrences(of: "\n", with: " ") + .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !compact.isEmpty else { return localizedThinkingText("已捕获思考内容", "Captured thinking content") } + return String(compact.prefix(72)) + (compact.count > 72 ? "…" : "") + } + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + HStack(spacing: 10) { + Image(systemName: "brain.head.profile") + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(Theme.accent) + .frame(width: 26, height: 26) + .background(Theme.accentSubtle, in: RoundedRectangle(cornerRadius: 7)) + + VStack(alignment: .leading, spacing: 2) { + Text(localizedThinkingText("思考", "Think")) + .font(.system(size: 13, weight: .semibold, design: .rounded)) + .foregroundStyle(Theme.textPrimary) + if !isExpanded { + Text(previewText) + .font(.system(size: 11)) + .foregroundStyle(Theme.textTertiary) + .lineLimit(1) + } + } + + Spacer() + + Text(localizedThinkingText("\(lineCount) 行", "\(lineCount) lines")) + .font(.system(size: 10, weight: .medium, design: .monospaced)) + .foregroundStyle(Theme.textTertiary) + + Image(systemName: "chevron.down") + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(Theme.textTertiary) + .rotationEffect(.degrees(isExpanded ? -180 : 0)) + .animation(.easeInOut(duration: 0.25), value: isExpanded) + } + .padding(.horizontal, 12) + .padding(.vertical, 10) + .contentShape(Rectangle()) + .onTapGesture { onToggle() } + + if isExpanded { + Rectangle().fill(Theme.borderSubtle).frame(height: 1) + + Text(text) + .font(.system(size: 13)) + .foregroundStyle(Theme.textSecondary) + .textSelection(.enabled) + .lineSpacing(4) + .frame(maxWidth: .infinity, alignment: .leading) + .fixedSize(horizontal: false, vertical: true) + .padding(.horizontal, 12) + .padding(.vertical, 10) + } + } + .background(Theme.bgElevated, in: RoundedRectangle(cornerRadius: 12)) + .overlay(RoundedRectangle(cornerRadius: 12).strokeBorder(Theme.border, lineWidth: 1)) + } +} + // MARK: - 单个 Skill 卡片（4 步进度） struct SkillCardView: View {