diff --git a/openless-all/app/src-tauri/src/asr/local/local_provider.rs b/openless-all/app/src-tauri/src/asr/local/local_provider.rs new file mode 100644 index 00000000..a295e93b --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/local_provider.rs @@ -0,0 +1,101 @@ +//! 本地 Qwen3-ASR 在 dictation 路径上的适配器。 +//! +//! 与 `WhisperBatchASR` 形状对齐：实现 `AudioConsumer` 缓冲 PCM，stop 时 +//! 调 `transcribe_stream`，期间每个稳定 token 通过 Tauri 事件 +//! `local-asr-token` 推到前端胶囊做实时显示。 + +#[cfg(target_os = "macos")] +use std::path::PathBuf; +#[cfg(target_os = "macos")] +use std::sync::Arc; + +#[cfg(target_os = "macos")] +use anyhow::{Context, Result}; +#[cfg(target_os = "macos")] +use parking_lot::Mutex; +#[cfg(target_os = "macos")] +use tauri::{AppHandle, Emitter}; + +#[cfg(target_os = "macos")] +use super::QwenAsrEngine; +#[cfg(target_os = "macos")] +use crate::asr::RawTranscript; + +#[cfg(target_os = "macos")] +pub struct LocalQwenAsr { + engine: Arc, + /// 16-bit LE PCM 字节缓冲（recorder 推什么我们存什么），在 transcribe 时再 + /// 转 f32 喂给 C 端。一次会话最多几 MB，clone 一次成本可接受。 + buffer: Mutex>, + app: AppHandle, +} + +#[cfg(target_os = "macos")] +impl LocalQwenAsr { + pub fn new(app: AppHandle, model_dir: &PathBuf) -> Result { + let engine = QwenAsrEngine::load(model_dir) + .with_context(|| format!("加载本地模型失败：{}", model_dir.display()))?; + Ok(Self { + engine: Arc::new(engine), + buffer: Mutex::new(Vec::new()), + app, + }) + } + + /// stop 时调用：把 buffer 的 i16 PCM 转 f32，跑流式转写，token 实时 + /// 通过事件吐到前端胶囊；最终文本一起返回供 polish/insert。 + pub async fn transcribe(self: Arc) -> Result { + let pcm_bytes = std::mem::take(&mut *self.buffer.lock()); + if pcm_bytes.is_empty() { + return Ok(RawTranscript { + text: String::new(), + duration_ms: 0, + }); + } + let duration_ms = (pcm_bytes.len() as u64 / 2) * 1000 / 16_000; + let samples_f32 = i16_le_bytes_to_f32(&pcm_bytes); + + // 注册 token 回调：每个稳定 token 抛 `local-asr-token` 事件。 + // capsule 前端按 sessionId 累积显示。 + let app = self.app.clone(); + self.engine.set_token_handler(Some(move |piece: &str| { + if let Err(e) = app.emit("local-asr-token", piece.to_string()) { + log::warn!("[local-asr] emit token failed: {e}"); + } + })); + + // qwen_transcribe_stream 是阻塞调用；用 spawn_blocking 防止占住 tokio runtime。 + let engine = Arc::clone(&self.engine); + let text = tokio::task::spawn_blocking(move || engine.transcribe_stream(&samples_f32)) + .await + .context("transcribe spawn_blocking join 失败")? + .context("qwen_transcribe_stream 失败")?; + + // 解绑回调，避免 idle 期 C 端任何后续触发。 + self.engine.set_token_handler::(None); + + Ok(RawTranscript { text, duration_ms }) + } + + pub fn cancel(&self) { + self.buffer.lock().clear(); + } +} + +#[cfg(target_os = "macos")] +impl crate::recorder::AudioConsumer for LocalQwenAsr { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + self.buffer.lock().extend_from_slice(pcm); + } +} + +#[cfg(target_os = "macos")] +fn i16_le_bytes_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(2) + .map(|c| { + let v = i16::from_le_bytes([c[0], c[1]]); + v as f32 / 32768.0 + }) + .collect() +} diff --git a/openless-all/app/src-tauri/src/asr/local/mod.rs b/openless-all/app/src-tauri/src/asr/local/mod.rs index 42738540..f5283d47 100644 --- a/openless-all/app/src-tauri/src/asr/local/mod.rs +++ b/openless-all/app/src-tauri/src/asr/local/mod.rs @@ -4,6 +4,7 @@ //! 的本地推理路径见 issue #256，本期不实现。 pub mod download; +mod local_provider; pub mod models; #[cfg(target_os = "macos")] @@ -13,6 +14,8 @@ mod qwen_ffi; #[cfg(target_os = "macos")] pub use qwen_engine::QwenAsrEngine; +#[cfg(target_os = "macos")] +pub use local_provider::LocalQwenAsr; pub use download::{DownloadManager, Mirror}; pub use models::{ModelId, ModelStatus}; diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index a2867b61..763d7ee1 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -54,6 +54,9 @@ enum SessionPhase { enum ActiveAsr { Volcengine(Arc), Whisper(Arc), + /// 本地 Qwen3-ASR；只在 macOS + 模型已下载时可达。 + #[cfg(target_os = "macos")] + Local(Arc), } struct SessionResource { @@ -798,6 +801,8 @@ fn cancel_active_asr(asr: ActiveAsr) { match asr { ActiveAsr::Volcengine(v) => v.cancel(), ActiveAsr::Whisper(w) => w.cancel(), + #[cfg(target_os = "macos")] + ActiveAsr::Local(local) => local.cancel(), } } @@ -1008,6 +1013,37 @@ async fn begin_session(inner: &Arc) -> Result<(), String> { let active_asr = CredentialsVault::get_active_asr(); + #[cfg(target_os = "macos")] + if crate::asr::local::is_local_qwen3(&active_asr) { + let local = match build_local_qwen3(inner) { + Ok(l) => l, + Err(e) => { + log::error!("[coord] 本地 Qwen3-ASR 初始化失败: {e:#}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + 0, + Some(format!("本地模型初始化失败: {e}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(format!("local ASR init failed: {e}")); + } + }; + store_asr_for_session( + inner, + current_session_id, + ActiveAsr::Local(Arc::clone(&local)), + ); + let consumer: Arc = local; + start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer) + .await?; + return Ok(()); + } + if is_whisper_compatible_provider(&active_asr) { let (api_key, base_url, model) = read_whisper_credentials(); let whisper = Arc::new(WhisperBatchASR::new(api_key, base_url, model)); @@ -1400,6 +1436,25 @@ async fn end_session(inner: &Arc) -> Result<(), String> { return Err(e.to_string()); } }, + #[cfg(target_os = "macos")] + ActiveAsr::Local(local) => match local.transcribe().await { + Ok(r) => r, + Err(e) => { + log::error!("[coord] local Qwen3-ASR transcribe failed: {e:#}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some(format!("本地识别失败: {e}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(e.to_string()); + } + }, }; // ASR 完成后 cancel 检查：用户在 transcribe 进行中按 Esc 时，这里就会命中。 @@ -1877,6 +1932,19 @@ fn ensure_microphone_permission(_inner: &Arc) -> Result<(), String> { fn ensure_asr_credentials() -> Result<(), String> { let active_asr = CredentialsVault::get_active_asr(); + + // 本地 Qwen3-ASR 没有"凭据"概念，但需要：(a) macOS 平台 (b) 模型已下载。 + if crate::asr::local::is_local_qwen3(&active_asr) { + #[cfg(not(target_os = "macos"))] + { + return Err("本地 ASR 当前仅支持 macOS（Windows 见 issue #256）".to_string()); + } + #[cfg(target_os = "macos")] + { + return ensure_local_qwen3_model_ready(); + } + } + if is_whisper_compatible_provider(&active_asr) { let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) .ok() @@ -1896,6 +1964,39 @@ fn ensure_asr_credentials() -> Result<(), String> { } } +#[cfg(target_os = "macos")] +fn ensure_local_qwen3_model_ready() -> Result<(), String> { + let prefs = || -> Result { + // 这里没法拿到 inner，直接读 preferences.json 即可（Coordinator 写盘后总是同步的）。 + crate::persistence::PreferencesStore::new() + .map_err(|e| e.to_string()) + .map(|s| s.get()) + }()?; + let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model) + .ok_or_else(|| format!("未知的本地模型 id: {}", prefs.local_asr_active_model))?; + if !crate::asr::local::models::is_downloaded(model_id) { + return Err(format!( + "本地模型 {} 未下载完整，请到 设置 → 模型设置 中下载", + model_id.as_str() + )); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn build_local_qwen3(inner: &Arc) -> anyhow::Result> { + let prefs = inner.prefs.get(); + let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model) + .ok_or_else(|| anyhow::anyhow!("未知本地模型 id: {}", prefs.local_asr_active_model))?; + let dir = crate::asr::local::models::model_dir(model_id)?; + let app = inner + .app + .lock() + .clone() + .ok_or_else(|| anyhow::anyhow!("AppHandle 未绑定"))?; + Ok(Arc::new(crate::asr::local::LocalQwenAsr::new(app, &dir)?)) +} + /// `whisper` 是 OpenAI 原生；`siliconflow` / `zhipu` / `groq` 都暴露 /// OpenAI 兼容的 `/audio/transcriptions`，统一走 `WhisperBatchASR`。 /// 新增 OpenAI 兼容 ASR 时只需在这里加一项。