diff --git a/openless-all/app/src-tauri/src/asr/local/cache.rs b/openless-all/app/src-tauri/src/asr/local/cache.rs new file mode 100644 index 00000000..0402c420 --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/cache.rs @@ -0,0 +1,132 @@ +//! 本地 Qwen3-ASR 引擎缓存。 +//! +//! 用途:避免每次 dictation 都重加载 1.2GB+ 模型。引擎一次 load 后驻留在内存, +//! 跨多次会话复用;用户在设置里决定"说完话即释放" / "保持 N 秒后释放" / +//! "不释放"。 +//! +//! 调度规则:每次会话结束后 spawn 一个 sleep+check 任务;任务在到点时检查 +//! `last_used`——如果中间又被使用过则不释放,否则 drop 引擎让 OS 回收 RAM。 + +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use parking_lot::Mutex; + +#[cfg(target_os = "macos")] +use super::QwenAsrEngine; + +pub struct LocalAsrCache { + #[cfg(target_os = "macos")] + inner: Mutex>, + #[cfg(not(target_os = "macos"))] + _phantom: (), +} + +#[cfg(target_os = "macos")] +struct CachedEngine { + model_id: String, + engine: Arc, + last_used: Instant, +} + +impl Default for LocalAsrCache { + fn default() -> Self { + Self::new() + } +} + +impl LocalAsrCache { + pub fn new() -> Self { + Self { + #[cfg(target_os = "macos")] + inner: Mutex::new(None), + #[cfg(not(target_os = "macos"))] + _phantom: (), + } + } + + /// 取已缓存的同 id 引擎,没有就加载(**阻塞、可能数秒**——调用方应放 + /// `spawn_blocking`)。模型 id 不同则把旧的 drop 再加载新的。 + #[cfg(target_os = "macos")] + pub fn get_or_load(&self, model_id: &str, model_dir: &Path) -> Result> { + { + let mut slot = self.inner.lock(); + if let Some(cached) = slot.as_mut() { + if cached.model_id == model_id { + cached.last_used = Instant::now(); + log::info!("[local-asr cache] reuse engine: {model_id}"); + return Ok(Arc::clone(&cached.engine)); + } + log::info!( + "[local-asr cache] active model changed {} -> {}, drop old", + cached.model_id, + model_id + ); + slot.take(); + } + } + log::info!("[local-asr cache] loading {model_id} from {}", model_dir.display()); + let engine = Arc::new(QwenAsrEngine::load(model_dir)?); + let mut slot = self.inner.lock(); + *slot = Some(CachedEngine { + model_id: model_id.to_string(), + engine: Arc::clone(&engine), + last_used: Instant::now(), + }); + log::info!("[local-asr cache] loaded {model_id}"); + Ok(engine) + } + + /// 标记最近使用时间——end_session 在调过 transcribe 之后调一下, + /// 让 release 计时器从这一刻重新算。 + pub fn touch(&self) { + #[cfg(target_os = "macos")] + { + if let Some(cached) = self.inner.lock().as_mut() { + cached.last_used = Instant::now(); + } + } + } + + /// 如果空闲时长 ≥ threshold,释放引擎。返回是否真释放了。 + pub fn release_if_idle(&self, idle_threshold: Duration) -> bool { + #[cfg(target_os = "macos")] + { + let mut slot = self.inner.lock(); + if let Some(cached) = slot.as_ref() { + if cached.last_used.elapsed() >= idle_threshold { + log::info!( + "[local-asr cache] release engine {} after idle {:?}", + cached.model_id, + cached.last_used.elapsed() + ); + slot.take(); + return true; + } + } + } + let _ = idle_threshold; + false + } + + /// 立刻释放(用户点"立即释放"或删模型时调)。 + pub fn release_now(&self) { + #[cfg(target_os = "macos")] + { + if let Some(cached) = self.inner.lock().take() { + log::info!("[local-asr cache] release engine {} on demand", cached.model_id); + } + } + } + + pub fn loaded_model_id(&self) -> Option { + #[cfg(target_os = "macos")] + { + return self.inner.lock().as_ref().map(|c| c.model_id.clone()); + } + #[cfg(not(target_os = "macos"))] + None + } +} diff --git a/openless-all/app/src-tauri/src/asr/local/local_provider.rs b/openless-all/app/src-tauri/src/asr/local/local_provider.rs index 06e381c5..4fe0bd3f 100644 --- a/openless-all/app/src-tauri/src/asr/local/local_provider.rs +++ b/openless-all/app/src-tauri/src/asr/local/local_provider.rs @@ -3,9 +3,10 @@ //! 与 `WhisperBatchASR` 形状对齐:实现 `AudioConsumer` 缓冲 PCM,stop 时 //! 调 `transcribe_stream`,期间每个稳定 token 通过 Tauri 事件 //! `local-asr-token` 推到前端胶囊做实时显示。 +//! +//! engine 现在由 `LocalAsrCache` 提供——Coordinator 在 build_local_qwen3 里 +//! 取已缓存的引擎再传进来,避免每次会话都重加载 1.2GB+ 模型。 -#[cfg(target_os = "macos")] -use std::path::PathBuf; #[cfg(target_os = "macos")] use std::sync::Arc; @@ -32,14 +33,12 @@ pub struct LocalQwenAsr { #[cfg(target_os = "macos")] impl LocalQwenAsr { - pub fn new(app: AppHandle, model_dir: &PathBuf) -> Result { - let engine = QwenAsrEngine::load(model_dir) - .with_context(|| format!("加载本地模型失败:{}", model_dir.display()))?; - Ok(Self { - engine: Arc::new(engine), + pub fn new(app: AppHandle, engine: Arc) -> Self { + Self { + engine, buffer: Mutex::new(Vec::new()), app, - }) + } } /// stop 时调用:把 buffer 的 i16 PCM 转 f32,跑流式转写,token 实时 diff --git a/openless-all/app/src-tauri/src/asr/local/mod.rs b/openless-all/app/src-tauri/src/asr/local/mod.rs index f5d64db3..1aca579f 100644 --- a/openless-all/app/src-tauri/src/asr/local/mod.rs +++ b/openless-all/app/src-tauri/src/asr/local/mod.rs @@ -3,11 +3,14 @@ //! 当前只在 macOS 编入 antirez/qwen-asr (纯 C + Accelerate);Windows 端 //! 的本地推理路径见 issue #256,本期不实现。 +pub mod cache; pub mod download; mod local_provider; pub mod models; pub mod test_run; +pub use cache::LocalAsrCache; + #[cfg(target_os = "macos")] mod qwen_engine; #[cfg(target_os = "macos")] diff --git a/openless-all/app/src-tauri/src/commands.rs b/openless-all/app/src-tauri/src/commands.rs index 1807da14..014e675b 100644 --- a/openless-all/app/src-tauri/src/commands.rs +++ b/openless-all/app/src-tauri/src/commands.rs @@ -121,8 +121,13 @@ pub fn set_credential(account: String, value: String) -> Result<(), String> { } #[tauri::command] -pub fn set_active_asr_provider(provider: String) -> Result<(), String> { - CredentialsVault::set_active_asr_provider(&provider).map_err(|e| e.to_string()) +pub fn set_active_asr_provider(coord: CoordinatorState<'_>, provider: String) -> Result<(), String> { + CredentialsVault::set_active_asr_provider(&provider).map_err(|e| e.to_string())?; + // 切到本地 ASR → 后台预加载模型,下次按 hotkey 时不必等数秒。 + if provider == crate::asr::local::PROVIDER_ID { + coord.preload_local_asr_in_background(); + } + Ok(()) } #[tauri::command] @@ -790,6 +795,44 @@ pub async fn local_asr_test_model( .map_err(|e| format!("{e:#}")) } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct LocalAsrEngineStatus { + pub loaded: bool, + pub model_id: Option, + pub keep_loaded_secs: u32, +} + +#[tauri::command] +pub fn local_asr_engine_status(coord: CoordinatorState<'_>) -> LocalAsrEngineStatus { + let prefs = coord.prefs().get(); + LocalAsrEngineStatus { + loaded: coord.local_asr_loaded_model().is_some(), + model_id: coord.local_asr_loaded_model(), + keep_loaded_secs: prefs.local_asr_keep_loaded_secs, + } +} + +#[tauri::command] +pub fn local_asr_release_engine(coord: CoordinatorState<'_>) { + coord.release_local_asr_engine(); +} + +#[tauri::command] +pub fn local_asr_preload(coord: tauri::State<'_, std::sync::Arc>) { + coord.preload_local_asr_in_background(); +} + +#[tauri::command] +pub fn local_asr_set_keep_loaded_secs( + coord: CoordinatorState<'_>, + seconds: u32, +) -> Result<(), String> { + let mut prefs = coord.prefs().get(); + prefs.local_asr_keep_loaded_secs = seconds; + coord.prefs().set(prefs).map_err(|e| e.to_string()) +} + // ─────────────────────────── unused but exported (silences dead_code) ─────────────────────────── #[allow(dead_code)] diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 763d7ee1..9825d550 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -126,6 +126,9 @@ struct Inner { prepared_windows_ime_session: Arc>>, state: Mutex, asr: Mutex>>, + /// 本地 Qwen3-ASR 引擎缓存。跨会话复用,避免每次重加载 1.2GB+ 模型。 + /// 释放时机由 prefs.local_asr_keep_loaded_secs 决定。 + local_asr_cache: Arc, recorder: Mutex>>, hotkey: Mutex>, hotkey_status: Mutex, @@ -232,10 +235,57 @@ impl Coordinator { qa_asr: Mutex::new(None), qa_recorder: Mutex::new(None), qa_stream_cancelled: Arc::new(AtomicBool::new(false)), + local_asr_cache: Arc::new(crate::asr::local::LocalAsrCache::new()), }), } } + /// 后台预加载本地 ASR 引擎;当用户在 UI 切到 local-qwen3 provider 时调一次。 + /// 加载是阻塞且数秒,所以放 spawn_blocking 里,不影响 UI 响应。 + /// 模型未下载或不在 macOS 上时静默跳过。 + pub fn preload_local_asr_in_background(self: &Arc) { + #[cfg(target_os = "macos")] + { + let inner = Arc::clone(&self.inner); + tauri::async_runtime::spawn(async move { + let prefs = inner.prefs.get(); + let model_id = match crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model) { + Some(m) => m, + None => return, + }; + if !crate::asr::local::models::is_downloaded(model_id) { + log::info!("[coord] local ASR preload skipped: model {} not downloaded", model_id.as_str()); + return; + } + let dir = match crate::asr::local::models::model_dir(model_id) { + Ok(d) => d, + Err(_) => return, + }; + let cache = Arc::clone(&inner.local_asr_cache); + let mid = model_id.as_str().to_string(); + let _ = tauri::async_runtime::spawn_blocking(move || { + if let Err(e) = cache.get_or_load(&mid, &dir) { + log::warn!("[coord] local ASR preload failed: {e:#}"); + } + }) + .await; + }); + } + #[cfg(not(target_os = "macos"))] + { + // no-op + } + } + + /// 释放当前缓存的本地 ASR 引擎(用户主动点 / 或 删除模型时调)。 + pub fn release_local_asr_engine(&self) { + self.inner.local_asr_cache.release_now(); + } + + pub fn local_asr_loaded_model(&self) -> Option { + self.inner.local_asr_cache.loaded_model_id() + } + pub fn bind_app(&self, handle: AppHandle) { *self.inner.app.lock() = Some(handle); } @@ -1015,7 +1065,7 @@ async fn begin_session(inner: &Arc) -> Result<(), String> { #[cfg(target_os = "macos")] if crate::asr::local::is_local_qwen3(&active_asr) { - let local = match build_local_qwen3(inner) { + let local = match build_local_qwen3(inner).await { Ok(l) => l, Err(e) => { log::error!("[coord] 本地 Qwen3-ASR 初始化失败: {e:#}"); @@ -1437,24 +1487,30 @@ async fn end_session(inner: &Arc) -> Result<(), String> { } }, #[cfg(target_os = "macos")] - ActiveAsr::Local(local) => match local.transcribe().await { - Ok(r) => r, - Err(e) => { - log::error!("[coord] local Qwen3-ASR transcribe failed: {e:#}"); - emit_capsule( - inner, - CapsuleState::Error, - 0.0, - elapsed, - Some(format!("本地识别失败: {e}")), - None, - ); - restore_prepared_windows_ime_session(inner, current_session_id); - inner.state.lock().phase = SessionPhase::Idle; - schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); - return Err(e.to_string()); + ActiveAsr::Local(local) => { + let result = local.transcribe().await; + // 无论成功失败都触一次缓存的 last_used + 调度释放 + inner.local_asr_cache.touch(); + schedule_local_asr_release(inner); + match result { + Ok(r) => r, + Err(e) => { + log::error!("[coord] local Qwen3-ASR transcribe failed: {e:#}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some(format!("本地识别失败: {e}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(e.to_string()); + } } - }, + } }; // ASR 完成后 cancel 检查:用户在 transcribe 进行中按 Esc 时,这里就会命中。 @@ -1983,8 +2039,25 @@ fn ensure_local_qwen3_model_ready() -> Result<(), String> { Ok(()) } +/// 一次 dictation 结束后,按 prefs.local_asr_keep_loaded_secs 决定何时释放 +/// 内存里的 Qwen3-ASR 引擎。0 = 立即释放;其它值 = sleep N 秒后看 last_used。 +/// 多次会话叠加多个 sleep 任务,每个独立 check:只要中间又被使用过就跳过释放。 +fn schedule_local_asr_release(inner: &Arc) { + let keep_secs = inner.prefs.get().local_asr_keep_loaded_secs; + let cache = Arc::clone(&inner.local_asr_cache); + if keep_secs == 0 { + cache.release_now(); + return; + } + let dur = std::time::Duration::from_secs(keep_secs as u64); + tauri::async_runtime::spawn(async move { + tokio::time::sleep(dur).await; + cache.release_if_idle(dur); + }); +} + #[cfg(target_os = "macos")] -fn build_local_qwen3(inner: &Arc) -> anyhow::Result> { +async fn build_local_qwen3(inner: &Arc) -> anyhow::Result> { let prefs = inner.prefs.get(); let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model) .ok_or_else(|| anyhow::anyhow!("未知本地模型 id: {}", prefs.local_asr_active_model))?; @@ -1994,7 +2067,14 @@ fn build_local_qwen3(inner: &Arc) -> anyhow::Result String { @@ -140,6 +145,10 @@ fn default_local_asr_mirror() -> String { "huggingface".into() } +fn default_local_asr_keep_loaded_secs() -> u32 { + 300 +} + fn default_qa_hotkey() -> Option { Some(QaHotkeyBinding::default()) } @@ -171,6 +180,7 @@ impl Default for UserPreferences { qa_save_history: false, local_asr_active_model: default_local_asr_model(), local_asr_mirror: default_local_asr_mirror(), + local_asr_keep_loaded_secs: default_local_asr_keep_loaded_secs(), } } } diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 1cadd387..2cf85d37 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -550,5 +550,17 @@ export const en: typeof zhCN = { testActual: 'Got', testStats: 'Audio {{audio}}s · Load {{load}}s · Transcribe {{transcribe}}s · Backend {{backend}}', testFailed: 'Test failed', + engineStatusLabel: 'Engine in memory', + engineLoaded: 'Loaded: {{model}} (~1.2-3.4 GB RAM)', + engineUnloaded: 'Not loaded (first transcription will load it, ~3-5 s)', + loadNow: 'Load now', + releaseNow: 'Release now', + keepLoadedLabel: 'Keep loaded for', + keepLoadedDesc: 'How long the engine stays in memory after the last use, before being freed.', + keepImmediate: 'Release immediately', + keep1min: '1 minute after last use', + keep5min: '5 minutes after last use (default)', + keep30min: '30 minutes after last use', + keepForever: 'Never release (always loaded)', }, }; diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index ba13a1fc..6ca83fa6 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -548,5 +548,17 @@ export const zhCN = { testActual: '识别', testStats: '音频时长 {{audio}}s · 加载 {{load}}s · 推理 {{transcribe}}s · 后端 {{backend}}', testFailed: '测试失败', + engineStatusLabel: '内存中的引擎', + engineLoaded: '已加载:{{model}}(约占 1.2-3.4 GB 内存)', + engineUnloaded: '未加载(首次听写需先加载,约 3-5 秒)', + loadNow: '立即加载', + releaseNow: '立即释放', + keepLoadedLabel: '保持加载多久', + keepLoadedDesc: '决定本地 ASR 用完后多久从内存释放,避免长期占用 1+ GB RAM。', + keepImmediate: '说完话立即释放', + keep1min: '上次使用后 1 分钟', + keep5min: '上次使用后 5 分钟(默认)', + keep30min: '上次使用后 30 分钟', + keepForever: '不释放(始终保留)', }, }; diff --git a/openless-all/app/src/lib/ipc.ts b/openless-all/app/src/lib/ipc.ts index 5a36db26..0d924c4f 100644 --- a/openless-all/app/src/lib/ipc.ts +++ b/openless-all/app/src/lib/ipc.ts @@ -55,6 +55,7 @@ const mockSettings: UserPreferences = { qaSaveHistory: false, localAsrActiveModel: 'qwen3-asr-0.6b', localAsrMirror: 'huggingface', + localAsrKeepLoadedSecs: 300, }; const mockHotkeyCapability: HotkeyCapability = { diff --git a/openless-all/app/src/lib/localAsr.ts b/openless-all/app/src/lib/localAsr.ts index 9157e9a3..ceacc19c 100644 --- a/openless-all/app/src/lib/localAsr.ts +++ b/openless-all/app/src/lib/localAsr.ts @@ -149,3 +149,29 @@ export function testLocalAsrModel(modelId: string): Promise }), ); } + +export interface LocalAsrEngineStatus { + loaded: boolean; + modelId: string | null; + keepLoadedSecs: number; +} + +export function getLocalAsrEngineStatus(): Promise { + return invokeOrMock('local_asr_engine_status', undefined, () => ({ + loaded: false, + modelId: null, + keepLoadedSecs: 300, + })); +} + +export function releaseLocalAsrEngine(): Promise { + return invokeOrMock('local_asr_release_engine', undefined, () => undefined); +} + +export function preloadLocalAsr(): Promise { + return invokeOrMock('local_asr_preload', undefined, () => undefined); +} + +export function setLocalAsrKeepLoadedSecs(seconds: number): Promise { + return invokeOrMock('local_asr_set_keep_loaded_secs', { seconds }, () => undefined); +} diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index 2959789d..69826f45 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -128,6 +128,9 @@ export interface UserPreferences { localAsrActiveModel: string; /** 本地模型下载源镜像('huggingface' / 'hf-mirror')。 */ localAsrMirror: string; + /** 本地 ASR 引擎在内存中的保留时长(秒)。0 = 说完话即释放; + * 300 = 默认 5 分钟;86400 ≈ 不释放(保持加载)。 */ + localAsrKeepLoadedSecs: number; } /** Rust 通过 `qa:state` 事件下发的 payload。 diff --git a/openless-all/app/src/pages/LocalAsr.tsx b/openless-all/app/src/pages/LocalAsr.tsx index 3ffd1bcf..9b166a40 100644 --- a/openless-all/app/src/pages/LocalAsr.tsx +++ b/openless-all/app/src/pages/LocalAsr.tsx @@ -15,12 +15,17 @@ import { deleteLocalAsrModel, downloadLocalAsrModel, fetchLocalAsrRemoteInfo, + getLocalAsrEngineStatus, getLocalAsrSettings, listLocalAsrModels, + preloadLocalAsr, + releaseLocalAsrEngine, setLocalAsrActiveModel, + setLocalAsrKeepLoadedSecs, setLocalAsrMirror, testLocalAsrModel, type LocalAsrDownloadProgress, + type LocalAsrEngineStatus, type LocalAsrModelStatus, type LocalAsrSettings, type LocalAsrTestResult, @@ -44,7 +49,18 @@ export function LocalAsr() { const [busyModelId, setBusyModelId] = useState(null); const [testingModelId, setTestingModelId] = useState(null); const [testResults, setTestResults] = useState>({}); + const [engineStatus, setEngineStatus] = useState(null); const refreshTimer = useRef(null); + const engineStatusTimer = useRef(null); + + const refreshEngineStatus = async () => { + try { + const status = await getLocalAsrEngineStatus(); + setEngineStatus(status); + } catch (err) { + console.warn('[localAsr] engine status query failed', err); + } + }; const refresh = async () => { try { @@ -52,6 +68,7 @@ export function LocalAsr() { const [s, list] = await Promise.all([getLocalAsrSettings(), listLocalAsrModels()]); setSettings(s); setModels(list); + void refreshEngineStatus(); // 拉远端真实尺寸(每个模型一次,结果留缓存) void Promise.all( list.map(async m => { @@ -94,7 +111,15 @@ export function LocalAsr() { useEffect(() => { void refresh(); - // refresh 内部已 fan-out 拉远端尺寸,不需要额外 effect + // 引擎状态每 5s 轮询一次,让 UI 能看到 release 计时器到点后的状态变化 + engineStatusTimer.current = window.setInterval(() => { + void refreshEngineStatus(); + }, 5000); + return () => { + if (engineStatusTimer.current !== null) { + window.clearInterval(engineStatusTimer.current); + } + }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); @@ -194,6 +219,34 @@ export function LocalAsr() { } }; + const handleKeepLoadedChange = async (seconds: number) => { + try { + await setLocalAsrKeepLoadedSecs(seconds); + await refresh(); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }; + + const handleReleaseEngine = async () => { + try { + await releaseLocalAsrEngine(); + await refreshEngineStatus(); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }; + + const handlePreload = async () => { + try { + await preloadLocalAsr(); + // 触发预加载后给后端几秒,再查状态 + window.setTimeout(() => void refreshEngineStatus(), 1500); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }; + const handleTest = async (modelId: string) => { setTestingModelId(modelId); setTestResults(prev => { @@ -274,6 +327,65 @@ export function LocalAsr() { + {/* 运行时设置卡:内存中的引擎状态 + 多久释放 + 立即释放 */} + {engineAvailable && ( + +
+
+
+
+ {t('localAsr.engineStatusLabel')} +
+
+ {engineStatus?.loaded + ? t('localAsr.engineLoaded', { model: engineStatus.modelId ?? '' }) + : t('localAsr.engineUnloaded')} +
+
+
+ {engineStatus?.loaded ? ( + void handleReleaseEngine()}> + {t('localAsr.releaseNow')} + + ) : ( + void handlePreload()}> + {t('localAsr.loadNow')} + + )} +
+
+
+
+
+ {t('localAsr.keepLoadedLabel')} +
+
+ {t('localAsr.keepLoadedDesc')} +
+
+ +
+
+
+ )} + {error && (
{error}