From 050927e7193d38018b730d3a48ef4e0c7f3648f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=95=E6=9F=8F=E9=9D=92?= Date: Wed, 10 Jun 2026 15:00:14 +0800 Subject: [PATCH] =?UTF-8?q?feat(asr):=20Apple=20Speech=20=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=20ASR=20=E6=8E=A5=E7=BA=BF=20=E2=80=94=20dictation/QA?= =?UTF-8?q?/cancel=20+=20=E5=B9=B3=E5=8F=B0=E9=97=A8=E6=8E=A7=20(issue=20#?= =?UTF-8?q?574)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 承接草稿 PR #625 的 provider 基础，补完后端接线： - coordinator: ActiveAsr::AppleSpeech 变体（macOS only） - asr_setup: build_apple_speech + ensure_asr_credentials 平台门控 + build_qa_asr_start QA 接线 + is_keyless_local_asr_provider - dictation_session/dictation_end: 听写 start + 批处理 transcribe（沿用 local_qwen 动态超时） - qa_session: QA transcribe arm - resources: cancel arm - commands: active_apple_speech_asr_is_supported（非 macOS 返回 false）+ keyless 校验白名单 + asr_configured_for_provider + set_active_asr_provider 非 macOS 拒绝门控 provider 仅 macOS 编译；非 macOS 由门控判不可用。前端 UI/i18n 已随 #625 落地。 验证：cargo check（macOS）通过、provider 单测 5/5 通过、npm run build 通过。 Co-Authored-By: Claude Fable 5 --- .../app/src-tauri/src/commands/credentials.rs | 6 ++ .../app/src-tauri/src/commands/providers.rs | 13 ++++ openless-all/app/src-tauri/src/coordinator.rs | 4 ++ .../src-tauri/src/coordinator/asr_setup.rs | 32 ++++++++++ .../src/coordinator/dictation_end.rs | 59 +++++++++++++++++++ .../src/coordinator/dictation_session.rs | 16 +++++ .../src-tauri/src/coordinator/qa_session.rs | 34 +++++++++++ .../src-tauri/src/coordinator/resources.rs | 2 + 8 files changed, 166 insertions(+) diff --git a/openless-all/app/src-tauri/src/commands/credentials.rs b/openless-all/app/src-tauri/src/commands/credentials.rs index 350d1e3e..1778199e 100644 --- a/openless-all/app/src-tauri/src/commands/credentials.rs +++ b/openless-all/app/src-tauri/src/commands/credentials.rs @@ -29,6 +29,7 @@ pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnap return volcengine_configured(snap); } if provider == crate::asr::local::PROVIDER_ID + || active_apple_speech_asr_is_supported(provider) || active_foundry_asr_is_supported(provider) || active_sherpa_asr_is_supported(provider) { @@ -168,6 +169,11 @@ pub async fn set_active_asr_provider( { return Err("sherpa-onnx local ASR is only available on Windows".to_string()); } + if provider == crate::asr::local::APPLE_SPEECH_PROVIDER_ID + && !active_apple_speech_asr_is_supported(&provider) + { + return Err("Apple Speech recognition is only available on macOS".to_string()); + } if CredentialsVault::get_active_asr() == provider { return Ok(()); } diff --git a/openless-all/app/src-tauri/src/commands/providers.rs b/openless-all/app/src-tauri/src/commands/providers.rs index 9fed8852..2f302e8e 100644 --- a/openless-all/app/src-tauri/src/commands/providers.rs +++ b/openless-all/app/src-tauri/src/commands/providers.rs @@ -246,10 +246,23 @@ async fn validate_bailian_asr_provider() -> Result<(), String> { pub(crate) fn active_asr_is_keyless_for_validation(provider: &str) -> bool { provider == crate::asr::local::PROVIDER_ID + || active_apple_speech_asr_is_supported(provider) || active_foundry_asr_is_supported(provider) || active_sherpa_asr_is_supported(provider) } +pub(crate) fn active_apple_speech_asr_is_supported(provider: &str) -> bool { + #[cfg(target_os = "macos")] + { + provider == crate::asr::local::APPLE_SPEECH_PROVIDER_ID + } + #[cfg(not(target_os = "macos"))] + { + let _ = provider; + false + } +} + pub(crate) fn active_foundry_asr_is_supported(provider: &str) -> bool { #[cfg(target_os = "windows")] { diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 1579dc3f..926a260b 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -119,6 +119,10 @@ pub(crate) enum ActiveAsr { /// 本地 Qwen3-ASR；只在 macOS + 模型已下载时可达。 #[cfg(target_os = "macos")] Local(Arc), + /// Apple Speech（SFSpeechRecognizer）系统本地 ASR；只在 macOS 可达。 + /// 无模型下载、无凭据，首次使用弹系统授权（issue #574）。 + #[cfg(target_os = "macos")] + AppleSpeech(Arc), } fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool { diff --git a/openless-all/app/src-tauri/src/coordinator/asr_setup.rs b/openless-all/app/src-tauri/src/coordinator/asr_setup.rs index f1b47e1b..326a5036 100644 --- a/openless-all/app/src-tauri/src/coordinator/asr_setup.rs +++ b/openless-all/app/src-tauri/src/coordinator/asr_setup.rs @@ -75,6 +75,19 @@ pub(crate) fn ensure_asr_credentials() -> Result<(), String> { } } + // Apple Speech 没有"凭据"也没有要下载的模型，只需：macOS 平台。 + // 系统语音识别资源由 OS 管理，首次使用时弹授权框（见 apple_speech_provider）。 + if crate::asr::local::is_apple_speech(&active_asr) { + #[cfg(not(target_os = "macos"))] + { + return Err("Apple Speech 语音识别仅支持 macOS".to_string()); + } + #[cfg(target_os = "macos")] + { + return Ok(()); + } + } + if crate::asr::local::foundry::is_foundry_local_whisper(&active_asr) { #[cfg(not(target_os = "windows"))] { @@ -124,6 +137,10 @@ pub(crate) fn is_keyless_local_asr_provider(id: &str) -> bool { if crate::asr::local::is_local_qwen3(id) { return true; } + #[cfg(target_os = "macos")] + if crate::asr::local::is_apple_speech(id) { + return true; + } #[cfg(target_os = "windows")] { crate::asr::local::foundry::is_foundry_local_whisper(id) @@ -261,6 +278,13 @@ pub(crate) async fn build_local_qwen3( Ok(Arc::new(crate::asr::local::LocalQwenAsr::new(app, engine))) } +/// 构建 Apple Speech provider。与 build_local_qwen3 不同：无模型、无缓存、无 +/// AppHandle 依赖，授权/识别由 provider 内部按需处理（首次弹系统授权框）。 +#[cfg(target_os = "macos")] +pub(crate) fn build_apple_speech() -> Arc { + Arc::new(crate::asr::local::AppleSpeechAsr::new()) +} + pub(crate) enum QaAsrStart { Volcengine { asr: Arc, @@ -390,6 +414,14 @@ pub(crate) async fn build_qa_asr_start( return Ok(QaAsrStart::Ready { active, consumer }); } + #[cfg(target_os = "macos")] + if crate::asr::local::is_apple_speech(active_asr) { + let local = build_apple_speech(); + let active = ActiveAsr::AppleSpeech(Arc::clone(&local)); + let consumer: Arc = local; + return Ok(QaAsrStart::Ready { active, consumer }); + } + match active_asr_provider_kind(active_asr) { ActiveAsrProviderKind::Bailian => Ok(QaAsrStart::Bailian { asr: Arc::new(BailianRealtimeASR::new(read_bailian_credentials())), diff --git a/openless-all/app/src-tauri/src/coordinator/dictation_end.rs b/openless-all/app/src-tauri/src/coordinator/dictation_end.rs index 09439d9b..25978fdb 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation_end.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation_end.rs @@ -359,6 +359,65 @@ pub(crate) async fn end_session(inner: &Arc) -> Result<(), String> { } } } + // Apple Speech：系统语音识别，无模型加载耗时。批处理 transcribe 受音频 + // 长度影响，沿用 local_qwen_transcribe_timeout 的动态超时公式（基础 15s + // 兜短录音，长录音按音频 0.6 倍 + 10s 余量），coordinator 侧再加一层防线。 + #[cfg(target_os = "macos")] + ActiveAsr::AppleSpeech(local) => { + debug_assert!(uses_global_timeout); + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = local_qwen_transcribe_timeout(audio_secs); + log::info!( + "[coord] Apple Speech transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + match tokio::time::timeout(timeout_duration, local.transcribe()).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + if inner.state.lock().cancelled { + log::info!( + "[coord] Apple Speech transcribe cancelled — discarding transcript" + ); + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + return Ok(()); + } + log::error!("[coord] Apple Speech transcribe failed: {e:#}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some(format!("本地识别失败: {e}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(e.to_string()); + } + Err(_) => { + log::error!( + "[coord] Apple Speech 动态超时 {}s（音频 {:.2}s）", + timeout_duration.as_secs(), + audio_secs + ); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some("识别超时".to_string()), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err("apple-speech global timeout".to_string()); + } + } + } }; // ASR 完成后 cancel 检查：用户在 transcribe 进行中按 Esc 时，这里就会命中。 diff --git a/openless-all/app/src-tauri/src/coordinator/dictation_session.rs b/openless-all/app/src-tauri/src/coordinator/dictation_session.rs index 480883af..3ab2589c 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation_session.rs @@ -209,6 +209,22 @@ pub(crate) async fn begin_session(inner: &Arc) -> Result<(), String> { return Ok(()); } + // Apple Speech：无模型加载，构建即用；停止录音后整段批处理识别，再复用 + // 现有 polish / insert / history 收尾路径（与 local-qwen3 同形）。 + #[cfg(target_os = "macos")] + if crate::asr::local::is_apple_speech(&active_asr) { + let local = build_apple_speech(); + store_asr_for_session( + inner, + current_session_id, + ActiveAsr::AppleSpeech(Arc::clone(&local)), + ); + let consumer: Arc = local; + start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer) + .await?; + return Ok(()); + } + if is_bailian_provider(&active_asr) { let asr = Arc::new(BailianRealtimeASR::new(read_bailian_credentials())); let bridge = Arc::new(DeferredAsrBridge::new()); diff --git a/openless-all/app/src-tauri/src/coordinator/qa_session.rs b/openless-all/app/src-tauri/src/coordinator/qa_session.rs index bd439ede..7afa3db9 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa_session.rs @@ -455,6 +455,40 @@ pub(crate) async fn end_qa_session(inner: &Arc) -> Result<(), String> { } } } + #[cfg(target_os = "macos")] + ActiveAsr::AppleSpeech(local) => { + debug_assert!(uses_global_timeout); + let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0; + let timeout_duration = local_qwen_transcribe_timeout(audio_secs); + log::info!( + "[coord] QA Apple Speech transcribe: audio={:.2}s timeout={}s", + audio_secs, + timeout_duration.as_secs() + ); + match tokio::time::timeout(timeout_duration, local.transcribe()).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + if inner.qa_state.lock().cancelled { + log::info!( + "[coord] QA Apple Speech transcribe cancelled — discarding transcript" + ); + finish_qa_idle_silently(inner); + return Ok(()); + } + log::error!("[coord] QA Apple Speech transcribe failed: {e:#}"); + finish_qa_with_error(inner, format!("本地识别失败: {e}")); + return Err(e.to_string()); + } + Err(_) => { + log::error!( + "[coord] QA Apple Speech transcribe timeout after {}s", + timeout_duration.as_secs() + ); + finish_qa_with_error(inner, "本地识别超时".to_string()); + return Err("apple-speech transcribe timeout".to_string()); + } + } + } }; // cancel race：用户在 transcribe 中按 Esc / dismiss → 静默退出。 diff --git a/openless-all/app/src-tauri/src/coordinator/resources.rs b/openless-all/app/src-tauri/src/coordinator/resources.rs index 72b91a99..71183687 100644 --- a/openless-all/app/src-tauri/src/coordinator/resources.rs +++ b/openless-all/app/src-tauri/src/coordinator/resources.rs @@ -75,6 +75,8 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) { ActiveAsr::SherpaOnnxLocal(local) => local.cancel(), #[cfg(target_os = "macos")] ActiveAsr::Local(local) => local.cancel(), + #[cfg(target_os = "macos")] + ActiveAsr::AppleSpeech(local) => local.cancel(), } }