feat(local-asr): coordinator 接 ActiveAsr::Local 第三分支 — P4
- local_provider.rs (LocalQwenAsr):实现 AudioConsumer 缓 i16 PCM → stop 时转 f32 → 跑 qwen_transcribe_stream,token 经 `local-asr-token` 事件实时推到前端胶囊;spawn_blocking 防止占住 tokio runtime - coordinator.rs: * ActiveAsr::Local(Arc<LocalQwenAsr>) cfg(macos) * start_dictation_for_session 优先走 local 分支再回落 whisper/volc * end_session 加 Local arm,失败静默回退(CLAUDE.md 不变量) * cancel_active_asr / ensure_asr_credentials 都补 Local 分支 * build_local_qwen3 helper:从 prefs 拿 model id → models::model_dir → 加载 * ensure_local_qwen3_model_ready:模型未下载时友好提示用户去模型设置页 - 非 macOS 选择 local-qwen3 时 ensure_asr_credentials 直接返回错误
This commit is contained in:
1 parent
51b638a52e
commit
4656ecc7c7
3 files changed
+205
No files matched your search
@@ -0,0 +1,101 @@
|
||||
//! 本地 Qwen3-ASR 在 dictation 路径上的适配器。
|
||||
//!
|
||||
//! 与 `WhisperBatchASR` 形状对齐:实现 `AudioConsumer` 缓冲 PCM,stop 时
|
||||
//! 调 `transcribe_stream`,期间每个稳定 token 通过 Tauri 事件
|
||||
//! `local-asr-token` 推到前端胶囊做实时显示。
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::path::PathBuf;
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use anyhow::{Context, Result};
|
||||
#[cfg(target_os = "macos")]
|
||||
use parking_lot::Mutex;
|
||||
#[cfg(target_os = "macos")]
|
||||
use tauri::{AppHandle, Emitter};
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use super::QwenAsrEngine;
|
||||
#[cfg(target_os = "macos")]
|
||||
use crate::asr::RawTranscript;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub struct LocalQwenAsr {
|
||||
engine: Arc<QwenAsrEngine>,
|
||||
/// 16-bit LE PCM 字节缓冲(recorder 推什么我们存什么),在 transcribe 时再
|
||||
/// 转 f32 喂给 C 端。一次会话最多几 MB,clone 一次成本可接受。
|
||||
buffer: Mutex<Vec<u8>>,
|
||||
app: AppHandle,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl LocalQwenAsr {
|
||||
pub fn new(app: AppHandle, model_dir: &PathBuf) -> Result<Self> {
|
||||
let engine = QwenAsrEngine::load(model_dir)
|
||||
.with_context(|| format!("加载本地模型失败:{}", model_dir.display()))?;
|
||||
Ok(Self {
|
||||
engine: Arc::new(engine),
|
||||
buffer: Mutex::new(Vec::new()),
|
||||
app,
|
||||
})
|
||||
}
|
||||
|
||||
/// stop 时调用:把 buffer 的 i16 PCM 转 f32,跑流式转写,token 实时
|
||||
/// 通过事件吐到前端胶囊;最终文本一起返回供 polish/insert。
|
||||
pub async fn transcribe(self: Arc<Self>) -> Result<RawTranscript> {
|
||||
let pcm_bytes = std::mem::take(&mut *self.buffer.lock());
|
||||
if pcm_bytes.is_empty() {
|
||||
return Ok(RawTranscript {
|
||||
text: String::new(),
|
||||
duration_ms: 0,
|
||||
});
|
||||
}
|
||||
let duration_ms = (pcm_bytes.len() as u64 / 2) * 1000 / 16_000;
|
||||
let samples_f32 = i16_le_bytes_to_f32(&pcm_bytes);
|
||||
|
||||
// 注册 token 回调:每个稳定 token 抛 `local-asr-token` 事件。
|
||||
// capsule 前端按 sessionId 累积显示。
|
||||
let app = self.app.clone();
|
||||
self.engine.set_token_handler(Some(move |piece: &str| {
|
||||
if let Err(e) = app.emit("local-asr-token", piece.to_string()) {
|
||||
log::warn!("[local-asr] emit token failed: {e}");
|
||||
}
|
||||
}));
|
||||
|
||||
// qwen_transcribe_stream 是阻塞调用;用 spawn_blocking 防止占住 tokio runtime。
|
||||
let engine = Arc::clone(&self.engine);
|
||||
let text = tokio::task::spawn_blocking(move || engine.transcribe_stream(&samples_f32))
|
||||
.await
|
||||
.context("transcribe spawn_blocking join 失败")?
|
||||
.context("qwen_transcribe_stream 失败")?;
|
||||
|
||||
// 解绑回调,避免 idle 期 C 端任何后续触发。
|
||||
self.engine.set_token_handler::<fn(&str)>(None);
|
||||
|
||||
Ok(RawTranscript { text, duration_ms })
|
||||
}
|
||||
|
||||
pub fn cancel(&self) {
|
||||
self.buffer.lock().clear();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
impl crate::recorder::AudioConsumer for LocalQwenAsr {
|
||||
fn consume_pcm_chunk(&self, pcm: &[u8]) {
|
||||
self.buffer.lock().extend_from_slice(pcm);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn i16_le_bytes_to_f32(bytes: &[u8]) -> Vec<f32> {
|
||||
bytes
|
||||
.chunks_exact(2)
|
||||
.map(|c| {
|
||||
let v = i16::from_le_bytes([c[0], c[1]]);
|
||||
v as f32 / 32768.0
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
//! 的本地推理路径见 issue #256,本期不实现。
|
||||
|
||||
pub mod download;
|
||||
mod local_provider;
|
||||
pub mod models;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
@@ -13,6 +14,8 @@ mod qwen_ffi;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub use qwen_engine::QwenAsrEngine;
|
||||
#[cfg(target_os = "macos")]
|
||||
pub use local_provider::LocalQwenAsr;
|
||||
|
||||
pub use download::{DownloadManager, Mirror};
|
||||
pub use models::{ModelId, ModelStatus};
|
||||
|
||||
@@ -54,6 +54,9 @@ enum SessionPhase {
|
||||
enum ActiveAsr {
|
||||
Volcengine(Arc<VolcengineStreamingASR>),
|
||||
Whisper(Arc<WhisperBatchASR>),
|
||||
/// 本地 Qwen3-ASR;只在 macOS + 模型已下载时可达。
|
||||
#[cfg(target_os = "macos")]
|
||||
Local(Arc<crate::asr::local::LocalQwenAsr>),
|
||||
}
|
||||
|
||||
struct SessionResource<T> {
|
||||
@@ -798,6 +801,8 @@ fn cancel_active_asr(asr: ActiveAsr) {
|
||||
match asr {
|
||||
ActiveAsr::Volcengine(v) => v.cancel(),
|
||||
ActiveAsr::Whisper(w) => w.cancel(),
|
||||
#[cfg(target_os = "macos")]
|
||||
ActiveAsr::Local(local) => local.cancel(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1008,6 +1013,37 @@ async fn begin_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
|
||||
let active_asr = CredentialsVault::get_active_asr();
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
if crate::asr::local::is_local_qwen3(&active_asr) {
|
||||
let local = match build_local_qwen3(inner) {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
log::error!("[coord] 本地 Qwen3-ASR 初始化失败: {e:#}");
|
||||
emit_capsule(
|
||||
inner,
|
||||
CapsuleState::Error,
|
||||
0.0,
|
||||
0,
|
||||
Some(format!("本地模型初始化失败: {e}")),
|
||||
None,
|
||||
);
|
||||
restore_prepared_windows_ime_session(inner, current_session_id);
|
||||
inner.state.lock().phase = SessionPhase::Idle;
|
||||
schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS);
|
||||
return Err(format!("local ASR init failed: {e}"));
|
||||
}
|
||||
};
|
||||
store_asr_for_session(
|
||||
inner,
|
||||
current_session_id,
|
||||
ActiveAsr::Local(Arc::clone(&local)),
|
||||
);
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = local;
|
||||
start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if is_whisper_compatible_provider(&active_asr) {
|
||||
let (api_key, base_url, model) = read_whisper_credentials();
|
||||
let whisper = Arc::new(WhisperBatchASR::new(api_key, base_url, model));
|
||||
@@ -1400,6 +1436,25 @@ async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
return Err(e.to_string());
|
||||
}
|
||||
},
|
||||
#[cfg(target_os = "macos")]
|
||||
ActiveAsr::Local(local) => match local.transcribe().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
log::error!("[coord] local Qwen3-ASR transcribe failed: {e:#}");
|
||||
emit_capsule(
|
||||
inner,
|
||||
CapsuleState::Error,
|
||||
0.0,
|
||||
elapsed,
|
||||
Some(format!("本地识别失败: {e}")),
|
||||
None,
|
||||
);
|
||||
restore_prepared_windows_ime_session(inner, current_session_id);
|
||||
inner.state.lock().phase = SessionPhase::Idle;
|
||||
schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS);
|
||||
return Err(e.to_string());
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// ASR 完成后 cancel 检查:用户在 transcribe 进行中按 Esc 时,这里就会命中。
|
||||
@@ -1877,6 +1932,19 @@ fn ensure_microphone_permission(_inner: &Arc<Inner>) -> Result<(), String> {
|
||||
|
||||
fn ensure_asr_credentials() -> Result<(), String> {
|
||||
let active_asr = CredentialsVault::get_active_asr();
|
||||
|
||||
// 本地 Qwen3-ASR 没有"凭据"概念,但需要:(a) macOS 平台 (b) 模型已下载。
|
||||
if crate::asr::local::is_local_qwen3(&active_asr) {
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
return Err("本地 ASR 当前仅支持 macOS(Windows 见 issue #256)".to_string());
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
return ensure_local_qwen3_model_ready();
|
||||
}
|
||||
}
|
||||
|
||||
if is_whisper_compatible_provider(&active_asr) {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
@@ -1896,6 +1964,39 @@ fn ensure_asr_credentials() -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn ensure_local_qwen3_model_ready() -> Result<(), String> {
|
||||
let prefs = || -> Result<crate::types::UserPreferences, String> {
|
||||
// 这里没法拿到 inner,直接读 preferences.json 即可(Coordinator 写盘后总是同步的)。
|
||||
crate::persistence::PreferencesStore::new()
|
||||
.map_err(|e| e.to_string())
|
||||
.map(|s| s.get())
|
||||
}()?;
|
||||
let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model)
|
||||
.ok_or_else(|| format!("未知的本地模型 id: {}", prefs.local_asr_active_model))?;
|
||||
if !crate::asr::local::models::is_downloaded(model_id) {
|
||||
return Err(format!(
|
||||
"本地模型 {} 未下载完整,请到 设置 → 模型设置 中下载",
|
||||
model_id.as_str()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn build_local_qwen3(inner: &Arc<Inner>) -> anyhow::Result<Arc<crate::asr::local::LocalQwenAsr>> {
|
||||
let prefs = inner.prefs.get();
|
||||
let model_id = crate::asr::local::ModelId::from_str(&prefs.local_asr_active_model)
|
||||
.ok_or_else(|| anyhow::anyhow!("未知本地模型 id: {}", prefs.local_asr_active_model))?;
|
||||
let dir = crate::asr::local::models::model_dir(model_id)?;
|
||||
let app = inner
|
||||
.app
|
||||
.lock()
|
||||
.clone()
|
||||
.ok_or_else(|| anyhow::anyhow!("AppHandle 未绑定"))?;
|
||||
Ok(Arc::new(crate::asr::local::LocalQwenAsr::new(app, &dir)?))
|
||||
}
|
||||
|
||||
/// `whisper` 是 OpenAI 原生;`siliconflow` / `zhipu` / `groq` 都暴露
|
||||
/// OpenAI 兼容的 `/audio/transcriptions`,统一走 `WhisperBatchASR`。
|
||||
/// 新增 OpenAI 兼容 ASR 时只需在这里加一项。
|
||||
|
||||
Reference in new issue
Block a user