diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..4bceb1ad --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "openless-all/app/src-tauri/vendor/qwen-asr"] + path = openless-all/app/src-tauri/vendor/qwen-asr + url = https://github.com/antirez/qwen-asr.git diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index 884e24d1..f4bd975e 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -3287,6 +3287,7 @@ dependencies = [ "arboard", "block2 0.5.1", "bytes", + "cc", "chrono", "core-foundation 0.10.1", "core-graphics 0.24.0", @@ -3295,6 +3296,7 @@ dependencies = [ "env_logger", "futures-util", "global-hotkey", + "libc", "log", "objc2 0.5.2", "objc2-app-kit 0.2.2", diff --git a/openless-all/app/src-tauri/Cargo.toml b/openless-all/app/src-tauri/Cargo.toml index e7a8609c..23835d95 100644 --- a/openless-all/app/src-tauri/Cargo.toml +++ b/openless-all/app/src-tauri/Cargo.toml @@ -12,6 +12,8 @@ crate-type = ["staticlib", "cdylib", "rlib"] [build-dependencies] tauri-build = { version = "2", features = [] } +# Compile antirez/qwen-asr C sources for the local ASR engine on macOS. +cc = "1.1" [dependencies] tauri = { version = "2", features = ["macos-private-api", "tray-icon"] } @@ -52,6 +54,8 @@ core-graphics = "0.24" objc2 = "0.5" objc2-foundation = "0.2" objc2-app-kit = "0.2" +# antirez/qwen-asr 用 malloc 分配返回字符串,必须用 C `free()` 释放。 +libc = "0.2" [target.'cfg(target_os = "windows")'.dependencies] raw-window-handle = "0.6" diff --git a/openless-all/app/src-tauri/build.rs b/openless-all/app/src-tauri/build.rs index 261851f6..23219128 100644 --- a/openless-all/app/src-tauri/build.rs +++ b/openless-all/app/src-tauri/build.rs @@ -1,3 +1,56 @@ fn main() { + #[cfg(target_os = "macos")] + build_qwen_asr_macos(); + tauri_build::build(); } + +/// 编译 antirez/qwen-asr 的 C 源(仅 macOS)。 +/// +/// 上游 Makefile `make blas` 等价配置:BLAS 加速通过 Accelerate framework, +/// `USE_BLAS` + `ACCELERATE_NEW_LAPACK` 是必要宏。 +/// `-march=native` 这里**不**用——分发二进制要可移植,cc crate 在 release 下 +/// 默认带 `-O2`,加上 `-O3` 提一档;NEON/AVX 在源码里有 `#ifdef` 自动分派。 +#[cfg(target_os = "macos")] +fn build_qwen_asr_macos() { + const VENDOR: &str = "vendor/qwen-asr"; + const SOURCES: &[&str] = &[ + "qwen_asr.c", + "qwen_asr_kernels.c", + "qwen_asr_kernels_generic.c", + "qwen_asr_kernels_neon.c", + "qwen_asr_kernels_avx.c", + "qwen_asr_audio.c", + "qwen_asr_encoder.c", + "qwen_asr_decoder.c", + "qwen_asr_tokenizer.c", + "qwen_asr_safetensors.c", + ]; + + let mut build = cc::Build::new(); + build + .include(VENDOR) + .define("USE_BLAS", None) + .define("ACCELERATE_NEW_LAPACK", None) + .flag("-O3") + .flag("-ffast-math") + // 上游开 `-Wall -Wextra`;我们把 antirez 的代码当三方依赖,把无关警告压成静默 + // 避免 build log 噪音淹没我们自己的告警。 + .flag("-Wno-unused-parameter") + .flag("-Wno-unused-variable") + .flag("-Wno-unused-function") + .flag("-Wno-sign-compare") + .warnings(false); + + for src in SOURCES { + let path = format!("{}/{}", VENDOR, src); + println!("cargo:rerun-if-changed={}", path); + build.file(path); + } + println!("cargo:rerun-if-changed={}/qwen_asr.h", VENDOR); + + build.compile("qwen_asr"); + + // BLAS = Accelerate + println!("cargo:rustc-link-lib=framework=Accelerate"); +} diff --git a/openless-all/app/src-tauri/src/asr/local/mod.rs b/openless-all/app/src-tauri/src/asr/local/mod.rs new file mode 100644 index 00000000..588ef42b --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/mod.rs @@ -0,0 +1,12 @@ +//! 本地 ASR 引擎入口。 +//! +//! 当前只在 macOS 编入 antirez/qwen-asr (纯 C + Accelerate);Windows 端 +//! 的本地推理路径见 issue #256,本期不实现。 + +#[cfg(target_os = "macos")] +mod qwen_engine; +#[cfg(target_os = "macos")] +mod qwen_ffi; + +#[cfg(target_os = "macos")] +pub use qwen_engine::QwenAsrEngine; diff --git a/openless-all/app/src-tauri/src/asr/local/qwen_engine.rs b/openless-all/app/src-tauri/src/asr/local/qwen_engine.rs new file mode 100644 index 00000000..0b298fd8 --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/qwen_engine.rs @@ -0,0 +1,135 @@ +//! antirez/qwen-asr 的安全 Rust 包装。 +//! +//! 当前只暴露**最小可用面**:`load` / `transcribe_audio` / `transcribe_stream` +//! + token 回调。后续接 coordinator 时再扩 prompt/language 设置。 + +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char, c_void}; +use std::path::Path; +use std::ptr; +use std::sync::Mutex; + +use anyhow::{Context, Result}; + +use super::qwen_ffi::{ + qwen_free, qwen_load, qwen_set_token_callback, qwen_transcribe_audio, qwen_transcribe_stream, + QwenCtx, +}; + +/// FnMut 闭包是 fat pointer,不能直接塞进 `*mut c_void`,所以包一层 Box。 +type TokenHandler = dyn FnMut(&str) + Send + 'static; +type TokenHandlerBox = Box>; + +pub struct QwenAsrEngine { + ctx: *mut QwenCtx, + /// 持有 token 回调的所有权;C 端拿到的是 `&**handler` 派生出来的 raw ptr, + /// 只要这个 Box 还活着,那个 raw ptr 就有效。Mutex 防止并发 set。 + token_handler: Mutex>, +} + +/// SAFETY: `qwen_ctx_t` 内部的 pthread/buffer 仅在单次 transcribe 期间被 C 端 +/// 自己用;外层不会从两个 Rust 线程并发调进同一个 ctx(由 coordinator 串行 +/// 化保证)。Send/Sync 在这一约束下成立。 +unsafe impl Send for QwenAsrEngine {} +unsafe impl Sync for QwenAsrEngine {} + +impl QwenAsrEngine { + /// 从模型目录加载(目录里需含 `config.json` / `model.safetensors*` / + /// `vocab.json` / `merges.txt`,结构见 antirez `download_model.sh`)。 + pub fn load(model_dir: &Path) -> Result { + let dir_str = model_dir + .to_str() + .with_context(|| format!("model dir 不是合法 UTF-8: {model_dir:?}"))?; + let c_dir = CString::new(dir_str).context("model dir 含 NUL 字节")?; + + // SAFETY: `c_dir` 在调用期间存活;返回 NULL 表示加载失败。 + let ctx = unsafe { qwen_load(c_dir.as_ptr()) }; + if ctx.is_null() { + anyhow::bail!("qwen_load 失败:{model_dir:?}"); + } + + Ok(Self { + ctx, + token_handler: Mutex::new(None), + }) + } + + /// 注册流式 token 回调;传 `None` 清空。重新注册会先解绑再装新回调。 + pub fn set_token_handler(&self, handler: Option) + where + F: FnMut(&str) + Send + 'static, + { + let mut slot = self.token_handler.lock().expect("token_handler poisoned"); + + // 先把 C 端那一侧切干净,再 drop 旧 Box,避免 C 在替换瞬间还持有旧指针。 + unsafe { qwen_set_token_callback(self.ctx, None, ptr::null_mut()) }; + *slot = None; + + if let Some(f) = handler { + let boxed: TokenHandlerBox = Box::new(Box::new(f)); + // boxed 的内部 `Box` 在堆上有稳定地址;取它的 &mut 转 raw。 + let userdata = boxed.as_ref() as *const Box as *mut c_void; + unsafe { + qwen_set_token_callback(self.ctx, Some(token_trampoline), userdata); + } + *slot = Some(boxed); + } + } + + /// 批式转写:一次性给完整音频(mono f32 16kHz)。 + pub fn transcribe_audio(&self, samples: &[f32]) -> Result { + // SAFETY: samples 在调用期间存活;返回是 C `malloc` 出的字符串。 + let raw = unsafe { + qwen_transcribe_audio(self.ctx, samples.as_ptr(), samples.len() as i32) + }; + if raw.is_null() { + anyhow::bail!("qwen_transcribe_audio 返回 NULL"); + } + let text = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { libc::free(raw as *mut c_void) }; + Ok(text) + } + + /// 流式转写:内部按 2s chunk 切片,token 通过 `set_token_handler` 注册的 + /// 回调实时吐出;返回值是最终完整文本。 + pub fn transcribe_stream(&self, samples: &[f32]) -> Result { + let raw = unsafe { + qwen_transcribe_stream(self.ctx, samples.as_ptr(), samples.len() as i32) + }; + if raw.is_null() { + anyhow::bail!("qwen_transcribe_stream 返回 NULL"); + } + let text = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { libc::free(raw as *mut c_void) }; + Ok(text) + } +} + +impl Drop for QwenAsrEngine { + fn drop(&mut self) { + if !self.ctx.is_null() { + // 先解绑回调,避免 C 端在 free 后还持有 userdata 指针。 + unsafe { + qwen_set_token_callback(self.ctx, None, ptr::null_mut()); + qwen_free(self.ctx); + } + self.ctx = ptr::null_mut(); + } + // token_handler 的 Box 由 Mutex 析构时释放。 + } +} + +/// C 蹦床:把 `userdata` 解回 `&mut Box` 并转发字符串。 +unsafe extern "C" fn token_trampoline(piece: *const c_char, userdata: *mut c_void) { + if userdata.is_null() || piece.is_null() { + return; + } + // SAFETY: userdata 是 set_token_handler 注册的 `*Box`。 + let handler: &mut Box = unsafe { &mut *(userdata as *mut Box) }; + let text = unsafe { CStr::from_ptr(piece) }.to_string_lossy(); + handler(&text); +} diff --git a/openless-all/app/src-tauri/src/asr/local/qwen_ffi.rs b/openless-all/app/src-tauri/src/asr/local/qwen_ffi.rs new file mode 100644 index 00000000..81a9ec75 --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/local/qwen_ffi.rs @@ -0,0 +1,40 @@ +//! 对 antirez/qwen-asr 公共 C API 的最小 FFI 声明。 +//! +//! 头文件见 `vendor/qwen-asr/qwen_asr.h`。这里**不**复刻 `qwen_ctx_t` +//! 内部布局——保持不透明指针即可,避免 pthread/对齐相关的脆弱假设。 + +use std::os::raw::{c_char, c_int, c_void}; + +/// 不透明的 qwen_ctx_t;只通过指针来回传。 +#[repr(C)] +pub struct QwenCtx { + _opaque: [u8; 0], +} + +/// `typedef void (*qwen_token_cb)(const char *piece, void *userdata);` +pub type QwenTokenCb = unsafe extern "C" fn(piece: *const c_char, userdata: *mut c_void); + +unsafe extern "C" { + pub fn qwen_load(model_dir: *const c_char) -> *mut QwenCtx; + pub fn qwen_free(ctx: *mut QwenCtx); + + pub fn qwen_set_token_callback( + ctx: *mut QwenCtx, + cb: Option, + userdata: *mut c_void, + ); + pub fn qwen_set_prompt(ctx: *mut QwenCtx, prompt: *const c_char) -> c_int; + pub fn qwen_set_force_language(ctx: *mut QwenCtx, language: *const c_char) -> c_int; + pub fn qwen_supported_languages_csv() -> *const c_char; + + pub fn qwen_transcribe_audio( + ctx: *mut QwenCtx, + samples: *const f32, + n_samples: c_int, + ) -> *mut c_char; + pub fn qwen_transcribe_stream( + ctx: *mut QwenCtx, + samples: *const f32, + n_samples: c_int, + ) -> *mut c_char; +} diff --git a/openless-all/app/src-tauri/src/asr/mod.rs b/openless-all/app/src-tauri/src/asr/mod.rs index 419de904..d7347091 100644 --- a/openless-all/app/src-tauri/src/asr/mod.rs +++ b/openless-all/app/src-tauri/src/asr/mod.rs @@ -6,6 +6,7 @@ //! `volcengine.rs`. mod frame; +pub mod local; pub mod volcengine; pub mod whisper; diff --git a/openless-all/app/src-tauri/vendor/qwen-asr b/openless-all/app/src-tauri/vendor/qwen-asr new file mode 160000 index 00000000..b00b789b --- /dev/null +++ b/openless-all/app/src-tauri/vendor/qwen-asr @@ -0,0 +1 @@ +Subproject commit b00b789b17051aea61e9717458171100662318a4