fix(asr): 修复 Windows 本地长语音只识别前段 (#769)

* Fix long Windows local ASR transcription

* fix(asr): enforce total chunk timeout budget

* fix(asr): align Foundry native runtime versions

---------

Co-authored-by: Chris233 <h-chris233@outlook.com>
This commit is contained in:
Duyi-WangandChris233 authored and GitHub committed 2026-07-12 15:29:47 +08:00
1 parent f2c23723ee
commit ef1561ebf7
8 files changed
+351 -77

No files matched your search

+1 -1
View File
@@ -120,7 +120,7 @@ libc = "0.2"
tauri-nspanel = { git = "https://github.com/ahkohd/tauri-nspanel", branch = "v2" }
[target.'cfg(target_os = "windows")'.dependencies]
foundry-local-sdk = { version = "1.1.0", features = ["winml"] }
foundry-local-sdk = { version = "=1.2.1", features = ["winml"] }
raw-window-handle = "0.6"
sherpa-onnx = { version = "1.13.2", default-features = false, features = ["static"] }
windows = { version = "0.58", features = [
@@ -17,6 +17,7 @@ mod imp {
const TARGET_RID_NATIVE_PREFIX: &str = "runtimes/win-x64/native/";
const CORE_DLL: &str = "Microsoft.AI.Foundry.Local.Core.dll";
const REQUIRED_DLLS: &[&str] = &[CORE_DLL, "onnxruntime.dll", "onnxruntime-genai.dll"];
const RUNTIME_VERSIONS_FILE: &str = "runtime-versions.txt";
const WINDOWS_APP_RUNTIME_INSTALLER_URL: &str =
"https://aka.ms/windowsappsdk/1.8/1.8.260416003/windowsappruntimeinstall-x64.exe";
const WINDOWS_APP_RUNTIME_INSTALLER_FILE: &str = "WindowsAppRuntimeInstall-x64.exe";
@@ -64,20 +65,22 @@ mod imp {
expected_file: &'static str,
}
// Keep these versions aligned with foundry-local-sdk's deps_versions_winml.json.
// The native core and ORT use a versioned C API and will crash on a mixed runtime.
const PACKAGES: &[NativePackage] = &[
NativePackage {
name: "Microsoft.AI.Foundry.Local.Core.WinML",
version: "1.0.0",
version: "1.2.1",
expected_file: CORE_DLL,
},
NativePackage {
name: "Microsoft.ML.OnnxRuntime.Foundry",
version: "1.23.2.3",
version: "1.26.0",
expected_file: "onnxruntime.dll",
},
NativePackage {
name: "Microsoft.ML.OnnxRuntimeGenAI.Foundry",
version: "0.13.2",
version: "0.14.1",
expected_file: "onnxruntime-genai.dll",
},
];
@@ -172,21 +175,78 @@ mod imp {
progress(&label, end_percent);
}
fs::write(
staging_dir.join(RUNTIME_VERSIONS_FILE),
expected_runtime_versions(),
)
.context("write Foundry native runtime version manifest")?;
if !required_libraries_present(&staging_dir) {
anyhow::bail!("Foundry 运行组件下载不完整");
}
if target_dir.exists() {
fs::remove_dir_all(&target_dir)
.with_context(|| format!("remove {}", target_dir.display()))?;
}
fs::rename(&staging_dir, &target_dir).with_context(|| {
format!("move {} to {}", staging_dir.display(), target_dir.display())
})?;
replace_runtime_dir(&staging_dir, &target_dir)?;
progress("Foundry Local 运行组件已下载", 100.0);
Ok(target_dir)
}
fn replace_runtime_dir(staging_dir: &Path, target_dir: &Path) -> Result<()> {
let parent = target_dir
.parent()
.context("resolve Foundry native runtime parent")?;
let backup_dir = parent.join(".runtime-backup");
if backup_dir.exists() {
if required_libraries_present(target_dir) {
fs::remove_dir_all(&backup_dir)
.with_context(|| format!("remove stale {}", backup_dir.display()))?;
} else {
if target_dir.exists() {
fs::remove_dir_all(target_dir)
.with_context(|| format!("remove incomplete {}", target_dir.display()))?;
}
fs::rename(&backup_dir, target_dir)
.context("recover interrupted Foundry runtime replacement")?;
}
}
if target_dir.exists() {
fs::rename(target_dir, &backup_dir).with_context(|| {
format!(
"back up Foundry runtime {} to {}",
target_dir.display(),
backup_dir.display()
)
})?;
}
if let Err(replace_error) = fs::rename(staging_dir, target_dir) {
if backup_dir.exists() {
fs::rename(&backup_dir, target_dir).with_context(|| {
format!(
"restore Foundry runtime {} after replacement failed: {replace_error}",
target_dir.display()
)
})?;
}
return Err(replace_error).with_context(|| {
format!(
"move {} to {}",
staging_dir.display(),
target_dir.display()
)
});
}
if backup_dir.exists() {
if let Err(error) = fs::remove_dir_all(&backup_dir) {
log::warn!(
"[foundry-asr] remove replaced runtime backup {} failed: {error}",
backup_dir.display()
);
}
}
Ok(())
}
fn windows_app_runtime_progress_range() -> (f64, f64) {
(
WINDOWS_APP_RUNTIME_PROGRESS_START,
@@ -528,6 +588,16 @@ if ($readyForFoundryX64) { exit 0 } else { exit 1 }
fn required_libraries_present(dir: &Path) -> bool {
REQUIRED_DLLS.iter().all(|dll| dir.join(dll).exists())
&& fs::read_to_string(dir.join(RUNTIME_VERSIONS_FILE))
.map(|versions| versions == expected_runtime_versions())
.unwrap_or(false)
}
fn expected_runtime_versions() -> String {
PACKAGES
.iter()
.map(|package| format!("{}={}\n", package.name, package.version))
.collect()
}
pub fn extract_native_libraries_from_nupkg(nupkg: &Path, out_dir: &Path) -> Result<usize> {
@@ -596,6 +666,68 @@ if ($readyForFoundryX64) { exit 0 } else { exit 1 }
);
}
#[test]
fn native_packages_match_foundry_sdk_1_2_1_winml_runtime() {
let packages: Vec<_> = super::PACKAGES
.iter()
.map(|package| (package.name, package.version))
.collect();
assert_eq!(
packages,
vec![
("Microsoft.AI.Foundry.Local.Core.WinML", "1.2.1"),
("Microsoft.ML.OnnxRuntime.Foundry", "1.26.0"),
("Microsoft.ML.OnnxRuntimeGenAI.Foundry", "0.14.1"),
]
);
}
#[test]
fn native_runtime_cache_requires_matching_package_versions() {
let root = std::env::temp_dir().join(format!(
"openless-foundry-runtime-version-test-{}",
uuid::Uuid::new_v4()
));
fs::create_dir_all(&root).unwrap();
for dll in super::REQUIRED_DLLS {
fs::write(root.join(dll), b"test").unwrap();
}
assert!(!super::required_libraries_present(&root));
fs::write(
root.join("runtime-versions.txt"),
concat!(
"Microsoft.AI.Foundry.Local.Core.WinML=1.2.1\n",
"Microsoft.ML.OnnxRuntime.Foundry=1.26.0\n",
"Microsoft.ML.OnnxRuntimeGenAI.Foundry=0.14.1\n",
),
)
.unwrap();
assert!(super::required_libraries_present(&root));
fs::remove_dir_all(root).unwrap();
}
#[test]
fn failed_runtime_replacement_restores_previous_runtime() {
let root = std::env::temp_dir().join(format!(
"openless-foundry-runtime-replace-test-{}",
uuid::Uuid::new_v4()
));
let target = root.join("runtime");
let missing_staging = root.join("missing-staging");
fs::create_dir_all(&target).unwrap();
fs::write(target.join("old-runtime.dll"), b"old").unwrap();
assert!(super::replace_runtime_dir(&missing_staging, &target).is_err());
assert_eq!(fs::read(target.join("old-runtime.dll")).unwrap(), b"old");
assert!(!root.join(".runtime-backup").exists());
fs::remove_dir_all(root).unwrap();
}
#[test]
fn windows_app_runtime_detection_requires_complete_package_set() {
let script = super::windows_app_runtime_detection_script();
@@ -23,6 +23,10 @@ use crate::asr::RawTranscript;
#[cfg(target_os = "windows")]
use super::foundry_runtime::FoundryLocalRuntime;
/// Foundry Local Whisper 属于 Whisper 系模型,原生解码窗口约 30s。每次 SDK
/// 请求保持在窗口内,再由 OpenLess 合并分片文本,避免长听写只返回第一段。
const FOUNDRY_WHISPER_CHUNK_LIMIT_MS: u64 = 30_000;
pub struct FoundryLocalWhisperAsr {
#[cfg(target_os = "windows")]
runtime: Arc<FoundryLocalRuntime>,
@@ -70,6 +74,12 @@ impl FoundryLocalWhisperAsr {
self.language_hint.as_deref()
}
/// 当前缓冲音频时长(毫秒)。Coordinator 在 transcribe() 调用前读取,
/// 用来给 Foundry Local Whisper 计算动态超时。不消费缓冲。
pub fn buffer_duration_ms(&self) -> u64 {
pcm_duration_ms(&self.buffer.lock())
}
pub async fn transcribe(&self, audio_timeout: std::time::Duration) -> Result<RawTranscript> {
let cancel_generation = self.cancel_generation.load(Ordering::SeqCst);
let pcm = self.buffer.lock().clone();
@@ -108,26 +118,57 @@ impl FoundryLocalWhisperAsr {
#[cfg(target_os = "windows")]
{
let wav_file = TempWavFile::create(pcm)?;
let text = self
.runtime
.transcribe_audio_file(
&self.model_alias,
&self.runtime_source,
self.language_hint(),
wav_file.path(),
audio_timeout,
)
.await
.with_context(|| {
format!(
"transcribe audio file with Foundry Local Whisper model {}",
self.model_alias
let chunks = crate::asr::whisper::split_pcm_by_duration(
pcm,
Some(FOUNDRY_WHISPER_CHUNK_LIMIT_MS),
);
let deadline = std::time::Instant::now()
.checked_add(audio_timeout)
.ok_or_else(|| anyhow::anyhow!("Foundry Local Whisper timeout is too large"))?;
if chunks.len() > 1 {
log::info!(
"[foundry-asr] splitting {:.2}s audio into {} chunks (limit={}ms)",
duration_ms as f64 / 1000.0,
chunks.len(),
FOUNDRY_WHISPER_CHUNK_LIMIT_MS
);
}
let mut texts = Vec::with_capacity(chunks.len());
for (index, chunk) in chunks.iter().enumerate() {
let wav_file = TempWavFile::create(chunk)?;
let remaining_timeout =
remaining_transcribe_timeout(deadline, std::time::Instant::now())
.with_context(|| {
format!(
"Foundry Local Whisper total timeout exhausted before chunk {}/{}",
index + 1,
chunks.len()
)
})?;
let text = self
.runtime
.transcribe_audio_file(
&self.model_alias,
&self.runtime_source,
self.language_hint(),
wav_file.path(),
remaining_timeout,
)
})?;
.await
.with_context(|| {
format!(
"transcribe Foundry Local Whisper chunk {}/{} with model {}",
index + 1,
chunks.len(),
self.model_alias
)
})?;
texts.push(trim_transcript_text(&text));
}
Ok(RawTranscript {
text: trim_transcript_text(&text),
text: crate::asr::whisper::join_transcript_chunks(&texts),
duration_ms,
})
}
@@ -151,6 +192,16 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
crate::asr::pcm::pcm_duration_ms(pcm)
}
fn remaining_transcribe_timeout(
deadline: std::time::Instant,
now: std::time::Instant,
) -> Result<std::time::Duration> {
deadline
.checked_duration_since(now)
.filter(|duration| !duration.is_zero())
.ok_or_else(|| anyhow::anyhow!("Foundry Local Whisper total timeout exhausted"))
}
fn pcm_to_wav(pcm: &[u8]) -> Vec<u8> {
let samples: Vec<i16> = pcm
.chunks_exact(2)
@@ -293,6 +344,49 @@ mod tests {
assert_eq!(&wav[44..], &[0x01, 0x00, 0xff, 0x7f]);
}
#[test]
fn foundry_provider_splits_long_pcm_at_whisper_window() {
let pcm = vec![0u8; 32_000 * 65];
let chunks = crate::asr::whisper::split_pcm_by_duration(
&pcm,
Some(super::FOUNDRY_WHISPER_CHUNK_LIMIT_MS),
);
assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].len(), 32_000 * 30);
assert_eq!(chunks[1].len(), 32_000 * 30);
assert_eq!(chunks[2].len(), 32_000 * 5);
}
#[test]
fn foundry_chunk_timeout_uses_remaining_total_budget() {
let started = std::time::Instant::now();
let deadline = started + std::time::Duration::from_secs(85);
assert_eq!(
super::remaining_transcribe_timeout(
deadline,
started + std::time::Duration::from_secs(30),
)
.unwrap(),
std::time::Duration::from_secs(55)
);
assert!(super::remaining_transcribe_timeout(deadline, deadline).is_err());
}
#[test]
fn foundry_provider_reports_buffer_duration_without_consuming() {
#[cfg(target_os = "windows")]
let (provider, _) = test_provider();
#[cfg(not(target_os = "windows"))]
let provider = test_provider();
provider.consume_pcm_chunk(&vec![0u8; 32_000]);
assert_eq!(provider.buffer_duration_ms(), 1000);
assert_eq!(provider.buffer.lock().len(), 32_000);
}
#[cfg(target_os = "windows")]
#[test]
fn foundry_provider_temp_wav_drop_removes_file() {
@@ -103,6 +103,19 @@ impl SherpaOnnxAsr {
self.language_hint.as_deref()
}
/// 当前缓冲音频时长(毫秒)。Offline 读 PCM buffer;Online 读 worker 已接收
/// 的 PCM 字节数。不消费缓冲。
pub fn buffer_duration_ms(&self) -> u64 {
match &self.mode {
SherpaProviderMode::Offline { buffer } => pcm_duration_ms(&buffer.lock()),
SherpaProviderMode::Online { worker } => worker
.lock()
.as_ref()
.map(|worker| pcm_duration_ms_from_bytes(worker.audio_bytes.load(Ordering::SeqCst)))
.unwrap_or(0),
}
}
pub async fn transcribe(&self, audio_timeout: Duration) -> Result<RawTranscript> {
match &self.mode {
SherpaProviderMode::Offline { buffer } => {
@@ -390,6 +403,18 @@ mod tests {
}
}
#[test]
fn offline_buffer_duration_reports_16k_pcm_duration_without_consuming() {
let provider = make_provider();
provider.consume_pcm_chunk(&vec![0u8; 32_000]);
assert_eq!(provider.buffer_duration_ms(), 1000);
match &provider.mode {
SherpaProviderMode::Offline { buffer } => assert_eq!(buffer.lock().len(), 32_000),
SherpaProviderMode::Online { .. } => panic!("expected offline provider"),
}
}
#[tokio::test]
async fn empty_buffer_transcribe_returns_empty_transcript() {
let provider = make_provider();
@@ -294,7 +294,7 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
super::pcm::pcm_duration_ms(pcm)
}
fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: Option<u64>) -> Vec<&[u8]> {
pub(crate) fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: Option<u64>) -> Vec<&[u8]> {
let Some(max_chunk_duration_ms) = max_chunk_duration_ms else {
return vec![pcm];
};
@@ -328,7 +328,7 @@ fn transcription_url(base_url: &str) -> Result<String> {
Ok(url.to_string())
}
fn join_transcript_chunks(chunks: &[String]) -> String {
pub(crate) fn join_transcript_chunks(chunks: &[String]) -> String {
let mut joined = String::new();
for chunk in chunks.iter().map(|chunk| chunk.trim()) {
if chunk.is_empty() {
+32 -23
View File
@@ -1575,15 +1575,21 @@ impl Coordinator {
.map_err(|_| "重新转录超时".to_string())?
.map_err(|e| e.to_string())?,
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
.map_err(|e| e.to_string())?,
ActiveAsr::FoundryLocalWhisper(local) => {
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
local
.transcribe(windows_local_asr_transcribe_timeout(audio_secs))
.await
.map_err(|e| e.to_string())?
}
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
.map_err(|e| e.to_string())?,
ActiveAsr::SherpaOnnxLocal(local) => {
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
local
.transcribe(windows_local_asr_transcribe_timeout(audio_secs))
.await
.map_err(|e| e.to_string())?
}
#[cfg(target_os = "macos")]
ActiveAsr::Local(local) => {
let dur =
@@ -2379,17 +2385,23 @@ mod tests {
assert!(!asr_transcribe_uses_global_timeout(&active_asr));
}
#[cfg(target_os = "windows")]
#[test]
fn foundry_audio_transcribe_timeout_is_separate_from_prepare() {
let timeout = foundry_audio_transcribe_timeout_duration();
fn windows_local_asr_timeout_floors_at_global_timeout_for_short_audio() {
assert_eq!(
timeout,
windows_local_asr_transcribe_timeout(5.0),
std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS)
);
}
#[test]
fn windows_local_asr_timeout_scales_with_audio_duration() {
// 65s 录音:65 × 1.0 = 65,+20 = 85s。长音频不再撞 30s 墙。
assert_eq!(
windows_local_asr_transcribe_timeout(65.0),
std::time::Duration::from_secs(85)
);
}
#[test]
fn local_qwen_timeout_floors_at_global_timeout_for_short_audio() {
// 5s 录音:5 × 0.6 = 3, +10 = 13, max(30) = 30。短录音兜底。
@@ -3021,9 +3033,13 @@ const POST_SESSION_COOLDOWN_MS: u64 = 600;
/// 网络超时预算;只在 ASR 自身超时机制失效时作为最后的防线触发。
const COORDINATOR_GLOBAL_TIMEOUT_SECS: u64 = 30;
#[cfg(target_os = "windows")]
fn foundry_audio_transcribe_timeout_duration() -> std::time::Duration {
std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS)
/// Windows 本地 batch ASR 的动态转写超时。Foundry 与 sherpa-onnx 当前使用
/// 同一预算:短音频至少 30s,长音频按整段时长向上取整后增加 20s 余量。
fn windows_local_asr_transcribe_timeout(audio_secs: f64) -> std::time::Duration {
let secs = (audio_secs.ceil() as u64)
.saturating_add(20)
.max(COORDINATOR_GLOBAL_TIMEOUT_SECS);
std::time::Duration::from_secs(secs)
}
/// 本地 Qwen3-ASR 的动态转写超时。固定 15 秒在长录音(≥ 30s)+ 慢机器
@@ -3048,13 +3064,6 @@ fn whisper_transcribe_timeout(audio_secs: f64) -> std::time::Duration {
std::time::Duration::from_secs(secs)
}
/// sherpa-onnx offline batch 暂与 Foundry 同档;后续按 Windows 真机 CPU/模型
/// 实测结果再调整。
#[cfg(target_os = "windows")]
fn sherpa_audio_transcribe_timeout_duration() -> std::time::Duration {
std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS)
}
pub(crate) fn validate_llm_endpoint(raw: &str) -> anyhow::Result<()> {
use std::net::IpAddr;
@@ -2247,10 +2247,14 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = windows_local_asr_transcribe_timeout(audio_secs);
log::info!(
"[coord] Foundry Local Whisper transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_foundry_local_asr_release(
inner,
@@ -2285,10 +2289,14 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = windows_local_asr_transcribe_timeout(audio_secs);
log::info!(
"[coord] sherpa-onnx transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_sherpa_onnx_release(
inner,
@@ -263,10 +263,9 @@ pub(super) async fn transcribe_overlay_dictation_asr(
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = windows_local_asr_transcribe_timeout(audio_secs);
match local.transcribe(timeout_duration).await {
Ok(raw) => {
schedule_foundry_local_asr_release(
_inner,
@@ -286,10 +285,9 @@ pub(super) async fn transcribe_overlay_dictation_asr(
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = windows_local_asr_transcribe_timeout(audio_secs);
match local.transcribe(timeout_duration).await {
Ok(raw) => {
schedule_sherpa_onnx_release(
_inner,
@@ -826,10 +824,14 @@ pub(super) async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(foundry_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = windows_local_asr_transcribe_timeout(audio_secs);
log::info!(
"[coord] QA Foundry Local Whisper transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_foundry_local_asr_release(inner, AsrReleaseSession::Qa(qa_session_id));
r
@@ -852,10 +854,14 @@ pub(super) async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
#[cfg(target_os = "windows")]
ActiveAsr::SherpaOnnxLocal(local) => {
debug_assert!(!uses_global_timeout);
match local
.transcribe(sherpa_audio_transcribe_timeout_duration())
.await
{
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
let timeout_duration = windows_local_asr_transcribe_timeout(audio_secs);
log::info!(
"[coord] QA sherpa-onnx transcribe: audio={:.2}s timeout={}s",
audio_secs,
timeout_duration.as_secs()
);
match local.transcribe(timeout_duration).await {
Ok(r) => {
schedule_sherpa_onnx_release(inner, AsrReleaseSession::Qa(qa_session_id));
r