fix(asr): Apple Speech 保留停顿分段，修复停顿前内容丢失 (#824)
* fix(asr): Apple Speech 保留停顿分段，修复停顿前内容丢失 Apple 设备端识别会在语音停顿处把音频切成多个话段（utterance），每个话段 的回调只携带该话段的文本，随后 partial 从空重新累计；isFinal 通常只跟着 最后一个话段出现。原实现「拿到第一个 isFinal 就返回」，导致停顿之前的所有 话段被整段丢弃——用户表现为「说话中途停顿思考，前面说的全没了」。 - 新增 SegmentAccumulator：以 speechRecognitionMetadata 非空作为话段边界 信号，逐话段落袋，结束后按 CJK / 空格语言规则拼接返回。 - 结束判定由「见到第一个 final 就收工」改为轮询 SFSpeechRecognitionTask .state == completed（辅以 isFinal 后静默的后备条件），兼容部分系统按话段 多次发 isFinal；completed 后留 250ms 收尾窗口，避免竞态截掉尾段。 - 三重防丢 / 防重：partial 骤缩时判定静默话段重置并抢救上一段；引擎重放 累计全文时按前缀 / 空白不敏感规则去重；识别中途报错时返回已累积话段而非 整段作废。 - 等待预算由固定 60s 硬顶改为 max(60s, 音频时长 + 30s)，长录音不再被 「等待语音识别结果超时」截断。 - 新增 12 个单元测试覆盖多话段累积、逐话段 final、静默重置抢救、去重、 拼接规则与等待预算。 单话段（无停顿）场景输出与修复前逐字节相同，云端识别路径不受影响，因此不 存在回归风险。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(asr): harden Apple Speech segment lifecycle * fix(asr): preserve repeated final-only utterances --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: baiqing <lbx12309@icloud.com>
This commit is contained in:
1 file changed
+750
-66
@@ -20,12 +20,12 @@
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use block2::RcBlock;
|
||||
use objc2::msg_send;
|
||||
use objc2::runtime::{AnyClass, AnyObject, Bool};
|
||||
use objc2::{msg_send, sel};
|
||||
use parking_lot::Mutex;
|
||||
|
||||
use crate::asr::wav::encode_wav_16k_mono;
|
||||
@@ -37,13 +37,24 @@ const SF_AUTH_DENIED: i64 = 1;
|
||||
const SF_AUTH_RESTRICTED: i64 = 2;
|
||||
const SF_AUTH_AUTHORIZED: i64 = 3;
|
||||
|
||||
/// 等待识别 / 授权回调的兜底超时。识别本身另有 coordinator 侧动态超时；
|
||||
/// 这里只防 block 永不回调导致线程永久阻塞。
|
||||
/// 等待识别回调的兜底超时**下限**。识别本身另有 coordinator 侧动态超时；这里只防
|
||||
/// block 永不回调导致线程永久阻塞。长录音按音频时长放大，见 `recognition_wait_budget`。
|
||||
const RECOGNITION_WAIT: Duration = Duration::from_secs(60);
|
||||
/// 识别等待的轮询步长。把 `recv_timeout(RECOGNITION_WAIT)` 的一次性阻塞拆成每
|
||||
/// `RECOGNITION_POLL` 一轮：每轮之间检查 `cancel_flag`，取消 / 上层超时后阻塞线程
|
||||
/// 最多再等这一步长（~100ms）就退出，而不是傻等满 `RECOGNITION_WAIT`（60s）。
|
||||
/// 识别等待的轮询步长。每轮之间检查 `cancel_flag` 与任务状态：取消 / 上层超时后阻塞
|
||||
/// 线程最多再等这一步长（~100ms）就退出，而不是傻等满整个等待预算。
|
||||
const RECOGNITION_POLL: Duration = Duration::from_millis(100);
|
||||
/// `SFSpeechRecognitionTaskState`（NS_ENUM(NSInteger)）的 completed。任务终结
|
||||
/// （成功、失败或取消）后进入该状态，是「不会再有回调」的权威信号。
|
||||
const SF_TASK_STATE_COMPLETED: i64 = 4;
|
||||
/// 观察到任务 completed 后再等这一小段，让已经在飞的最后一次 resultHandler 回调
|
||||
/// 落进累积器，避免「state 先翻转、回调后到」的竞态把最后一个话段截掉。
|
||||
const COMPLETION_GRACE: Duration = Duration::from_millis(250);
|
||||
/// 后备终止条件：已见 isFinal 且此后静默这么久，视为识别结束。只防 `state` 轮询
|
||||
/// 因系统差异拿不到 completed 时无限等待；正常路径由 completed + COMPLETION_GRACE
|
||||
/// 快速收账，不受此值影响。取 5s 是因为多话段场景 isFinal 可能逐话段出现，话段间
|
||||
/// 的回调空窗（对应音频里的长停顿）必须远小于该阈值，否则会提前收账截掉后文——
|
||||
/// 批处理识别消化静音远快于实时，5s 空窗足够安全。
|
||||
const FINAL_QUIESCENCE: Duration = Duration::from_secs(5);
|
||||
const AUTHORIZATION_WAIT: Duration = Duration::from_secs(30);
|
||||
/// 识别引擎就绪（isAvailable）的轮询等待：刚 init 的 recognizer 常瞬时不可用（异步
|
||||
/// 加载语言资源），稍等即就绪。等满仍不可用才报错——修「有时用不了」的竞态。
|
||||
@@ -199,7 +210,7 @@ fn transcribe_pcm_blocking(
|
||||
let path_str = path
|
||||
.to_str()
|
||||
.ok_or_else(|| anyhow!("临时 wav 路径含非 UTF-8 字符: {}", path.display()))?;
|
||||
let text = recognize_file(path_str, locale, cancel_flag, active_task)?;
|
||||
let text = recognize_file(path_str, locale, duration_ms, cancel_flag, active_task)?;
|
||||
|
||||
Ok(RawTranscript { text, duration_ms })
|
||||
}
|
||||
@@ -254,13 +265,19 @@ fn ensure_authorized() -> Result<()> {
|
||||
/// 用 `SFSpeechURLRecognitionRequest` 对给定 wav 文件做一次批处理识别，
|
||||
/// 把 `recognitionTaskWithRequest:resultHandler:` 的异步回调同步化。
|
||||
///
|
||||
/// 等待不再是一次性 `recv_timeout(RECOGNITION_WAIT)`，而是每 `RECOGNITION_POLL`
|
||||
/// 一轮的轮询：每轮检查 `cancel_flag`，置位则 `cancel` 底层任务并返回「已取消」错误，
|
||||
/// 让上层动态超时抛弃 / `cancel()` 触发时阻塞线程在 ~100ms 内退出。返回前无论成败
|
||||
/// 都清空 `active_task`（RAII guard 兜底 `?` 早退）。
|
||||
/// **多话段累积（修「停顿后前文丢失」）**：设备端识别会在语音停顿处把音频切成多个
|
||||
/// 话段（utterance），逐话段回调、逐话段重置文本（见 `SegmentAccumulator` 文档）。
|
||||
/// 因此不能「见到第一个 isFinal 就收工」——resultHandler 只负责把每次回调喂进
|
||||
/// `SegmentAccumulator`；等待循环以 `task.state == completed`（辅以 isFinal 后静默
|
||||
/// 的后备条件）判定识别真正结束，再把所有话段拼接返回。
|
||||
///
|
||||
/// 等待是每 `RECOGNITION_POLL` 一轮的轮询：每轮检查 `cancel_flag`，置位则 `cancel`
|
||||
/// 底层任务并返回「已取消」错误，让上层动态超时抛弃 / `cancel()` 触发时阻塞线程在
|
||||
/// ~100ms 内退出。返回前无论成败都清空 `active_task`（RAII guard 兜底 `?` 早退）。
|
||||
fn recognize_file(
|
||||
wav_path: &str,
|
||||
locale: Option<&str>,
|
||||
duration_ms: u64,
|
||||
cancel_flag: &AtomicBool,
|
||||
active_task: &Mutex<Option<SendableTask>>,
|
||||
) -> Result<String> {
|
||||
@@ -279,14 +296,19 @@ fn recognize_file(
|
||||
// 语言回退系统默认（可能走网络）以保底能用。
|
||||
configure_on_device(recognizer, request);
|
||||
|
||||
let (tx, rx) = mpsc::channel::<RecognitionOutcome>();
|
||||
// resultHandler: void(^)(SFSpeechRecognitionResult *result, NSError *error)
|
||||
// 显式开启 partial 回调：话段边界信号（speechRecognitionMetadata 非空的结果）
|
||||
// 出现在非 final 回调里，关掉 partial 就拿不到边界、无从累积。
|
||||
// SAFETY: `request` 是 SFSpeechURLRecognitionRequest（父类提供该 BOOL setter）。
|
||||
let _: () = unsafe { msg_send![request, setShouldReportPartialResults: Bool::new(true)] };
|
||||
|
||||
let shared = Arc::new(Mutex::new(RecognitionShared::default()));
|
||||
let shared_cb = Arc::clone(&shared);
|
||||
// resultHandler: void(^)(SFSpeechRecognitionResult *result, NSError *error)。
|
||||
// 回调只做「解包 + 喂累积器」，结束判定完全交给下面的等待循环。
|
||||
let block = RcBlock::new(move |result: *mut AnyObject, error: *mut AnyObject| {
|
||||
let outcome = build_outcome(result, error);
|
||||
// 只取第一个 final（或第一个 error）。后续重复回调忽略。
|
||||
if outcome.is_terminal() {
|
||||
let _ = tx.send(outcome);
|
||||
}
|
||||
let (recognized, callback_error) = extract_callback(result, error);
|
||||
let mut s = shared_cb.lock();
|
||||
s.record_callback(recognized, callback_error, Instant::now());
|
||||
});
|
||||
|
||||
log::info!("[apple-speech] starting recognitionTaskWithRequest");
|
||||
@@ -306,36 +328,76 @@ fn recognize_file(
|
||||
*active_task.lock() = Some(SendableTask(task));
|
||||
let _task_guard = ActiveTaskGuard(active_task);
|
||||
|
||||
// 轮询等待：每轮 recv 最多 RECOGNITION_POLL，累计超过 RECOGNITION_WAIT 则超时。
|
||||
// 每轮先查 cancel_flag —— 置位即真正 cancel 任务并返回「已取消」错误、放弃等待。
|
||||
let deadline = std::time::Instant::now() + RECOGNITION_WAIT;
|
||||
// 轮询等待：每轮先查 cancel_flag，再看错误 / 终止条件；超过按音频时长放大的
|
||||
// 等待预算则超时（外层 coordinator 的动态超时通常先于它触发，这里只防回调失联）。
|
||||
let deadline = Instant::now() + recognition_wait_budget(duration_ms);
|
||||
loop {
|
||||
if cancel_flag.load(Ordering::SeqCst) {
|
||||
// 若 cancel() 尚未取走句柄（例如超时路径只置了 flag 没调 cancel），这里补发
|
||||
// 一次 cancel，确保底层识别任务被真正终止，而不是留它在后台跑满。
|
||||
if let Some(t) = active_task.lock().take() {
|
||||
// SAFETY: 见 SendableTask 文档 —— `cancel` 无参、可跨线程调用，仅调用不解引用。
|
||||
let _: () = unsafe { msg_send![t.0, cancel] };
|
||||
}
|
||||
bail!("语音识别已取消");
|
||||
}
|
||||
match rx.recv_timeout(RECOGNITION_POLL) {
|
||||
Ok(RecognitionOutcome::Final(text)) => return Ok(text),
|
||||
Ok(RecognitionOutcome::Failed(message)) => bail!("语音识别失败: {message}"),
|
||||
Ok(RecognitionOutcome::Pending) => unreachable!("Pending 不会被发送"),
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
if std::time::Instant::now() >= deadline {
|
||||
bail!("等待语音识别结果超时");
|
||||
let now = Instant::now();
|
||||
let mut s = shared.lock();
|
||||
let decision = s.lifecycle.decide(
|
||||
now,
|
||||
cancel_flag.load(Ordering::SeqCst),
|
||||
s.error.is_some(),
|
||||
now >= deadline,
|
||||
);
|
||||
match decision {
|
||||
RecognitionDecision::Cancel => {
|
||||
drop(s);
|
||||
// 若 cancel() 尚未取走句柄（例如超时路径只置了 flag 没调 cancel），这里
|
||||
// 补发一次 cancel，确保底层识别任务被真正终止，而不是留它在后台跑满。
|
||||
if let Some(t) = active_task.lock().take() {
|
||||
// SAFETY: 见 SendableTask 文档 —— `cancel` 无参、可跨线程调用，仅调用不解引用。
|
||||
let _: () = unsafe { msg_send![t.0, cancel] };
|
||||
}
|
||||
// 未到总时限：继续下一轮（回到循环顶部再查 cancel_flag）。
|
||||
bail!("语音识别已取消");
|
||||
}
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => {
|
||||
bail!("等待语音识别结果失败：回调通道已断开");
|
||||
RecognitionDecision::Error => {
|
||||
let Some(err) = s.error.take() else {
|
||||
bail!("语音识别失败：终止状态缺少错误详情");
|
||||
};
|
||||
// result 与 error 同次到达时，record_callback 已先折叠 result；这里抢救只
|
||||
// 收账一次，不会因错误回放已提交的 final。
|
||||
let salvaged = s.acc.salvage();
|
||||
if salvaged.is_empty() {
|
||||
bail!("语音识别失败: {err}");
|
||||
}
|
||||
log::warn!(
|
||||
"[apple-speech] recognition error after {} segment(s); returning salvaged text: {err}",
|
||||
s.acc.segment_count()
|
||||
);
|
||||
return Ok(salvaged);
|
||||
}
|
||||
RecognitionDecision::Finish => {
|
||||
let text = s.acc.salvage();
|
||||
log::info!(
|
||||
"[apple-speech] recognition finished: {} segment(s), {} chars",
|
||||
s.acc.segment_count(),
|
||||
text.chars().count()
|
||||
);
|
||||
return Ok(text);
|
||||
}
|
||||
RecognitionDecision::Timeout => bail!("等待语音识别结果超时"),
|
||||
RecognitionDecision::Wait => drop(s),
|
||||
}
|
||||
|
||||
std::thread::sleep(RECOGNITION_POLL);
|
||||
// SAFETY: `task` 在本栈帧内被 recognizer 强引用存活（见上）；`state` 是无参
|
||||
// 只读属性，返回 NSInteger（i64）。跨线程读一个整型属性，最坏读到瞬时旧值，
|
||||
// 下一轮（~100ms 后）即追上，不影响正确性。
|
||||
let state: i64 = unsafe { msg_send![task, state] };
|
||||
if state == SF_TASK_STATE_COMPLETED {
|
||||
shared.lock().lifecycle.record_completed(Instant::now());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 识别等待预算：音频时长 + 30s，且不低于 `RECOGNITION_WAIT`。批处理识别通常远快于
|
||||
/// 实时，但长录音（多话段逐段吐结果）不该被固定 60s 硬顶截断——旧实现对超过 60s
|
||||
/// 才识别完的长录音会直接报「等待超时」。外层 coordinator 的动态超时仍然先兜底。
|
||||
fn recognition_wait_budget(duration_ms: u64) -> Duration {
|
||||
RECOGNITION_WAIT.max(Duration::from_millis(duration_ms).saturating_add(Duration::from_secs(30)))
|
||||
}
|
||||
|
||||
/// 保证 `recognize_file` 任意退出路径（含 `?` 早退、正常返回、取消/超时）都把
|
||||
/// `active_task` 清回 `None`，避免悬挂的 task 句柄被后续 `cancel()` 误用。
|
||||
struct ActiveTaskGuard<'a>(&'a Mutex<Option<SendableTask>>);
|
||||
@@ -384,43 +446,372 @@ fn configure_on_device(recognizer: *mut AnyObject, request: *mut AnyObject) {
|
||||
}
|
||||
}
|
||||
|
||||
/// 识别回调的归一化结果。
|
||||
enum RecognitionOutcome {
|
||||
/// 还没拿到 final（partial），不发送。
|
||||
Pending,
|
||||
Final(String),
|
||||
Failed(String),
|
||||
/// resultHandler 回调与等待循环之间的共享状态（block 侧写，轮询侧读）。
|
||||
#[derive(Default)]
|
||||
struct RecognitionShared {
|
||||
acc: SegmentAccumulator,
|
||||
lifecycle: RecognitionLifecycle,
|
||||
/// 第一个识别错误（保留首个，后续忽略）。
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
impl RecognitionOutcome {
|
||||
fn is_terminal(&self) -> bool {
|
||||
!matches!(self, RecognitionOutcome::Pending)
|
||||
struct RecognizedCallback {
|
||||
text: String,
|
||||
/// 本次结果带 `speechRecognitionMetadata`（非空）——一个话段（utterance）
|
||||
/// 到此结束，`text` 是该话段的完整文本。
|
||||
utterance_ended: bool,
|
||||
is_final: bool,
|
||||
}
|
||||
|
||||
impl RecognitionShared {
|
||||
fn record_callback(
|
||||
&mut self,
|
||||
recognized: Option<RecognizedCallback>,
|
||||
error: Option<String>,
|
||||
at: Instant,
|
||||
) {
|
||||
if let Some(result) = recognized {
|
||||
if result.utterance_ended {
|
||||
log::info!(
|
||||
"[apple-speech] utterance boundary: segment captured ({} chars)",
|
||||
result.text.chars().count()
|
||||
);
|
||||
}
|
||||
self.acc
|
||||
.fold(&result.text, result.utterance_ended, result.is_final);
|
||||
self.lifecycle.record_callback(at, result.is_final);
|
||||
}
|
||||
if self.error.is_none() {
|
||||
self.error = error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `(result, error)` 回调参数提取最终文本或错误。
|
||||
fn build_outcome(result: *mut AnyObject, error: *mut AnyObject) -> RecognitionOutcome {
|
||||
if !error.is_null() {
|
||||
return RecognitionOutcome::Failed(ns_error_description(error));
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum RecognitionDecision {
|
||||
Wait,
|
||||
Finish,
|
||||
Cancel,
|
||||
Error,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecognitionLifecycle {
|
||||
completed_at: Option<Instant>,
|
||||
last_callback_at: Option<Instant>,
|
||||
saw_final: bool,
|
||||
}
|
||||
|
||||
impl RecognitionLifecycle {
|
||||
fn record_callback(&mut self, at: Instant, is_final: bool) {
|
||||
self.last_callback_at = Some(at);
|
||||
self.saw_final |= is_final;
|
||||
}
|
||||
|
||||
fn record_completed(&mut self, at: Instant) {
|
||||
self.completed_at.get_or_insert(at);
|
||||
}
|
||||
|
||||
fn decide(
|
||||
&self,
|
||||
now: Instant,
|
||||
cancelled: bool,
|
||||
has_error: bool,
|
||||
deadline_reached: bool,
|
||||
) -> RecognitionDecision {
|
||||
if cancelled {
|
||||
return RecognitionDecision::Cancel;
|
||||
}
|
||||
if has_error {
|
||||
return RecognitionDecision::Error;
|
||||
}
|
||||
|
||||
let completion_settled = self
|
||||
.completed_at
|
||||
.map(|at| now.saturating_duration_since(at) >= COMPLETION_GRACE)
|
||||
.unwrap_or(false)
|
||||
&& self
|
||||
.last_callback_at
|
||||
.map(|at| now.saturating_duration_since(at) >= COMPLETION_GRACE)
|
||||
.unwrap_or(true);
|
||||
let final_quiesced = self.saw_final
|
||||
&& self
|
||||
.last_callback_at
|
||||
.map(|at| now.saturating_duration_since(at) >= FINAL_QUIESCENCE)
|
||||
.unwrap_or(false);
|
||||
if completion_settled || final_quiesced {
|
||||
return RecognitionDecision::Finish;
|
||||
}
|
||||
if deadline_reached {
|
||||
return RecognitionDecision::Timeout;
|
||||
}
|
||||
RecognitionDecision::Wait
|
||||
}
|
||||
}
|
||||
|
||||
/// 从 `(result, error)` 同时解包识别结果与错误。Apple 允许二者同次出现；调用方必须先
|
||||
/// 折叠结果、再记录错误，确保错误抢救包含这次最后文本且只收账一次。
|
||||
fn extract_callback(
|
||||
result: *mut AnyObject,
|
||||
error: *mut AnyObject,
|
||||
) -> (Option<RecognizedCallback>, Option<String>) {
|
||||
let callback_error = if !error.is_null() {
|
||||
Some(ns_error_description(error))
|
||||
} else if result.is_null() {
|
||||
Some("识别返回空结果".to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if result.is_null() {
|
||||
return RecognitionOutcome::Failed("识别返回空结果".to_string());
|
||||
return (None, callback_error);
|
||||
}
|
||||
// result.isFinal —— 只有 final 才取文本；partial 让上层继续等。
|
||||
// SAFETY: `result` 非空，是 `SFSpeechRecognitionResult`；`isFinal` 无参返回 BOOL。
|
||||
let is_final: Bool = unsafe { msg_send![result, isFinal] };
|
||||
if !is_final.as_bool() {
|
||||
return RecognitionOutcome::Pending;
|
||||
}
|
||||
// speechRecognitionMetadata 非空 = 一个话段结束（macOS 11.3+）。老系统没有该
|
||||
// selector，先 respondsToSelector 探测，避免直接调用未知 selector 崩溃。
|
||||
// SAFETY: `respondsToSelector:` 是 NSObject 协议方法，参数为 Sel，返回 BOOL。
|
||||
let has_metadata_sel: Bool =
|
||||
unsafe { msg_send![result, respondsToSelector: sel!(speechRecognitionMetadata)] };
|
||||
let utterance_ended = if has_metadata_sel.as_bool() {
|
||||
// SAFETY: 上面已确认 selector 存在；无参返回对象指针（可能为 nil）。
|
||||
let metadata: *mut AnyObject = unsafe { msg_send![result, speechRecognitionMetadata] };
|
||||
!metadata.is_null()
|
||||
} else {
|
||||
false
|
||||
};
|
||||
// result.bestTranscription.formattedString → NSString → Rust String。
|
||||
// SAFETY: `result` 是 final 的 `SFSpeechRecognitionResult`，`bestTranscription`
|
||||
// 非空（final 结果保证有 transcription）；`formattedString` 返回 NSString。
|
||||
// SAFETY: `result` 非空；`bestTranscription` 返回 SFTranscription（可能为 nil），
|
||||
// `formattedString` 返回 NSString。
|
||||
let transcription: *mut AnyObject = unsafe { msg_send![result, bestTranscription] };
|
||||
if transcription.is_null() {
|
||||
return RecognitionOutcome::Final(String::new());
|
||||
let text = if transcription.is_null() {
|
||||
String::new()
|
||||
} else {
|
||||
let formatted: *mut AnyObject = unsafe { msg_send![transcription, formattedString] };
|
||||
ns_string_to_rust(formatted)
|
||||
};
|
||||
let recognized = RecognizedCallback {
|
||||
text,
|
||||
utterance_ended,
|
||||
is_final: is_final.as_bool(),
|
||||
};
|
||||
(Some(recognized), callback_error)
|
||||
}
|
||||
|
||||
/// 跨话段累积识别文本（修「停顿后前文丢失」，issue：Apple Speech 停顿截断）。
|
||||
///
|
||||
/// Apple 设备端识别（`requiresOnDeviceRecognition`）会在语音停顿处把音频切成多个
|
||||
/// 「话段」(utterance)：每个话段结束时回调一次带 `speechRecognitionMetadata` 的结果
|
||||
/// （其文本**只覆盖该话段**），随后 partial 文本从空重新累计；`isFinal` 通常只在最后
|
||||
/// 一个话段出现（个别系统版本按话段多次 isFinal）。旧实现只取第一个 isFinal 的文本，
|
||||
/// 停顿之前的所有话段被整段丢弃——这正是「说话中间停顿思考，前面内容全没了」的根因。
|
||||
/// 这里把每个话段落袋，识别结束时按 CJK 规则拼接返回。
|
||||
///
|
||||
/// 云端（服务器）识别没有话段重置：partial 全程累计、final 为全文。此时 `segments`
|
||||
/// 只会收到一条 final 全文（或经前缀替换归并），行为与旧实现一致。
|
||||
#[derive(Default)]
|
||||
struct SegmentAccumulator {
|
||||
/// 已结束话段的文本，按时间顺序。
|
||||
segments: Vec<String>,
|
||||
/// 当前话段最新 partial 文本。
|
||||
current: String,
|
||||
/// 自上次明确边界提交后，是否见过新一代 partial。它把「下一话段」与同一任务在
|
||||
/// 结尾重放 final 全文区分开，避免用跨话段文本前缀猜测身份。
|
||||
current_generation_active: bool,
|
||||
/// 最近一次 metadata 边界提交后，任务可能在完成时重放的累计全文。只有明确边界
|
||||
/// 才能创建这个候选；纯 isFinal 序列即使文本相同也必须视为独立话段。
|
||||
cumulative_replay_candidate: Option<String>,
|
||||
}
|
||||
|
||||
impl SegmentAccumulator {
|
||||
/// 喂入一次识别回调。`utterance_ended` / `is_final` 的文本视为所在话段的完整
|
||||
/// 文本并落袋；普通 partial 只更新 `current`，除非检测到「静默重置」。
|
||||
fn fold(&mut self, text: &str, utterance_ended: bool, is_final: bool) {
|
||||
if utterance_ended {
|
||||
// metadata 是 Apple 给出的独立 utterance 证据；即使相邻文本相同或互为
|
||||
// 前缀也必须分别提交，不能把正常复述/自我修正当累计回放吞掉。
|
||||
let segment = if text.trim().is_empty() {
|
||||
std::mem::take(&mut self.current)
|
||||
} else {
|
||||
text.to_string()
|
||||
};
|
||||
self.push_segment(&segment);
|
||||
self.current.clear();
|
||||
self.current_generation_active = false;
|
||||
self.cumulative_replay_candidate = Some(normalized(&self.joined()));
|
||||
} else if is_final {
|
||||
let segment = if text.trim().is_empty() {
|
||||
std::mem::take(&mut self.current)
|
||||
} else {
|
||||
text.to_string()
|
||||
};
|
||||
// 只有 metadata 边界创建的快照能证明这是同一 task 的累计全文重放；不能仅
|
||||
// 因 final 文本等于 joined 就去重，否则连续两个相同的 final-only 话段会丢失。
|
||||
let normalized_segment = normalized(&segment);
|
||||
let is_cumulative_replay = !self.current_generation_active
|
||||
&& self.cumulative_replay_candidate.as_deref() == Some(normalized_segment.as_str());
|
||||
if !is_cumulative_replay {
|
||||
self.push_segment(&segment);
|
||||
self.cumulative_replay_candidate = None;
|
||||
}
|
||||
self.current.clear();
|
||||
self.current_generation_active = false;
|
||||
} else if self.reset_detected(text) {
|
||||
// 防守路径：没有 metadata 边界回调、partial 却骤缩——设备端识别已悄悄
|
||||
// 重开话段。把上一话段已见的最长 partial 先落袋，再从新文本重新累计。
|
||||
let previous = std::mem::take(&mut self.current);
|
||||
self.push_segment(&previous);
|
||||
self.current = text.to_string();
|
||||
self.current_generation_active = true;
|
||||
self.cumulative_replay_candidate = None;
|
||||
} else {
|
||||
self.current = text.to_string();
|
||||
self.current_generation_active = true;
|
||||
self.cumulative_replay_candidate = None;
|
||||
}
|
||||
}
|
||||
let formatted: *mut AnyObject = unsafe { msg_send![transcription, formattedString] };
|
||||
RecognitionOutcome::Final(ns_string_to_rust(formatted))
|
||||
|
||||
/// partial 骤缩视为话段重置。阈值保守（原文本 ≥12 字符且新文本缩到 1/3 以下）：
|
||||
/// 识别器正常的假设修正只会小幅增删，不会缩水到这个程度。
|
||||
fn reset_detected(&self, text: &str) -> bool {
|
||||
let current_chars = self.current.chars().count();
|
||||
let new_chars = text.chars().count();
|
||||
current_chars >= 12 && new_chars.saturating_mul(3) < current_chars
|
||||
}
|
||||
|
||||
/// 明确话段落袋。调用方先依据 metadata / generation / final 状态判定提交身份；此处
|
||||
/// 不做跨话段文本启发式去重，避免吞掉正常复述与前缀式自我修正。
|
||||
fn push_segment(&mut self, text: &str) {
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
return;
|
||||
}
|
||||
self.segments.push(trimmed.to_string());
|
||||
}
|
||||
|
||||
/// 结束收账：把残余 partial 落袋后返回全部话段的拼接文本。
|
||||
fn salvage(&mut self) -> String {
|
||||
if self.current_generation_active {
|
||||
let current = std::mem::take(&mut self.current);
|
||||
self.push_segment(¤t);
|
||||
self.current_generation_active = false;
|
||||
}
|
||||
self.joined()
|
||||
}
|
||||
|
||||
fn segment_count(&self) -> usize {
|
||||
self.segments.len()
|
||||
}
|
||||
|
||||
/// 话段拼接：汉字、平假名、片假名及中日标点按无空格书写习惯连接；其它脚本
|
||||
/// （包括韩文、俄文、阿文）默认补词间空格。润色模式下 LLM 仍会再整理。
|
||||
fn joined(&self) -> String {
|
||||
let mut out = String::new();
|
||||
for segment in &self.segments {
|
||||
if out.is_empty() {
|
||||
out.push_str(segment);
|
||||
continue;
|
||||
}
|
||||
let join_bare = matches!(
|
||||
(out.chars().last(), segment.chars().next()),
|
||||
(Some(prev), Some(next)) if should_join_without_space(prev, next)
|
||||
);
|
||||
if !join_bare {
|
||||
out.push(' ');
|
||||
}
|
||||
out.push_str(segment);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
fn should_join_without_space(prev: char, next: char) -> bool {
|
||||
(is_han_or_japanese(prev) && is_han_or_japanese(next))
|
||||
|| (is_cjk_punctuation(prev) && is_han_or_japanese(next))
|
||||
|| (is_han_or_japanese(prev) && is_cjk_punctuation(next))
|
||||
|| is_opening_punctuation(prev)
|
||||
|| is_closing_punctuation(next)
|
||||
}
|
||||
|
||||
fn is_han_or_japanese(c: char) -> bool {
|
||||
matches!(
|
||||
c as u32,
|
||||
0x3400..=0x4DBF
|
||||
| 0x4E00..=0x9FFF
|
||||
| 0xF900..=0xFAFF
|
||||
| 0x20000..=0x2FA1F
|
||||
| 0x3040..=0x30FF
|
||||
| 0x31F0..=0x31FF
|
||||
| 0xFF66..=0xFF9D
|
||||
)
|
||||
}
|
||||
|
||||
fn is_cjk_punctuation(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'、' | '。'
|
||||
| '，'
|
||||
| '！'
|
||||
| '？'
|
||||
| '：'
|
||||
| '；'
|
||||
| '「'
|
||||
| '」'
|
||||
| '『'
|
||||
| '』'
|
||||
| '【'
|
||||
| '】'
|
||||
| '《'
|
||||
| '》'
|
||||
| '〈'
|
||||
| '〉'
|
||||
| '・'
|
||||
| '〜'
|
||||
| '…'
|
||||
| '—'
|
||||
)
|
||||
}
|
||||
|
||||
fn is_opening_punctuation(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'(' | '[' | '{' | '（' | '［' | '｛' | '「' | '『' | '【' | '《' | '〈'
|
||||
)
|
||||
}
|
||||
|
||||
fn is_closing_punctuation(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
',' | '.'
|
||||
| '!'
|
||||
| '?'
|
||||
| ':'
|
||||
| ';'
|
||||
| ')'
|
||||
| ']'
|
||||
| '}'
|
||||
| '，'
|
||||
| '。'
|
||||
| '！'
|
||||
| '？'
|
||||
| '：'
|
||||
| '；'
|
||||
| '）'
|
||||
| '］'
|
||||
| '｝'
|
||||
| '、'
|
||||
| '」'
|
||||
| '』'
|
||||
| '】'
|
||||
| '》'
|
||||
| '〉'
|
||||
)
|
||||
}
|
||||
|
||||
/// 空白不敏感比较用：剔除所有空白字符。话段拼接与引擎全文重放的分隔符可能不同
|
||||
/// （我们按 CJK 规则拼、引擎按自己的习惯拼），只比内容不比空白。
|
||||
fn normalized(s: &str) -> String {
|
||||
s.chars().filter(|c| !c.is_whitespace()).collect()
|
||||
}
|
||||
|
||||
fn speech_recognizer_class() -> Result<&'static AnyClass> {
|
||||
@@ -435,7 +826,10 @@ fn create_recognizer(locale: Option<&str>) -> Result<*mut AnyObject> {
|
||||
let cls = speech_recognizer_class()?;
|
||||
let recognizer: *mut AnyObject = match locale.and_then(ns_locale) {
|
||||
Some(ns_loc) => {
|
||||
log::info!("[apple-speech] recognizer locale = {}", locale.unwrap_or(""));
|
||||
log::info!(
|
||||
"[apple-speech] recognizer locale = {}",
|
||||
locale.unwrap_or("")
|
||||
);
|
||||
// SAFETY: `cls` 是 SFSpeechRecognizer 类；`alloc` 得未初始化实例，
|
||||
// `initWithLocale:` 用有效 NSLocale 初始化，返回实例移交调用方（ARC 管理）。
|
||||
unsafe {
|
||||
@@ -711,4 +1105,294 @@ mod tests {
|
||||
assert_send::<SendableTask>();
|
||||
assert_send::<Arc<Mutex<Option<SendableTask>>>>();
|
||||
}
|
||||
|
||||
// ---- SegmentAccumulator：停顿多话段累积（修「停顿后前文丢失」） ----
|
||||
|
||||
#[test]
|
||||
fn server_style_growing_partials_keep_full_final() {
|
||||
// 云端识别：partial 全程累计、final 为全文 —— 行为必须与旧实现一致。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("hello", false, false);
|
||||
acc.fold("hello there", false, false);
|
||||
acc.fold("hello there how are you", false, true);
|
||||
assert_eq!(acc.salvage(), "hello there how are you");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_device_pause_segments_are_all_kept() {
|
||||
// 用户 bug 复现：停顿产生话段边界（metadata），旧实现只留最后一段。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("今天天气", false, false);
|
||||
acc.fold("今天天气很好", true, false); // 停顿 → 话段 1 结束
|
||||
acc.fold("我们", false, false); // partial 从空重来
|
||||
acc.fold("我们去公园", false, true); // 最后话段以 isFinal 收尾
|
||||
assert_eq!(acc.salvage(), "今天天气很好我们去公园");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn per_segment_finals_are_all_kept() {
|
||||
// 个别系统按话段多次 isFinal：每个 final 都要落袋，不能见到第一个就收工。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("第一段内容", false, true);
|
||||
acc.fold("第二段内容", false, true);
|
||||
assert_eq!(acc.salvage(), "第一段内容第二段内容");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_per_segment_finals_are_distinct_utterances() {
|
||||
// 两个相邻话段内容可以完全相同；不能把第二个 final 当任务级全文重放吞掉。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("hello", false, true);
|
||||
acc.fold("hello", false, true);
|
||||
assert_eq!(acc.salvage(), "hello hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silent_reset_without_metadata_is_salvaged() {
|
||||
// 防守路径：没有 metadata 边界、partial 骤缩 → 上一话段先落袋。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("这是停顿之前说的很长一段话啊", false, false); // 14 字符
|
||||
acc.fold("后", false, false); // 骤缩 → 判定重置
|
||||
acc.fold("后半段", false, true);
|
||||
assert_eq!(acc.salvage(), "这是停顿之前说的很长一段话啊后半段");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_revision_is_not_treated_as_reset() {
|
||||
// 识别器正常的假设修正（小幅缩短）不能触发重置，否则会人为造出重复段。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("hello there my friend", false, false);
|
||||
acc.fold("hello there my frien", false, false); // 仅缩 1 字符
|
||||
acc.fold("hello there my friends", false, true);
|
||||
assert_eq!(acc.salvage(), "hello there my friends");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equal_boundary_segments_are_distinct_utterances() {
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("hello", true, false);
|
||||
acc.fold("hello", true, false);
|
||||
assert_eq!(acc.salvage(), "hello hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn longer_prefix_boundary_segment_is_not_a_cumulative_replay() {
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("好的", true, false);
|
||||
acc.fold("好的我们继续", true, false);
|
||||
assert_eq!(acc.salvage(), "好的好的我们继续");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shorter_prefix_boundary_segment_is_not_a_cumulative_replay() {
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("好的我们继续", true, false);
|
||||
acc.fold("好的", true, false);
|
||||
assert_eq!(acc.salvage(), "好的我们继续好的");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_text_replay_at_final_is_not_duplicated() {
|
||||
// 防守：逐话段落袋之后，final 若重放「累计全文」（分隔符可能与我们不同），
|
||||
// 空白不敏感去重必须把它忽略，不得把全文再拼一遍。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("今天天气很好", true, false);
|
||||
acc.fold("我们去公园", true, false);
|
||||
acc.fold("今天天气很好 我们去公园", false, true);
|
||||
assert_eq!(acc.salvage(), "今天天气很好我们去公园");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_boundary_text_falls_back_to_partial() {
|
||||
// 边界结果偶见空文本：兜底用当前话段已见的最长 partial，不丢内容。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("前半句", false, false);
|
||||
acc.fold("", true, false);
|
||||
acc.fold("后半句", false, true);
|
||||
assert_eq!(acc.salvage(), "前半句后半句");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn salvage_includes_residual_partial() {
|
||||
// 错误兜底路径：final 没等到，也要把已见 partial 抢救回来。
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("说到一半", false, false);
|
||||
assert_eq!(acc.salvage(), "说到一半");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_segments_join_with_space_cjk_join_bare() {
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold("first part", true, false);
|
||||
acc.fold("second part", true, false);
|
||||
assert_eq!(acc.salvage(), "first part second part");
|
||||
|
||||
let mut mixed = SegmentAccumulator::default();
|
||||
mixed.fold("中文段落", true, false);
|
||||
mixed.fold("english tail", true, false);
|
||||
assert_eq!(mixed.salvage(), "中文段落 english tail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_cjk_non_ascii_segments_keep_word_spaces() {
|
||||
for (first, second, expected) in [
|
||||
("привет", "мир", "привет мир"),
|
||||
("مرحبا", "بالعالم", "مرحبا بالعالم"),
|
||||
("안녕", "하세요", "안녕 하세요"),
|
||||
] {
|
||||
let mut acc = SegmentAccumulator::default();
|
||||
acc.fold(first, true, false);
|
||||
acc.fold(second, true, false);
|
||||
assert_eq!(acc.salvage(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chinese_and_japanese_scripts_join_without_spaces_around_native_punctuation() {
|
||||
let mut chinese = SegmentAccumulator::default();
|
||||
chinese.fold("你好，", true, false);
|
||||
chinese.fold("我们继续", true, false);
|
||||
assert_eq!(chinese.salvage(), "你好，我们继续");
|
||||
|
||||
let mut japanese = SegmentAccumulator::default();
|
||||
japanese.fold("今日は", true, false);
|
||||
japanese.fold("晴れです。", true, false);
|
||||
assert_eq!(japanese.salvage(), "今日は晴れです。");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognition_wait_budget_scales_with_audio_length() {
|
||||
// 短音频维持 60s 下限；长音频按时长 + 30s 放大，不再被固定硬顶截断。
|
||||
assert_eq!(recognition_wait_budget(5_000), RECOGNITION_WAIT);
|
||||
assert_eq!(recognition_wait_budget(300_000), Duration::from_secs(330));
|
||||
}
|
||||
|
||||
// ---- RecognitionLifecycle：完成 / 迟到回调 / 静默与终止优先级 ----
|
||||
|
||||
#[test]
|
||||
fn completed_task_waits_for_the_full_grace_period() {
|
||||
let start = Instant::now();
|
||||
let mut lifecycle = RecognitionLifecycle::default();
|
||||
lifecycle.record_completed(start);
|
||||
|
||||
assert_eq!(
|
||||
lifecycle.decide(
|
||||
start + COMPLETION_GRACE - Duration::from_millis(1),
|
||||
false,
|
||||
false,
|
||||
false
|
||||
),
|
||||
RecognitionDecision::Wait
|
||||
);
|
||||
assert_eq!(
|
||||
lifecycle.decide(start + COMPLETION_GRACE, false, false, false),
|
||||
RecognitionDecision::Finish
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn callback_after_completed_restarts_the_grace_window() {
|
||||
let start = Instant::now();
|
||||
let late = start + Duration::from_millis(200);
|
||||
let mut lifecycle = RecognitionLifecycle::default();
|
||||
lifecycle.record_completed(start);
|
||||
lifecycle.record_callback(late, false);
|
||||
|
||||
assert_eq!(
|
||||
lifecycle.decide(
|
||||
late + COMPLETION_GRACE - Duration::from_millis(1),
|
||||
false,
|
||||
false,
|
||||
false
|
||||
),
|
||||
RecognitionDecision::Wait
|
||||
);
|
||||
assert_eq!(
|
||||
lifecycle.decide(late + COMPLETION_GRACE, false, false, false),
|
||||
RecognitionDecision::Finish
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_callback_without_completed_uses_silent_fallback() {
|
||||
let start = Instant::now();
|
||||
let mut lifecycle = RecognitionLifecycle::default();
|
||||
lifecycle.record_callback(start, true);
|
||||
|
||||
assert_eq!(
|
||||
lifecycle.decide(
|
||||
start + FINAL_QUIESCENCE - Duration::from_millis(1),
|
||||
false,
|
||||
false,
|
||||
false
|
||||
),
|
||||
RecognitionDecision::Wait
|
||||
);
|
||||
assert_eq!(
|
||||
lifecycle.decide(start + FINAL_QUIESCENCE, false, false, false),
|
||||
RecognitionDecision::Finish
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_wins_over_error_timeout_and_completion() {
|
||||
let start = Instant::now();
|
||||
let mut lifecycle = RecognitionLifecycle::default();
|
||||
lifecycle.record_completed(start);
|
||||
assert_eq!(
|
||||
lifecycle.decide(start + COMPLETION_GRACE, true, true, true),
|
||||
RecognitionDecision::Cancel
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_wins_over_timeout_and_completion_when_not_cancelled() {
|
||||
let start = Instant::now();
|
||||
let mut lifecycle = RecognitionLifecycle::default();
|
||||
lifecycle.record_completed(start);
|
||||
assert_eq!(
|
||||
lifecycle.decide(start + COMPLETION_GRACE, false, true, true),
|
||||
RecognitionDecision::Error
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn result_and_error_in_one_callback_salvages_the_result_once() {
|
||||
let start = Instant::now();
|
||||
let mut shared = RecognitionShared::default();
|
||||
shared.record_callback(
|
||||
Some(RecognizedCallback {
|
||||
text: "已经识别的内容".to_string(),
|
||||
utterance_ended: false,
|
||||
is_final: true,
|
||||
}),
|
||||
Some("尾部错误".to_string()),
|
||||
start,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
shared
|
||||
.lifecycle
|
||||
.decide(start, false, shared.error.is_some(), false),
|
||||
RecognitionDecision::Error
|
||||
);
|
||||
assert_eq!(shared.acc.salvage(), "已经识别的内容");
|
||||
assert_eq!(shared.acc.salvage(), "已经识别的内容");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settled_completion_wins_over_timeout_but_timeout_ends_plain_waiting() {
|
||||
let start = Instant::now();
|
||||
let mut completed = RecognitionLifecycle::default();
|
||||
completed.record_completed(start);
|
||||
assert_eq!(
|
||||
completed.decide(start + COMPLETION_GRACE, false, false, true),
|
||||
RecognitionDecision::Finish
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
RecognitionLifecycle::default().decide(start, false, false, true),
|
||||
RecognitionDecision::Timeout
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user