fix(history): 支持已完成条目重新转录 (#1079)

* fix(history): 支持已完成条目重新转录

Co-authored-by: Cursor <cursoragent@cursor.com>

* docs: 更新 Issue 1046 交付记录

Co-authored-by: Cursor <cursoragent@cursor.com>

* docs: 记录上游 Issue 1046 PR

* fix(history): preserve records and split long Qwen audio

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Chris233 <h-chris233@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-16 22:22:48 +08:00
1 parent 9b55ae1a8c
commit cbde1fc267
7 files changed
+276 -74

No files matched your search

@@ -4276,6 +4276,15 @@ impl OpenLessBackend {
.ok_or_else(|| {
BackendError::new(BackendErrorCode::InvalidArgument, "history entry not found")
})?;
if !matches!(
entry.error_code.as_deref(),
Some("transcribeFailed" | "emptyTranscript")
) {
return Err(BackendError::new(
BackendErrorCode::InvalidState,
"history entry is not a failed transcription",
));
}
entry.raw_transcript = text.clone();
entry.final_text = text;
entry.error_code = None;
@@ -7589,21 +7598,35 @@ mod tests {
let mut entry = history_session("one");
backend.append_history(entry.clone(), 30, Some(20)).unwrap();
let asr_call = crate::auxiliary::AsrCallLabel {
provider: "channel-b".into(),
model: Some("model-b".into()),
};
let completed_error = backend
.apply_history_retranscription(
&entry.id,
"must not replace history".into(),
&asr_call,
1,
)
.unwrap_err();
assert_eq!(completed_error.code, BackendErrorCode::InvalidState);
entry.final_text = "updated".to_string();
entry.error_code = Some("polishFailed".to_string());
assert!(backend.update_history_entry(entry.clone()).unwrap());
let polish_error = backend
.apply_history_retranscription(&entry.id, "must remain a preview".into(), &asr_call, 1)
.unwrap_err();
assert_eq!(polish_error.code, BackendErrorCode::InvalidState);
entry.error_code = Some("transcribeFailed".to_string());
assert!(backend.update_history_entry(entry.clone()).unwrap());
assert!(!backend
.update_history_entry(history_session("missing"))
.unwrap());
let retranscribed = backend
.apply_history_retranscription(
&entry.id,
"retranscribed".into(),
&crate::auxiliary::AsrCallLabel {
provider: "channel-b".into(),
model: Some("model-b".into()),
},
480,
)
.apply_history_retranscription(&entry.id, "retranscribed".into(), &asr_call, 480)
.unwrap();
assert_eq!(retranscribed.raw_transcript, "retranscribed");
assert_eq!(retranscribed.final_text, "retranscribed");
@@ -7620,8 +7643,8 @@ mod tests {
assert!(backend.list_history().unwrap().is_empty());
assert_eq!(backend.list_activity().unwrap()[0].chars, 42);
assert_eq!(backend.snapshot().history_revision, 6);
for expected_revision in 1..=6 {
assert_eq!(backend.snapshot().history_revision, 7);
for expected_revision in 1..=7 {
assert_eq!(
events.try_recv().unwrap().kind,
BackendEventKind::HistoryChanged(HistoryChange {
@@ -22,6 +22,9 @@ use crate::asr::local::sherpa::{
SherpaPreparePhase, SherpaPrepareProgressPayload, SherpaRuntimeStatus, PROVIDER_ID,
};
const QWEN3_ASR_CHUNK_LIMIT_MS: u64 = 30_000;
const QWEN3_ASR_MAX_NEW_TOKENS: i32 = 256;
#[cfg(target_os = "windows")]
use sherpa_onnx::{
OfflineParaformerModelConfig, OfflineQwen3ASRModelConfig, OfflineRecognizer,
@@ -225,12 +228,16 @@ impl SherpaOnnxRuntime {
if pcm.is_empty() {
return Ok(String::new());
}
if sherpa_target(alias)?.sherpa_execution_mode()
!= Some(openless_core::LocalAsrExecutionMode::Offline)
{
let target = sherpa_target(alias)?;
if target.sherpa_execution_mode() != Some(openless_core::LocalAsrExecutionMode::Offline) {
anyhow::bail!("sherpa-onnx model {alias} is online-only; use streaming API");
}
let audio_ms = pcm_duration_ms(pcm);
let pcm_chunks = sherpa_offline_chunks(&target, pcm)
.into_iter()
.map(|chunk| chunk.to_vec())
.collect::<Vec<_>>();
let chunk_count = pcm_chunks.len();
let loaded_alias = self.ensure_loaded(alias, model_dir).await?;
let loaded = self
.state
@@ -242,7 +249,7 @@ impl SherpaOnnxRuntime {
let started = Instant::now();
let result = transcribe_loaded_model(
loaded,
pcm.to_vec(),
pcm_chunks,
language_hint.map(str::to_string),
audio_timeout,
)
@@ -251,9 +258,10 @@ impl SherpaOnnxRuntime {
match &result {
Ok(text) => {
log::info!(
"[sherpa-asr] transcribe finished model={} audio_ms={} elapsed_ms={} text_chars={}",
"[sherpa-asr] transcribe finished model={} audio_ms={} chunks={} elapsed_ms={} text_chars={}",
alias,
audio_ms,
chunk_count,
elapsed_ms,
text.chars().count()
);
@@ -262,9 +270,10 @@ impl SherpaOnnxRuntime {
Err(error) => {
let message = format!("{error:#}");
log::warn!(
"[sherpa-asr] transcribe failed model={} audio_ms={} elapsed_ms={} error={}",
"[sherpa-asr] transcribe failed model={} audio_ms={} chunks={} elapsed_ms={} error={}",
alias,
audio_ms,
chunk_count,
elapsed_ms,
message
);
@@ -426,6 +435,15 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
crate::asr::pcm::pcm_duration_ms(pcm)
}
fn sherpa_offline_chunks<'a>(
target: &openless_core::LocalAsrTarget,
pcm: &'a [u8],
) -> Vec<&'a [u8]> {
let limit = (target.sherpa_family() == Some(openless_core::SherpaModelFamily::Qwen3Asr))
.then_some(QWEN3_ASR_CHUNK_LIMIT_MS);
openless_core::asr::whisper::split_pcm_by_duration(pcm, limit)
}
enum LoadedModel {
Offline(LoadedOfflineModel),
Online(LoadedOnlineModel),
@@ -515,6 +533,7 @@ fn create_offline_recognizer(alias: &str, dir: &Path) -> Result<OfflineRecognize
encoder: Some(path_to_string(&dir.join("encoder.int8.onnx"))?),
decoder: Some(path_to_string(&dir.join("decoder.int8.onnx"))?),
tokenizer: Some(path_to_string(&dir.join("tokenizer"))?),
max_new_tokens: QWEN3_ASR_MAX_NEW_TOKENS,
..Default::default()
};
config.model_config.num_threads = 3;
@@ -569,25 +588,29 @@ fn path_to_string(path: &Path) -> Result<String> {
#[cfg(target_os = "windows")]
async fn transcribe_loaded_model(
loaded: LoadedOfflineModel,
pcm: Vec<u8>,
pcm_chunks: Vec<Vec<u8>>,
language_hint: Option<String>,
audio_timeout: std::time::Duration,
) -> Result<String> {
tokio::time::timeout(audio_timeout, async move {
tokio::task::spawn_blocking(move || {
let samples = pcm_s16le_to_f32(&pcm)?;
let stream = loaded.recognizer.create_stream();
if let Some(language) = language_hint.as_deref().filter(|value| !value.is_empty()) {
if stream.has_option("language") {
stream.set_option("language", language);
let mut texts = Vec::with_capacity(pcm_chunks.len());
for pcm in pcm_chunks {
let samples = pcm_s16le_to_f32(&pcm)?;
let stream = loaded.recognizer.create_stream();
if let Some(language) = language_hint.as_deref().filter(|value| !value.is_empty()) {
if stream.has_option("language") {
stream.set_option("language", language);
}
}
stream.accept_waveform(16_000, &samples);
loaded.recognizer.decode(&stream);
let result = stream
.get_result()
.ok_or_else(|| anyhow::anyhow!("sherpa-onnx returned no result"))?;
texts.push(result.text);
}
stream.accept_waveform(16_000, &samples);
loaded.recognizer.decode(&stream);
let result = stream
.get_result()
.ok_or_else(|| anyhow::anyhow!("sherpa-onnx returned no result"))?;
Ok(result.text)
Ok(openless_core::asr::whisper::join_transcript_chunks(&texts))
})
.await
.map_err(|e| anyhow::anyhow!("sherpa-onnx transcribe join failed: {e:#}"))?
@@ -599,7 +622,7 @@ async fn transcribe_loaded_model(
#[cfg(not(target_os = "windows"))]
async fn transcribe_loaded_model(
_loaded: LoadedOfflineModel,
_pcm: Vec<u8>,
_pcm_chunks: Vec<Vec<u8>>,
_language_hint: Option<String>,
_audio_timeout: std::time::Duration,
) -> Result<String> {
@@ -936,6 +959,29 @@ mod tests {
assert!(format!("{:#}", result.unwrap_err()).contains("online-only"));
}
#[test]
fn only_qwen3_offline_audio_is_split_at_thirty_seconds() {
let qwen = sherpa_target("qwen3-asr-0.6b-int8").unwrap();
let thirty_seconds = vec![0; 32_000 * 30];
assert_eq!(
sherpa_offline_chunks(&qwen, &thirty_seconds),
vec![thirty_seconds.as_slice()]
);
let pcm = vec![0; 32_000 * 65];
let chunks = sherpa_offline_chunks(&qwen, &pcm);
assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].len(), 32_000 * 30);
assert_eq!(chunks[1].len(), 32_000 * 30);
assert_eq!(chunks[2].len(), 32_000 * 5);
let sense_voice = sherpa_target("sense-voice-small-zh").unwrap();
assert_eq!(
sherpa_offline_chunks(&sense_voice, &pcm),
vec![pcm.as_slice()]
);
}
#[tokio::test]
async fn create_online_session_rejects_offline_model_alias() {
let runtime = SherpaOnnxRuntime::new();
@@ -203,22 +203,44 @@ fn copy_recording_to_mobile_url(
Ok(())
}
/// 对一条「转录失败」历史条目的归档录音用**当前** ASR provider 重新转录(issue #613)。
#[derive(serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HistoryRetranscriptionResult {
text: String,
updated_entry: Option<DictationSession>,
}
fn should_replace_failed_history(error_code: Option<&str>) -> bool {
matches!(error_code, Some("transcribeFailed" | "emptyTranscript"))
}
/// 对一条有归档录音的历史条目用**当前** ASR provider 重新转录(issue #613 / #1046)。
///
/// 流程:读 `recordings/<id>.wav` → 取 PCM(跳过 44 字节 WAV 头)→ 现 provider 重转
/// → 成功则原地回写该条历史的 rawTranscript / finalText、清除 error_code,返回新文本。
/// → 失败记录原地修复;已完成 / 润色失败记录仅返回临时结果,不覆盖历史事实。
///
/// 仅重新转写音频,不调用 LLM 润色。失败时
/// 不动历史、不删录音,把错误返回给前端提示,用户可重试。返回更新后的整条记录给前端
/// 局部刷新。
/// 不动历史、不删录音,把错误返回给前端提示,用户可重试。
#[tauri::command]
pub async fn retranscribe_recording(
core: CoreState<'_>,
session_id: String,
) -> Result<DictationSession, String> {
) -> Result<HistoryRetranscriptionResult, String> {
if !is_valid_session_id(&session_id) {
return Err("invalid session id".into());
}
let entry = core
.list_history()
.map_err(|e| e.to_string())?
.into_iter()
.find(|entry| entry.id == session_id)
.ok_or_else(|| "history entry not found".to_string())?;
if entry.has_audio_recording != Some(true) {
return Err("history entry has no archived recording".into());
}
if entry.pipeline_mode.as_deref() == Some("multimodal") {
return Err("multimodal history does not support retranscription".into());
}
let path =
crate::persistence::recording_path_for_session(&session_id).map_err(|e| e.to_string())?;
let wav = tokio::fs::read(&path).await.map_err(|e| {
@@ -248,6 +270,34 @@ pub async fn retranscribe_recording(
}
let retranscribe_ms = retranscribe_started.elapsed().as_millis() as u64;
core.apply_history_retranscription(&session_id, text, &asr_call_label, retranscribe_ms)
.map_err(|e| e.to_string())
let updated_entry = if should_replace_failed_history(entry.error_code.as_deref()) {
Some(
core.apply_history_retranscription(
&session_id,
text.clone(),
&asr_call_label,
retranscribe_ms,
)
.map_err(|e| e.to_string())?,
)
} else {
None
};
Ok(HistoryRetranscriptionResult {
text,
updated_entry,
})
}
#[cfg(test)]
mod retranscription_tests {
use super::should_replace_failed_history;
#[test]
fn only_failed_transcriptions_are_replaced() {
assert!(should_replace_failed_history(Some("transcribeFailed")));
assert!(should_replace_failed_history(Some("emptyTranscript")));
assert!(!should_replace_failed_history(Some("polishFailed")));
assert!(!should_replace_failed_history(None));
}
}
@@ -0,0 +1,48 @@
import { canRetranscribeHistoryEntry } from './history-retranscribe';
function assert(condition: boolean, message: string) {
if (!condition) throw new Error(message);
}
const archivedEntry = {
hasAudioRecording: true,
pipelineMode: 'traditional',
} as const;
const entryWithError = (errorCode: string | null) => ({ ...archivedEntry, errorCode });
assert(
canRetranscribeHistoryEntry(entryWithError(null)),
'completed entries with an archived recording should be retranscribable',
);
assert(
canRetranscribeHistoryEntry(entryWithError('polishFailed')),
'entries whose polishing failed should still be retranscribable',
);
assert(
canRetranscribeHistoryEntry(entryWithError('transcribeFailed')),
'entries whose transcription failed should be retranscribable',
);
assert(
!canRetranscribeHistoryEntry({
hasAudioRecording: false,
pipelineMode: 'traditional',
}),
'entries without an archived recording should not show retranscription',
);
assert(
!canRetranscribeHistoryEntry({
hasAudioRecording: null,
pipelineMode: undefined,
}),
'legacy entries without recording metadata should not show retranscription',
);
assert(
!canRetranscribeHistoryEntry({
...archivedEntry,
pipelineMode: 'multimodal',
}),
'multimodal entries should not show an unsupported retranscription action',
);
console.log('history-retranscribe: all assertions passed');
@@ -0,0 +1,14 @@
import type { DictationSession } from './types';
/**
* 重新转录需要一份仍存在的 WAV 归档。多模态历史目前没有对应的重转录
* provider 通道,避免展示一个点击后必然失败的按钮。
*
* 成功转录、润色失败和转录失败的条目都可能有可用录音,用户都应能用同一份
* 音频重新验证当前 ASR provider。是否回写失败记录由后端决定。
*/
export function canRetranscribeHistoryEntry(
session: Pick<DictationSession, 'hasAudioRecording' | 'pipelineMode'>,
): boolean {
return session.hasAudioRecording === true && session.pipelineMode !== 'multimodal';
}
+14 -9
View File
@@ -26,13 +26,18 @@ export function readAudioRecording(sessionId: string): Promise<string> {
return invokeOrMock('read_audio_recording', { sessionId }, () => 'data:audio/wav;base64,');
}
/** 用当前 ASR provider 对一条「转录失败」历史条目的归档录音重新转录(issue #613)。
* 成功时后端原地回写该条历史的 rawTranscript / finalText 并清除错误码,返回更新后的整条记录。
* 失败时抛出错误(如「重新转录仍未识别到语音」/「recording not found」),录音保留不丢。 */
export function retranscribeRecording(sessionId: string): Promise<DictationSession> {
return invokeOrMock(
'retranscribe_recording',
{ sessionId },
() => mockHistory[0],
) as Promise<DictationSession>;
export interface HistoryRetranscriptionResult {
text: string;
updatedEntry: DictationSession | null;
}
/** 用当前 ASR provider 对一条有归档录音的历史条目重新转录(issue #613 / #1046)。
* 转录失败记录会被修复;已完成 / 润色失败记录只返回临时结果,不覆盖原历史。
* 失败时抛出错误(如「重新转录仍未识别到语音」/「recording not found」),录音保留不丢。
* 成功、润色失败和转录失败的条目都可调用,后端再次校验录音与能力边界。 */
export function retranscribeRecording(sessionId: string): Promise<HistoryRetranscriptionResult> {
return invokeOrMock('retranscribe_recording', { sessionId }, () => ({
text: mockHistory[0].rawTranscript,
updatedEntry: null,
})) as Promise<HistoryRetranscriptionResult>;
}
+44 -28
View File
@@ -22,6 +22,7 @@ import {
packDisplayName,
resolveRepolishRetryPackIdWithFallback,
} from '../lib/history-repolish';
import { canRetranscribeHistoryEntry } from '../lib/history-retranscribe';
import { useMobileLayout } from '../lib/useMobileLayout';
import type { DictationSession, PolishMode, StylePack } from '../lib/types';
import { countCodePoints } from '../lib/unicode';
@@ -83,6 +84,10 @@ export function History() {
const [justCopiedRaw, setJustCopiedRaw] = useState(false);
// 「重新转录」进行中:禁用按钮 + 显示「转录中…」,避免重复点击发起多次 ASR。
const [retranscribing, setRetranscribing] = useState(false);
const [retranscriptionResult, setRetranscriptionResult] = useState<{
sessionId: string;
text: string;
} | null>(null);
// 录音文件 lazily-detected missing 状态:retention / 条数 cap 清理后磁盘上 wav
// 可能已被删,但 history 条目 hasAudioRecording 仍写 true。任一组件
// (播放 / 导出)首次 IPC 拿到 'recording not found' 时把 id 加进来,
@@ -300,23 +305,30 @@ export function History() {
}
};
// 对一条「转录失败 / 没识别到语音」的历史用当前 ASR provider 重新转录(issue #613)。
// 后端读 recordings/<id>.wav → 重转 → 原地回写该条 rawTranscript/finalText、清 errorCode,
// 返回整条记录;前端据此局部刷新。失败保留 + 自动重试已让这些条目的录音留得住,这里给
// 持久失败(重试也没救回来)一个手动重转入口。
// 失败记录沿用 #613 的原地修复;已经插入过文字的完成 / 润色失败记录只显示临时结果,
// 避免把事后重转文本伪装成当时实际插入的历史事实。
const onRetranscribe = async () => {
if (!item || !item.hasAudioRecording) return;
if (!item || !canRetranscribeHistoryEntry(item)) return;
const sessionId = item.id;
setRetranscribing(true);
setRetranscriptionResult(null);
setActionError(null);
try {
const updated = await retranscribeRecording(item.id);
setItems((prev) => prev.map((s) => (s.id === updated.id ? updated : s)));
const result = await retranscribeRecording(sessionId);
if (result.updatedEntry) {
const updatedEntry = result.updatedEntry;
setItems((prev) =>
prev.map((entry) => (entry.id === updatedEntry.id ? updatedEntry : entry)),
);
} else {
setRetranscriptionResult({ sessionId, text: result.text });
}
} catch (error) {
console.error('[history] retranscribe failed', error);
const msg = errorMessage(error);
// wav 已被 retention / 条数 cap 清理:隐藏入口,不报错(用户没干错事)。
if (msg.includes('recording not found') || msg.includes('not found')) {
markAudioMissing(item.id);
markAudioMissing(sessionId);
return;
}
setActionError(t('history.retranscribeFailed', { err: msg }));
@@ -604,21 +616,17 @@ export function History() {
{t('history.exportRecording')}
</Btn>
)}
{item.hasAudioRecording &&
!audioMissingIds.has(item.id) &&
item.pipelineMode !== 'multimodal' &&
(item.errorCode === 'transcribeFailed' ||
item.errorCode === 'emptyTranscript') && (
<Btn
icon="refresh"
variant="ghost"
size="sm"
disabled={retranscribing}
onClick={() => void onRetranscribe()}
>
{retranscribing ? t('history.retranscribing') : t('history.retranscribe')}
</Btn>
)}
{canRetranscribeHistoryEntry(item) && !audioMissingIds.has(item.id) && (
<Btn
icon="refresh"
variant="ghost"
size="sm"
disabled={retranscribing}
onClick={() => void onRetranscribe()}
>
{retranscribing ? t('history.retranscribing') : t('history.retranscribe')}
</Btn>
)}
<Btn icon="trash" variant="ghost" size="sm" onClick={onDelete}>
{t('common.delete')}
</Btn>
@@ -635,6 +643,14 @@ export function History() {
key={`audio-${item.id}`}
/>
)}
{retranscriptionResult?.sessionId === item.id && (
<div style={{ marginBottom: 14 }}>
<HistoryResultCard
title={t('history.retranscribe')}
text={retranscriptionResult.text}
/>
</div>
)}
{/* 流水线明细:识别 / 润色 / 插入 三步各占一行 —— 左列步骤名、中列
provider·model(或插入目标),右列该步耗时/状态。旧历史没有模型与
耗时字段时对应行自动隐藏,只剩插入行 = 改版前的信息量。 */}
@@ -912,8 +928,8 @@ interface RepolishResult {
* 结果只在本次查看时显示,不写回历史条目:历史的 finalText 是「当时真的插进去的那段
* 文字」,是一条事实记录,不该被事后试算覆盖。面板顶部的说明也把这点直说了。
*
* 注意这里只重跑润色,不重跑识别 —— 成功听写的录音在插入后就删了(隐私设计),
* 原文是唯一还在的输入。真正的「重新转录」入口仍只对留有录音的失败条目开放。
* 注意这里只重跑润色,不重跑识别 —— 没有归档录音的历史只能使用原文。
* 「重新转录」入口对所有仍留有录音的传统 ASR 条目开放。
*/
function RepolishPanel({
session,
@@ -1101,7 +1117,7 @@ function RepolishPanel({
{results.length > 0 && (
<div style={{ display: 'grid', gridTemplateColumns: mobile ? '1fr' : '1fr 1fr', gap: 12 }}>
{results.map((result) => (
<RepolishResultCard key={result.key} title={result.title} text={result.text} />
<HistoryResultCard key={result.key} title={result.title} text={result.text} />
))}
</div>
)}
@@ -1109,7 +1125,7 @@ function RepolishPanel({
);
}
function RepolishResultCard({ title, text }: { title: string; text: string }) {
function HistoryResultCard({ title, text }: { title: string; text: string }) {
const { t } = useTranslation();
const [copied, setCopied] = useState(false);
@@ -1120,7 +1136,7 @@ function RepolishResultCard({ title, text }: { title: string; text: string }) {
setCopied(true);
window.setTimeout(() => setCopied(false), 1500);
} catch (error) {
console.error('[history] failed to copy repolish result', error);
console.error('[history] failed to copy result', error);
}
};