chore: 整合后端更新与文档,准备 0.0.4 发布
- 后端: ASR 性能优化与热词幻觉修复 (asr.py / main.py),调整 config.example.yaml vllm_max_model_len - 补充 docs AI/ASR 服务提供商文档 - 同步客户端原生层(src-tauri)、overlay、recorder 等既有改动 - .gitignore 忽略 we-media/ 运营资料
This commit is contained in:
1 parent
be137722ca
commit
a7e2dd6590
19 files changed
+691
-106
No files matched your search
@@ -7,6 +7,8 @@ config.yaml
|
||||
# ── Development-only (not for open source) ──
|
||||
dev-docs/
|
||||
dev-scripts/
|
||||
dev-notes/
|
||||
we-media/
|
||||
sayit说话波形图效果.html
|
||||
|
||||
# ── Server dev artifacts ──
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
# SayIt
|
||||
|
||||
**开口即成稿 — 用说话代替打字,AI 实时把口语变成书面表达。**
|
||||
**随口说,出色写 — 用说话代替打字,AI 实时把口语变成书面表达。**
|
||||
|
||||
按下快捷键开始说话,再按一次,润色后的文字自动输入到光标位置。
|
||||
|
||||
|
||||
@@ -23,3 +23,10 @@ pub async fn update_overlay_state(data: Value, app: AppHandle, window_state: Sta
|
||||
window_state.update_overlay_state(&app, &data);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reported by the overlay webview in response to an `overlay-ping` event.
|
||||
/// Used to detect WebView2 unresponsiveness.
|
||||
#[tauri::command]
|
||||
pub fn overlay_pong(seq: u64) {
|
||||
crate::window::record_overlay_pong(seq);
|
||||
}
|
||||
@@ -302,6 +302,7 @@ fn main() {
|
||||
commands::window::show_overlay,
|
||||
commands::window::hide_overlay,
|
||||
commands::window::update_overlay_state,
|
||||
commands::window::overlay_pong,
|
||||
// Paste / Context
|
||||
commands::paste::paste_text,
|
||||
commands::paste::get_probe_result,
|
||||
|
||||
@@ -1,12 +1,30 @@
|
||||
use serde_json::Value;
|
||||
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
|
||||
use std::sync::Mutex;
|
||||
use tauri::{AppHandle, Emitter, Manager, WebviewUrl, WebviewWindowBuilder};
|
||||
|
||||
use crate::commands::system::write_log_line;
|
||||
|
||||
const OVERLAY_DEFAULT_BASE_WIDTH: f64 = 360.0;
|
||||
const OVERLAY_BASE_HEIGHT: f64 = 56.0;
|
||||
const OVERLAY_FALLBACK_WIDTH: f64 = 520.0;
|
||||
const OVERLAY_FALLBACK_HEIGHT: f64 = 224.0;
|
||||
|
||||
/// Time of last received pong from overlay (epoch millis). 0 = never.
|
||||
/// Used to detect WebView2 unresponsiveness in `show_overlay`.
|
||||
static LAST_PONG_MS: AtomicI64 = AtomicI64::new(0);
|
||||
/// Monotonic id for ping/pong correlation
|
||||
static PING_SEQ: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub fn record_overlay_pong(_seq: u64) {
|
||||
let now = chrono::Utc::now().timestamp_millis();
|
||||
LAST_PONG_MS.store(now, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn now_ms() -> i64 {
|
||||
chrono::Utc::now().timestamp_millis()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum OverlayLayout {
|
||||
Base,
|
||||
@@ -31,25 +49,93 @@ impl WindowState {
|
||||
let is_fallback = layout == OverlayLayout::Fallback;
|
||||
let base_w = *self.overlay_base_width.lock().unwrap();
|
||||
|
||||
let bounds = calc_overlay_bounds(app, is_fallback, base_w);
|
||||
let (mon_w, mon_h, mon_scale) = monitor_info(app);
|
||||
|
||||
if let Some(overlay) = app.get_webview_window("overlay") {
|
||||
let bounds = calc_overlay_bounds(app, is_fallback, base_w);
|
||||
let _ = overlay.set_position(tauri::Position::Logical(tauri::LogicalPosition::new(bounds.0, bounds.1)));
|
||||
let _ = overlay.set_size(tauri::Size::Logical(tauri::LogicalSize::new(bounds.2, bounds.3)));
|
||||
let _ = overlay.show();
|
||||
let _ = overlay.set_always_on_top(true);
|
||||
// ── Diagnostic snapshot before any state change ──
|
||||
let was_visible = overlay.is_visible().unwrap_or(false);
|
||||
let outer_pos = overlay.outer_position()
|
||||
.map(|p| format!("{},{}", p.x, p.y))
|
||||
.unwrap_or_else(|_| "err".to_string());
|
||||
let outer_size = overlay.outer_size()
|
||||
.map(|s| format!("{}x{}", s.width, s.height))
|
||||
.unwrap_or_else(|_| "err".to_string());
|
||||
|
||||
// Probe webview responsiveness — send a ping; pong arrival is logged async.
|
||||
let seq = PING_SEQ.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
let last_pong = LAST_PONG_MS.load(Ordering::SeqCst);
|
||||
let pong_age_ms = if last_pong == 0 { -1 } else { now_ms() - last_pong };
|
||||
|
||||
let ping_emit_err = overlay.emit("overlay-ping", seq).err()
|
||||
.map(|e| format!("{:?}", e));
|
||||
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] show begin handle=present layout={:?} fallback={} was_visible={} \
|
||||
outer_pos={} outer_size={} target_bounds={:?} monitor={}x{}@{:.2} \
|
||||
ping_seq={} last_pong_age_ms={} ping_emit_err={:?}",
|
||||
layout, is_fallback, was_visible,
|
||||
outer_pos, outer_size, bounds, mon_w, mon_h, mon_scale,
|
||||
seq, pong_age_ms, ping_emit_err,
|
||||
));
|
||||
|
||||
let pos_err = overlay.set_position(tauri::Position::Logical(
|
||||
tauri::LogicalPosition::new(bounds.0, bounds.1)
|
||||
)).err().map(|e| format!("{:?}", e));
|
||||
let size_err = overlay.set_size(tauri::Size::Logical(
|
||||
tauri::LogicalSize::new(bounds.2, bounds.3)
|
||||
)).err().map(|e| format!("{:?}", e));
|
||||
let show_err = overlay.show().err().map(|e| format!("{:?}", e));
|
||||
let aot_err = overlay.set_always_on_top(true).err().map(|e| format!("{:?}", e));
|
||||
set_overlay_interactivity(&overlay, is_fallback);
|
||||
|
||||
// Re-snapshot to confirm visibility actually changed
|
||||
let now_visible = overlay.is_visible().unwrap_or(false);
|
||||
let now_outer_pos = overlay.outer_position()
|
||||
.map(|p| format!("{},{}", p.x, p.y))
|
||||
.unwrap_or_else(|_| "err".to_string());
|
||||
|
||||
let any_err = pos_err.is_some() || size_err.is_some()
|
||||
|| show_err.is_some() || aot_err.is_some();
|
||||
let level = if any_err { "WARN" } else { "INFO" };
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] show end [{}] now_visible={} now_outer_pos={} \
|
||||
pos_err={:?} size_err={:?} show_err={:?} aot_err={:?}",
|
||||
level, now_visible, now_outer_pos,
|
||||
pos_err, size_err, show_err, aot_err,
|
||||
));
|
||||
} else {
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] show begin handle=MISSING layout={:?} fallback={} \
|
||||
target_bounds={:?} monitor={}x{}@{:.2} — recreating",
|
||||
layout, is_fallback, bounds, mon_w, mon_h, mon_scale,
|
||||
));
|
||||
self.create_overlay(app, is_fallback, base_w);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hide_overlay(&self, app: &AppHandle) {
|
||||
// Reset to base layout when hiding
|
||||
*self.overlay_layout.lock().unwrap() = OverlayLayout::Base;
|
||||
let prev_layout = {
|
||||
let mut g = self.overlay_layout.lock().unwrap();
|
||||
let prev = g.clone();
|
||||
*g = OverlayLayout::Base;
|
||||
prev
|
||||
};
|
||||
|
||||
if let Some(overlay) = app.get_webview_window("overlay") {
|
||||
set_overlay_interactivity(&overlay, false);
|
||||
let _ = overlay.hide();
|
||||
let was_visible = overlay.is_visible().unwrap_or(false);
|
||||
let hide_err = overlay.hide().err().map(|e| format!("{:?}", e));
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] hide prev_layout={:?} was_visible={} hide_err={:?}",
|
||||
prev_layout, was_visible, hide_err,
|
||||
));
|
||||
} else {
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] hide handle=MISSING prev_layout={:?}",
|
||||
prev_layout,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,11 +206,21 @@ impl WindowState {
|
||||
match builder.build() {
|
||||
Ok(overlay) => {
|
||||
set_overlay_interactivity(&overlay, is_fallback);
|
||||
let _ = overlay.show();
|
||||
let show_err = overlay.show().err().map(|e| format!("{:?}", e));
|
||||
let now_visible = overlay.is_visible().unwrap_or(false);
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] create_overlay OK is_fallback={} bounds={:?} \
|
||||
show_err={:?} now_visible={}",
|
||||
is_fallback, bounds, show_err, now_visible,
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("[overlay] create_overlay: FAILED: {}", e);
|
||||
log::error!("Failed to create overlay window: {}", e);
|
||||
write_log_line(&format!(
|
||||
"[overlay-diag] create_overlay FAILED is_fallback={} bounds={:?} err={:?}",
|
||||
is_fallback, bounds, e,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -134,6 +230,17 @@ fn set_overlay_interactivity(overlay: &tauri::WebviewWindow, interactive: bool)
|
||||
let _ = overlay.set_ignore_cursor_events(!interactive);
|
||||
}
|
||||
|
||||
/// Returns (logical_width, logical_height, scale_factor) of primary monitor.
|
||||
fn monitor_info(app: &AppHandle) -> (f64, f64, f64) {
|
||||
if let Some(monitor) = app.primary_monitor().ok().flatten() {
|
||||
let size = monitor.size();
|
||||
let scale = monitor.scale_factor();
|
||||
(size.width as f64 / scale, size.height as f64 / scale, scale)
|
||||
} else {
|
||||
(1920.0, 1080.0, 1.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns (x, y, width, height)
|
||||
fn calc_overlay_bounds(app: &AppHandle, is_fallback: bool, base_w: f64) -> (f64, f64, f64, f64) {
|
||||
let (desired_w, desired_h) = if is_fallback {
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
// 外观设置页面 — 主题 + 悬浮窗样式 + 预览
|
||||
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { Card, CardContent } from '@/components/ui/card'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { themeList } from '@/themes'
|
||||
import { switchTheme, getActiveThemeId } from '@/stores/theme'
|
||||
import { getSetting, setSetting } from '@/services/store'
|
||||
import { refreshRecorderSettings } from '@/services/recorder'
|
||||
import { refreshOverlaySettings } from '@/services/recorder'
|
||||
import { OVERLAY_WIDTH_PRESETS, type OverlayWidthPreset } from '@/services/recorder/types'
|
||||
import { type OverlayWaveTheme } from './utils'
|
||||
|
||||
@@ -46,26 +46,37 @@ function getTimerColor(theme: OverlayWaveTheme): string {
|
||||
}
|
||||
|
||||
function OverlayPreview({ theme, showDuration, barCount }: { theme: OverlayWaveTheme; showDuration: boolean; barCount: number }) {
|
||||
const [bars, setBars] = useState<number[]>(Array.from({ length: barCount }, () => 3))
|
||||
|
||||
useEffect(() => {
|
||||
setBars(Array.from({ length: barCount }, () => 3))
|
||||
}, [barCount])
|
||||
const barRefs = useRef<Array<HTMLDivElement | null>>([])
|
||||
|
||||
useEffect(() => {
|
||||
const heights = new Array(barCount).fill(3)
|
||||
let running = true
|
||||
const animate = () => {
|
||||
let rafId = 0
|
||||
let lastFrame = 0
|
||||
const FRAME_INTERVAL = 1000 / 30 // 30fps 足够流畅,且不占满主线程
|
||||
|
||||
const animate = (now: number) => {
|
||||
if (!running) return
|
||||
setBars((prev) =>
|
||||
prev.map((v) => {
|
||||
if (now - lastFrame >= FRAME_INTERVAL) {
|
||||
lastFrame = now
|
||||
for (let i = 0; i < heights.length; i++) {
|
||||
const target = 3 + Math.random() * 15
|
||||
return v + (target - v) * 0.15
|
||||
}),
|
||||
)
|
||||
requestAnimationFrame(animate)
|
||||
heights[i] = heights[i] + (target - heights[i]) * 0.15
|
||||
const el = barRefs.current[i]
|
||||
if (el) {
|
||||
const h = Math.min(18, Math.max(3, heights[i]))
|
||||
el.style.height = `${h}px`
|
||||
el.style.opacity = String(0.7 + (h / 18) * 0.3)
|
||||
}
|
||||
}
|
||||
}
|
||||
rafId = requestAnimationFrame(animate)
|
||||
}
|
||||
rafId = requestAnimationFrame(animate)
|
||||
return () => {
|
||||
running = false
|
||||
cancelAnimationFrame(rafId)
|
||||
}
|
||||
requestAnimationFrame(animate)
|
||||
return () => { running = false }
|
||||
}, [barCount])
|
||||
|
||||
return (
|
||||
@@ -73,17 +84,17 @@ function OverlayPreview({ theme, showDuration, barCount }: { theme: OverlayWaveT
|
||||
{/* 1:1 还原真实悬浮窗样式 */}
|
||||
<div className="flex items-center rounded-full border border-slate-600 bg-black px-4 py-2 shadow-[0_6px_16px_rgba(0,0,0,0.35)]">
|
||||
<div className="flex items-center gap-[2px]" style={{ height: '20px' }}>
|
||||
{bars.map((height, index) => {
|
||||
const h = Math.min(18, Math.max(3, height))
|
||||
const color = getBarColor(index, bars.length, theme)
|
||||
{Array.from({ length: barCount }, (_, index) => {
|
||||
const color = getBarColor(index, barCount, theme)
|
||||
return (
|
||||
<div
|
||||
key={index}
|
||||
ref={(el) => { barRefs.current[index] = el }}
|
||||
className="w-[2.5px] rounded-full"
|
||||
style={{
|
||||
backgroundColor: color,
|
||||
height: `${h}px`,
|
||||
opacity: 0.7 + (h / 18) * 0.3,
|
||||
height: '3px',
|
||||
opacity: 0.7,
|
||||
transition: 'height 50ms ease-out, opacity 50ms ease-out',
|
||||
}}
|
||||
/>
|
||||
@@ -130,20 +141,20 @@ export default function AppearancePage() {
|
||||
const handleOverlayThemeChange = async (theme: OverlayWaveTheme) => {
|
||||
setOverlayWaveTheme(theme)
|
||||
await setSetting('overlayWaveTheme', theme)
|
||||
await refreshRecorderSettings()
|
||||
await refreshOverlaySettings()
|
||||
}
|
||||
|
||||
const handleToggleDuration = async () => {
|
||||
const next = !overlayShowDuration
|
||||
setOverlayShowDuration(next)
|
||||
await setSetting('overlayShowDuration', next)
|
||||
await refreshRecorderSettings()
|
||||
await refreshOverlaySettings()
|
||||
}
|
||||
|
||||
const handleOverlayWidthChange = async (preset: OverlayWidthPreset) => {
|
||||
setOverlayWidth(preset)
|
||||
await setSetting('overlayWidth', preset)
|
||||
await refreshRecorderSettings()
|
||||
await refreshOverlaySettings()
|
||||
}
|
||||
|
||||
return (
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { startWebviewKeyboardFallback } from '../services/webviewKeyboardFallback'
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import { listen } from '@tauri-apps/api/event'
|
||||
import { invoke } from '@tauri-apps/api/core'
|
||||
import Overlay from './Overlay'
|
||||
import '../index.css'
|
||||
|
||||
@@ -11,6 +13,13 @@ document.head.appendChild(style)
|
||||
|
||||
void startWebviewKeyboardFallback()
|
||||
|
||||
// Health check: respond to ping from main process so it can detect
|
||||
// WebView2 unresponsiveness. Bug 003 diagnostic.
|
||||
void listen<number>('overlay-ping', (event) => {
|
||||
const seq = typeof event.payload === 'number' ? event.payload : 0
|
||||
void invoke('overlay_pong', { seq }).catch(() => {})
|
||||
})
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<Overlay />
|
||||
|
||||
@@ -44,7 +44,7 @@ export default function About() {
|
||||
<span className="rounded-full bg-muted/50 px-2.5 py-0.5 text-xs text-muted-foreground">v{currentVersion}</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void shellOpen('https://github.com/crosswk/')}
|
||||
onClick={() => void shellOpen('https://github.com/crosswk/SayIt')}
|
||||
className="flex h-6 w-6 items-center justify-center rounded-full bg-muted/50 text-muted-foreground transition-colors hover:bg-foreground/15 hover:text-foreground"
|
||||
title="GitHub"
|
||||
>
|
||||
|
||||
+147
-46
@@ -21,8 +21,17 @@ import {
|
||||
import { loadAudioAsDataUrl } from '@/services/audioFileService'
|
||||
import { segmentAsrText } from '@/services/textSegmenter'
|
||||
import { applyTextReplacements } from '@/services/textReplacement'
|
||||
import { getProvider } from '@/services/transcription'
|
||||
import { reconnectProvider } from '@/services/recorder'
|
||||
import {
|
||||
BUILTIN_SET_WORDS_KEY,
|
||||
BUILTIN_SET_ACTIVE_KEY,
|
||||
CUSTOM_THEMES_KEY,
|
||||
CUSTOM_THEME_ACTIVE_KEY,
|
||||
composeHotwords,
|
||||
normalizeBuiltinSetActive,
|
||||
normalizeBuiltinSetWords,
|
||||
normalizeCustomThemeActive,
|
||||
normalizeCustomThemes,
|
||||
} from '@/services/hotwords/model'
|
||||
|
||||
const HISTORY_PAGE_SIZE = 100
|
||||
|
||||
@@ -102,50 +111,146 @@ export default function History() {
|
||||
const pcmData = bytes.slice(44)
|
||||
const chunk = pcmData.buffer.slice(pcmData.byteOffset, pcmData.byteOffset + pcmData.byteLength)
|
||||
|
||||
// Diagnostic: compute peak amplitude of the PCM data being sent
|
||||
const pcmInt16 = new Int16Array(chunk)
|
||||
let reprocessPeak = 0
|
||||
for (let i = 0; i < pcmInt16.length; i++) {
|
||||
const v = Math.abs(pcmInt16[i])
|
||||
if (v > reprocessPeak) reprocessPeak = v
|
||||
}
|
||||
const reprocessPeakNorm = reprocessPeak / 32768
|
||||
const reprocessDurSec = pcmInt16.length / 16000
|
||||
console.log('[reprocess-diag] PCM stats', {
|
||||
byteLength: chunk.byteLength,
|
||||
samples: pcmInt16.length,
|
||||
durationSec: reprocessDurSec.toFixed(2),
|
||||
peakInt16: reprocessPeak,
|
||||
peakNormalized: reprocessPeakNorm.toFixed(4),
|
||||
wouldBeSilent: reprocessPeakNorm < 0.01,
|
||||
})
|
||||
|
||||
const preset = await getActivePreset()
|
||||
const aiEnabled = await getSetting('aiEnabled', false)
|
||||
|
||||
// 使用当前 Provider 重新识别
|
||||
const provider = getProvider()
|
||||
let reprocessDone = false
|
||||
// 加载热词
|
||||
let hotwords: string[] = []
|
||||
try {
|
||||
const [rawSetWords, rawSetActive, rawCustomThemes, rawCustomThemeActive] = await Promise.all([
|
||||
getSetting(BUILTIN_SET_WORDS_KEY, {}),
|
||||
getSetting(BUILTIN_SET_ACTIVE_KEY, {}),
|
||||
getSetting(CUSTOM_THEMES_KEY, []),
|
||||
getSetting(CUSTOM_THEME_ACTIVE_KEY, {}),
|
||||
])
|
||||
const setWords = normalizeBuiltinSetWords(rawSetWords as Record<string, unknown>)
|
||||
const setActive = normalizeBuiltinSetActive(rawSetActive as Record<string, unknown>)
|
||||
const themes = normalizeCustomThemes(rawCustomThemes)
|
||||
const themeActive = normalizeCustomThemeActive(rawCustomThemeActive as Record<string, unknown>, themes)
|
||||
hotwords = composeHotwords([], setWords, setActive, themes, themeActive)
|
||||
} catch { /* ignore */ }
|
||||
|
||||
const clientMeta = await bridge.getClientRuntimeInfo().catch(() => null)
|
||||
|
||||
// 使用独立的 WebSocket 连接进行重新识别,避免干扰 RecorderOrchestrator 的全局连接。
|
||||
const { getWSUrl } = await import('@/services/runtimeConfig')
|
||||
const wsUrl = getWSUrl()
|
||||
|
||||
const result = await new Promise<{ asrText: string; llmText: string; asrMs: number; llmMs: number; durationSec: number; asrEngine?: string; asrModel?: string }>((resolve, reject) => {
|
||||
const timeout = window.setTimeout(() => {
|
||||
if (!reprocessDone) { reprocessDone = true; provider.disconnect() }
|
||||
reconnectProvider()
|
||||
try { socket.close() } catch { /* ignore */ }
|
||||
reject(new Error('重新识别超时'))
|
||||
}, 60000)
|
||||
}, 30_000) // ASR 最多 30 秒
|
||||
|
||||
void provider.connect({
|
||||
onFinal: (r) => {
|
||||
clearTimeout(timeout)
|
||||
reprocessDone = true
|
||||
provider.disconnect()
|
||||
reconnectProvider()
|
||||
resolve(r)
|
||||
},
|
||||
onError: (msg) => {
|
||||
clearTimeout(timeout)
|
||||
if (!reprocessDone) { reprocessDone = true; provider.disconnect() }
|
||||
reconnectProvider()
|
||||
reject(new Error(msg))
|
||||
},
|
||||
onDone: () => {
|
||||
// 如果没有 final 就 done 了,说明没有结果
|
||||
},
|
||||
}).then(async () => {
|
||||
const clientMeta = await bridge.getClientRuntimeInfo().catch(() => null)
|
||||
provider.start({
|
||||
systemPrompt: aiEnabled ? preset.systemPrompt : undefined,
|
||||
disableAi: !aiEnabled,
|
||||
clientMeta,
|
||||
const socket = new WebSocket(wsUrl)
|
||||
socket.binaryType = 'arraybuffer'
|
||||
|
||||
let resolved = false
|
||||
|
||||
socket.onopen = () => {
|
||||
// 发送 start
|
||||
const startMsg: Record<string, unknown> = {
|
||||
cmd: 'start',
|
||||
source: 'history_reprocess',
|
||||
})
|
||||
provider.sendAudio(chunk)
|
||||
provider.stop()
|
||||
}).catch((err) => {
|
||||
clearTimeout(timeout)
|
||||
reject(err)
|
||||
})
|
||||
disable_ai: !aiEnabled,
|
||||
}
|
||||
if (aiEnabled && preset.systemPrompt) startMsg.system_prompt = preset.systemPrompt
|
||||
if (clientMeta) {
|
||||
startMsg.client_meta = {
|
||||
user_id: clientMeta.userId,
|
||||
device_id: clientMeta.deviceId,
|
||||
hostname: clientMeta.hostname,
|
||||
client_version: clientMeta.clientVersion,
|
||||
platform: clientMeta.platform,
|
||||
os_version: clientMeta.osVersion,
|
||||
local_ip: clientMeta.localIp,
|
||||
system_locale: clientMeta.systemLocale,
|
||||
cpu_cores: clientMeta.cpuCores,
|
||||
memory_mb: clientMeta.memoryMb,
|
||||
}
|
||||
}
|
||||
if (hotwords.length > 0) startMsg.hotwords = hotwords
|
||||
socket.send(JSON.stringify(startMsg))
|
||||
|
||||
// 分片发送 PCM 数据
|
||||
const CHUNK_SIZE = 32000
|
||||
const totalBytes = chunk.byteLength
|
||||
for (let offset = 0; offset < totalBytes; offset += CHUNK_SIZE) {
|
||||
const end = Math.min(offset + CHUNK_SIZE, totalBytes)
|
||||
socket.send(chunk.slice(offset, end))
|
||||
}
|
||||
|
||||
// 发送 stop
|
||||
socket.send(JSON.stringify({ cmd: 'stop' }))
|
||||
}
|
||||
|
||||
socket.onmessage = (ev) => {
|
||||
if (typeof ev.data !== 'string') return
|
||||
try {
|
||||
const msg = JSON.parse(ev.data)
|
||||
console.log('[reprocess-diag] ws message:', msg.type, msg)
|
||||
if (msg.type === 'final') {
|
||||
resolved = true
|
||||
clearTimeout(timeout)
|
||||
socket.close()
|
||||
resolve({
|
||||
asrText: msg.asr_text || '',
|
||||
llmText: msg.llm_text || '',
|
||||
asrMs: msg.asr_ms || 0,
|
||||
llmMs: msg.llm_ms || 0,
|
||||
durationSec: Number(msg.duration_sec || 0),
|
||||
asrEngine: msg.asr_engine || undefined,
|
||||
asrModel: msg.asr_model || undefined,
|
||||
})
|
||||
} else if (msg.type === 'done' && !resolved) {
|
||||
// 没有 final 就 done 了(后端判定为静音/无结果)
|
||||
resolved = true
|
||||
clearTimeout(timeout)
|
||||
socket.close()
|
||||
resolve({ asrText: '', llmText: '', asrMs: 0, llmMs: 0, durationSec: 0 })
|
||||
} else if (msg.type === 'error') {
|
||||
resolved = true
|
||||
clearTimeout(timeout)
|
||||
socket.close()
|
||||
reject(new Error(msg.message || 'backend error'))
|
||||
}
|
||||
} catch { /* ignore parse errors */ }
|
||||
}
|
||||
|
||||
socket.onerror = () => {
|
||||
if (!resolved) {
|
||||
resolved = true
|
||||
clearTimeout(timeout)
|
||||
reject(new Error('WebSocket 连接错误'))
|
||||
}
|
||||
}
|
||||
|
||||
socket.onclose = (ev) => {
|
||||
console.log('[reprocess-diag] ws closed', { code: ev.code, reason: ev.reason, resolved })
|
||||
if (!resolved) {
|
||||
resolved = true
|
||||
clearTimeout(timeout)
|
||||
reject(new Error(`WebSocket 连接意外关闭 code=${ev.code}`))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// 极速模式下 llmText === asrText(后端未经 LLM 处理时直接复制 asrText)
|
||||
@@ -161,14 +266,10 @@ export default function History() {
|
||||
llmMs: result.llmMs,
|
||||
charCount: (result.llmText || result.asrText).length,
|
||||
isEmpty: !(result.llmText || result.asrText).trim(),
|
||||
workMode: provider.mode,
|
||||
aiProvider: provider.mode === 'server' ? 'server' : undefined,
|
||||
aiModel: provider.mode === 'server' ? undefined : undefined,
|
||||
asrProvider: provider.mode === 'local'
|
||||
? String(await getSetting('localAsr.modelId', 'sensevoice-small'))
|
||||
: provider.mode === 'cloud_api'
|
||||
? resolveAsrDisplayModel(String(await getSetting('cloudAsr.provider', '')))
|
||||
: (result.asrModel || result.asrEngine || 'server').replace(/^.*\//, ''),
|
||||
workMode: 'server',
|
||||
aiProvider: 'server',
|
||||
aiModel: undefined,
|
||||
asrProvider: (result.asrModel || result.asrEngine || 'server').replace(/^.*\//, ''),
|
||||
})
|
||||
|
||||
void loadRecords(debouncedKeyword, visibleCount, favoriteOnly)
|
||||
|
||||
@@ -28,16 +28,31 @@ export function close() {
|
||||
|
||||
// ─── Overlay ───
|
||||
|
||||
/**
|
||||
* Bug 003 diagnostic helper — log overlay IPC failures to runtime events
|
||||
* (which mirror to sayit.log via appendDebugLog).
|
||||
*/
|
||||
function logOverlayIpcError(op: string, err: unknown) {
|
||||
try {
|
||||
// Lazy import to avoid circular dep with debugLog
|
||||
void import('./debugLog').then(({ addRuntimeEvent }) => {
|
||||
addRuntimeEvent('error', 'overlay-ipc', `${op} failed`, { error: String(err) })
|
||||
})
|
||||
} catch {
|
||||
console.error(`[overlay-ipc] ${op} failed:`, err)
|
||||
}
|
||||
}
|
||||
|
||||
export function showOverlay() {
|
||||
invoke('show_overlay')
|
||||
invoke('show_overlay').catch((err) => logOverlayIpcError('show_overlay', err))
|
||||
}
|
||||
|
||||
export function hideOverlay() {
|
||||
invoke('hide_overlay')
|
||||
invoke('hide_overlay').catch((err) => logOverlayIpcError('hide_overlay', err))
|
||||
}
|
||||
|
||||
export function updateOverlay(data: unknown) {
|
||||
invoke('update_overlay_state', { data })
|
||||
invoke('update_overlay_state', { data }).catch((err) => logOverlayIpcError('update_overlay_state', err))
|
||||
}
|
||||
|
||||
// ─── Paste / Context ───
|
||||
|
||||
@@ -33,6 +33,11 @@ export async function refreshRecorderSettings() {
|
||||
await orchestrator.refreshRuntimeSettings()
|
||||
}
|
||||
|
||||
/** 仅刷新 overlay 显示设置(主题/长度/时长)— 轻量,避免触发全量录音缓存刷新 */
|
||||
export async function refreshOverlaySettings() {
|
||||
await orchestrator.refreshOverlaySettings()
|
||||
}
|
||||
|
||||
/** 工作模式切换后重新连接 */
|
||||
export function reconnectProvider() {
|
||||
orchestrator.reconnectProvider()
|
||||
|
||||
@@ -316,6 +316,12 @@ export class RecorderOrchestrator {
|
||||
this.ensureConnection()
|
||||
}
|
||||
|
||||
/** 仅刷新 overlay 显示设置(主题/长度/时长),不触碰录音相关缓存。
|
||||
* 用于外观设置页面,避免改个颜色就触发十几个 IPC 调用。 */
|
||||
async refreshOverlaySettings() {
|
||||
await this.overlayService.refreshSettings()
|
||||
}
|
||||
|
||||
// ── State machine ──
|
||||
|
||||
private transition(to: RecorderState): boolean {
|
||||
|
||||
@@ -44,6 +44,9 @@ const PTT_LAB_VK = 0xA3
|
||||
let pttCode = ''
|
||||
let pttSetting = ''
|
||||
let pttKeyDown = false
|
||||
let hfCode = ''
|
||||
let hfSetting = ''
|
||||
let hfKeyDown = false
|
||||
let labKeyDown = false
|
||||
let labEnabled = false
|
||||
let started = false
|
||||
@@ -75,6 +78,15 @@ function handleKeyDown(e: KeyboardEvent) {
|
||||
return
|
||||
}
|
||||
|
||||
// 免提键(HF):在 keydown 时阻止默认行为,keyup 时触发 toggle
|
||||
if (hfCode && e.code === hfCode && pttCode !== hfCode) {
|
||||
hfKeyDown = true
|
||||
if (isModifierSetting(hfSetting)) {
|
||||
e.preventDefault()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// PTT Lab 键(右 Ctrl)
|
||||
if (labEnabled && e.code === PTT_LAB_CODE && !labKeyDown) {
|
||||
// 如果 PTT 主键也是右 Ctrl,不重复处理 lab
|
||||
@@ -112,6 +124,20 @@ function handleKeyUp(e: KeyboardEvent) {
|
||||
return
|
||||
}
|
||||
|
||||
// 免提键:keyup 时 emit toggle-hands-free
|
||||
if (hfCode && e.code === hfCode && hfKeyDown && pttCode !== hfCode) {
|
||||
hfKeyDown = false
|
||||
if (isModifierSetting(hfSetting)) {
|
||||
e.preventDefault()
|
||||
}
|
||||
console.log('[webview-kb] toggle-hands-free (webview fallback)', { code: e.code, hfSetting })
|
||||
emit('toggle-hands-free', {
|
||||
source: 'webview_fallback',
|
||||
vk: SETTING_TO_VK[hfSetting] || 0,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// PTT Lab 键
|
||||
if (labEnabled && e.code === PTT_LAB_CODE && labKeyDown) {
|
||||
if (pttCode === PTT_LAB_CODE) return
|
||||
@@ -137,7 +163,18 @@ export async function refreshPTTSetting() {
|
||||
}
|
||||
pttCode = SETTING_TO_CODE[pttSetting] || ''
|
||||
pttKeyDown = false
|
||||
|
||||
try {
|
||||
const hfSettingVal = await getSetting('shortcutHandsFree', 'AltRight')
|
||||
hfSetting = String(hfSettingVal || 'AltRight')
|
||||
} catch {
|
||||
hfSetting = 'AltRight'
|
||||
}
|
||||
hfCode = SETTING_TO_CODE[hfSetting] || ''
|
||||
hfKeyDown = false
|
||||
|
||||
console.log('[webview-kb] PTT setting refreshed:', pttSetting, '→ code:', pttCode)
|
||||
console.log('[webview-kb] HF setting refreshed:', hfSetting, '→ code:', hfCode)
|
||||
}
|
||||
|
||||
/** PTT Lab 启用/禁用 */
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SayIt AI 润色供应商
|
||||
|
||||
SayIt 有两个核心功能:语音转文本(ASR)和 AI 文本润色。ASR 部分可以参考 [豆包语音识别凭证获取](豆包ASR凭据获取.md) 和 [阿里云千问语音识别](aliyun.md)。
|
||||
|
||||
AI 润色功能可以去除口癖词(嗯、啊、那个)、修正语音识别错误、自动分段排版等,都可以通过 Prompt 自定义控制。
|
||||
|
||||
SayIt 已对接国内常用的 AI 服务商(通义千问、DeepSeek、豆包等),同时支持 OpenAI 兼容格式,可以接入其他供应商或第三方中转。
|
||||
|
||||
## 1. 最佳实践
|
||||
|
||||
**推荐 DeepSeek deepseek-v4-flash 作为默认选择。** 质量最好、价格极低、速度可接受。
|
||||
|
||||
DeepSeek V4 Flash(2026 年 4 月发布)是 284B 参数的 MoE 模型,13B 激活参数,支持 1M 上下文。关闭思考模式后速度接近旧版 deepseek-chat,但质量明显更好,价格还更便宜(输入 1 元/百万 Token,输出 2 元/百万 Token)。
|
||||
|
||||
如果对延迟极度敏感(需要 <500ms),可以用通义千问 qwen3.6-flash,速度最快但质量略逊。
|
||||
|
||||
| 供应商 | 模型 | 短文本 | 长文本 | 每月成本 |
|
||||
| --------------| -------------------| -----------| ------------| ----------|
|
||||
| **DeepSeek** | deepseek-v4-flash | **803ms** | **2089ms** | ~1.5 元 |
|
||||
| 通义千问 | qwen3.6-flash | 393ms | 1308ms | ~3.8 元 |
|
||||
|
||||
> 月成本按每天 100 次、每次约 100 字估算。V4 Flash 默认开启 thinking,会导致延迟飙升到 2~7 秒。SayIt 已在代码中自动关闭思考模式,无需用户手动配置。
|
||||
|
||||
## 2. 如何选择AI模型
|
||||
|
||||
- **速度**:总耗时由 TTFT(Time To First Token,首个字生成时间)和 TPS(每秒 Token 数)决定。语音转写场景下,总耗时超过 1~2 秒会有明显等待感。
|
||||
- **质量**:能否准确执行 Prompt 指令——去口癖、识别自我修正、格式化列表、纠正专有名词大小写等。
|
||||
- **价格**:V4 Flash 的自动 prompt 缓存机制让重复 system prompt 的场景成本极低。
|
||||
|
||||
## 3. 实测数据
|
||||
|
||||
以下数据是 2026 年 5 月实测的,使用非流式请求(和 SayIt 实际调用方式一致),每条用例跑了 3 次取中位数。
|
||||
|
||||
### 速度
|
||||
|
||||
| 供应商 | 模型 | 短文本 | 中等文本 | 长文本(~200字) |
|
||||
| ----------| -------------------------------| --------| ----------| ------------------|
|
||||
| DeepSeek | deepseek-v4-flash(关闭思考) | 803ms | 971ms | 2089ms |
|
||||
| 通义千问 | qwen3.6-flash | 393ms | 489ms | 1308ms |
|
||||
|
||||
|
||||
### 质量
|
||||
|
||||
两家都能正确去口癖、识别列表结构、纠正专有名词大小写(DeepSeek、ChatGPT、OpenAI 等)。V4 Flash 在专有名词纠正和格式化方面更稳定,输出更简洁。
|
||||
|
||||
|
||||
## 4. 关于豆包
|
||||
|
||||
豆包的语音识别(ASR)是目前中文场景下最好的,SayIt 推荐使用豆包 ASR。但豆包的 AI 文本模型(lite 和 mini 都测了)延迟极不稳定,中长文本经常超过 10 秒,不适合语音转写的润色场景。建议 ASR 用豆包,AI 润色用 DeepSeek 或通义千问。
|
||||
|
||||
## 5. 价格参考
|
||||
|
||||
| 供应商 | 模型 | 输入(百万Token) | 输出(百万Token) |
|
||||
| ----------| ----------------------| -------------------| -------------------|
|
||||
| DeepSeek | deepseek-v4-flash | 1 元 | 2 元 |
|
||||
| 通义千问 | qwen3.6-flash | 1.8 元 | 10.8 元 |
|
||||
| 豆包 | doubao-seed-2-0-mini | 0.2 元 | 2 元 |
|
||||
|
||||
|
||||
|
||||
## 6. 配置指南
|
||||
|
||||
> 建议直接使用官方 API。第三方中转站延迟普遍更高。
|
||||
|
||||
### DeepSeek(推荐)
|
||||
|
||||
- API 地址:`https://api.deepseek.com`
|
||||
- 模型:`deepseek-v4-flash`
|
||||
- 获取 API Key:[platform.deepseek.com](https://platform.deepseek.com/api_keys)
|
||||
- **注意**:SayIt 会自动在请求中添加 `thinking: {type: "disabled"}` 关闭思考模式以降低延迟
|
||||

|
||||
|
||||
### 通义千问
|
||||
|
||||
- API 地址:`https://dashscope.aliyuncs.com/compatible-mode`
|
||||
- 模型:`qwen3.6-flash`
|
||||
- 获取 API Key:[百炼平台](https://bailian.console.aliyun.com/?spm=a2c4g.11186623.0.0.60905ec6iyaRqr&tab=model#/api-key)
|
||||

|
||||
|
||||
### 豆包(火山方舟)
|
||||
- API 地址:`https://ark.cn-beijing.volces.com/api/v3`
|
||||
- 模型:`doubao-seed-2-0-mini-260215`
|
||||

|
||||
@@ -12,10 +12,10 @@ AI 润色模型推荐使用 **deepseek-v4-flash** 质量好、价格低、速度
|
||||
|
||||
如果对延迟极度敏感(需要 <500ms),可以用通义千问 qwen3.6-flash,速度最快但准确度稍微差点。
|
||||
|
||||
| 供应商 | 模型 | 短文本 | 长文本(~200字) | 输入价格 | 输出价格 |
|
||||
| ------------ | ----------------- | ------ | ---------------- | ---------------- | ----------------- |
|
||||
| **DeepSeek** | deepseek-v4-flash | 803ms | 2089ms | 1 元/百万Token | 2 元/百万Token |
|
||||
| 通义千问 | qwen3.6-flash | 393ms | 1308ms | 1.8 元/百万Token | 10.8 元/百万Token |
|
||||
| 供应商 | 模型 | 短文本 | 长文本(~200字) | 输入价格 | 输出价格 |
|
||||
| --------------| -------------------| --------| ------------------| ------------------| -------------------|
|
||||
| **DeepSeek** | deepseek-v4-flash | 803ms | 2089ms | 1 元/百万Token | 2 元/百万Token |
|
||||
| 通义千问 | qwen3.6-flash | 393ms | 1308ms | 1.8 元/百万Token | 10.8 元/百万Token |
|
||||
|
||||
## 2. 配置指南
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# SayIt ASR 语音识别配置
|
||||
|
||||
SayIt 支持三种语音识别模式:
|
||||
|
||||
- **云端 API**:调用云服务商的语音识别接口,准确率最高,适合大多数个人用户
|
||||
- **本地推理**:使用本地模型离线识别,不需要网络,但准确率和速度不如云端
|
||||
- **服务器模式**:自己部署后端服务,适合团队或有定制需求的场景
|
||||
|
||||
对个人用户来说,**云端 API 是最推荐的方式**,下面主要介绍这个。
|
||||
|
||||
## 1. 最佳实践
|
||||
|
||||
中文语音识别准确率最高的是**豆包流式语音识别 2.0**(Seed-ASR-2.0),其次是阿里的**千问 ASR**(qwen3-asr-flash)。
|
||||
|
||||
SayIt 对豆包和千问 ASR 都做了流式优化——按住说话时,音频会实时发送给服务端,松手后只需要等待最后一小段音频的处理时间,通常几百毫秒就能拿到结果。
|
||||
|
||||
另外 SayIt 还支持**千问 3.5 Omni** 多模态模型,这类模型原生支持音频输入、文本输出,可以在转录的同时执行 Prompt 指令(比如去口癖、翻译等),相当于 ASR + AI 润色一步到位。
|
||||
|
||||
| 供应商 | 模型 | 特点 | 价格 |
|
||||
| --------------| --------------------------| --------------------------| -------------------------|
|
||||
| **豆包** | Seed-ASR-2.0 | 中文准确率最高,推荐首选 | 1.00 元/小时 |
|
||||
| **阿里千问** | qwen3-asr-flash-realtime | 流式识别,速度极快 | 1.19 元/小时 |
|
||||
| **阿里千问** | qwen3.5-omni-flash | ASR + AI 一步到位 | 按 Token 计费(见下方) |
|
||||
|
||||
|
||||
## 2. 使用豆包语音识别
|
||||
|
||||
SayIt 支持使用字节跳动「豆包流式语音识别模型 2.0」作为语音识别引擎。豆包 ASR 是目前中文语音识别准确率最高的模型,推荐使用。
|
||||
|
||||
1. 注册 / 登录火山引擎
|
||||
|
||||
打开[火山引擎语音服务控制台](https://console.volcengine.com/speech/service/10038),用手机号注册或登录即可。已有账号的直接登录,没有账号会引导你先注册。
|
||||
[图片]
|
||||
|
||||
|
||||
2. 完成实名认证
|
||||
|
||||
火山引擎的 API 服务需要先做实名认证,没认证的话后面创建应用会卡住。进入[实名认证页面](https://console.volcengine.com/user/authentication/detail/),按页面提示走完就行。
|
||||
[图片]
|
||||
|
||||
|
||||
3. 创建应用并获取 APP ID
|
||||
|
||||
进入[应用管理页面](https://console.volcengine.com/speech/app1),点击「创建应用」。填写应用名称和简介(随便写就行),关键一步:一定要勾选「豆包流式语音识别模型 2.0 小时版」,不然后面用不了。
|
||||
[图片]
|
||||
|
||||
|
||||
4. 获取APP ID 和 Access Token
|
||||
|
||||
进入 [API 服务中心页面](https://console.volcengine.com/speech/service/10038),点击「豆包流式语音识别模型2.0」(配图是旧版界面)。页面上会显示你的 APP ID,鼠标放到 Access Token 上面,点击小眼睛图标,可以看到 Access Token 信息。
|
||||
[图片]
|
||||
|
||||
记录下来这两个信息填到 SayIt 软件中。选择 语音引擎 - 云 API 模式 - 选择ASR供应商为豆包 ASR,填写凭证信息后可以保存测试。
|
||||
[图片]
|
||||
|
||||
|
||||
豆包开通试用后,半年内有 20 小时的[免费额度](https://www.volcengine.com/docs/6561/1359369?lang=zh)。超出免费额度后,按量计费:豆包流式语音识别模型 2.0 — 1 元/小时。[后付费价格详情](https://www.volcengine.com/docs/6561/1359370?lang=zh)
|
||||
|
||||
|
||||
## 3. 阿里千问 ASR
|
||||
|
||||
阿里百炼平台提供了两个语音识别模型,都可以在 SayIt 中使用。
|
||||
|
||||
**qwen3-asr-flash / qwen3-asr-flash-realtime**
|
||||
|
||||
纯语音识别模型,中文准确率仅次于豆包。有两个版本:
|
||||
- **qwen3-asr-flash**:非流式,录完再发,速度也很快
|
||||
- **qwen3-asr-flash-realtime**:流式,边录边发,松手后几乎立刻出结果,体验极佳
|
||||
|
||||
两者准确率一样,推荐使用流式版本。适合搭配独立的 AI 润色供应商使用(比如通义千问或 DeepSeek 做文本校对)。
|
||||
|
||||
**qwen3.5-omni-flash**
|
||||
|
||||
多模态模型,可以直接接收音频输入并输出文本。和普通 ASR + AI 润色的两步流程不同,Omni 模型一步完成,可以在 System Prompt 中直接控制输出格式(比如去口癖、翻译、列表排版等)。
|
||||
|
||||
适合不想分别配置 ASR 和 AI 润色的用户,配置更简单。
|
||||
|
||||
**配置方式**
|
||||
|
||||
在 SayIt 设置中选择千问 ASR 系列,需要填写 API Key。
|
||||
|
||||
- 获取 API Key:[阿里百炼平台](https://bailian.console.aliyun.com/cn-beijing?tab=model#/api-key)
|
||||
|
||||
## 4. 价格
|
||||
|
||||
下面是各个模型按需使用的价格参考。
|
||||
|
||||
| 模型 | 价格 | 免费额度 |
|
||||
| ---------------------------------------| -----------------| ------------------------|
|
||||
| 豆包 Seed-ASR-2.0 | 1.00 元/小时 | 20 小时(半年内) |
|
||||
| 千问 qwen3-asr-flash(非流式) | 0.79 元/小时 | 10 小时(90天内) |
|
||||
| 千问 qwen3-asr-flash-realtime(流式) | 1.19 元/小时 | 10 小时(90天内) |
|
||||
| 千问 qwen3.5-omni-flash | 约 2.3 元/小时* | 100 万 Token(90天内) |
|
||||
|
||||
*Omni 按 Token 计费(每秒音频 ≈ 25 token),2.3 元为粗略估算。详见[千问价格文档](https://help.aliyun.com/zh/model-studio/model-pricing#95f0464a10q5c)。
|
||||
+94
-11
@@ -61,7 +61,7 @@ class ASREngine:
|
||||
self._model = Qwen3ASRModel.LLM(
|
||||
model=config.asr.model,
|
||||
gpu_memory_utilization=config.asr.vllm_gpu_util,
|
||||
max_new_tokens=512,
|
||||
max_new_tokens=2048,
|
||||
max_model_len=config.asr.vllm_max_model_len,
|
||||
)
|
||||
self._backend = "vllm"
|
||||
@@ -70,7 +70,7 @@ class ASREngine:
|
||||
import torch
|
||||
self._model = Qwen3ASRModel.from_pretrained(
|
||||
config.asr.model, dtype=torch.bfloat16,
|
||||
device_map=config.asr.device, max_new_tokens=512,
|
||||
device_map=config.asr.device, max_new_tokens=2048,
|
||||
)
|
||||
self._backend = "transformers"
|
||||
logger.info("Qwen3-ASR ready (%s)", self._backend)
|
||||
@@ -235,15 +235,16 @@ class ASREngine:
|
||||
job_text_parts[idx] = []
|
||||
continue
|
||||
|
||||
if duration <= 120.0:
|
||||
# ≤ 2min: send whole audio, no split — accuracy first.
|
||||
# VAD-based filtering was removed because it aggressively
|
||||
# discards whispered speech (low-energy audio misclassified
|
||||
# as silence), causing severe recognition loss.
|
||||
if duration <= 60.0:
|
||||
# ≤ 1min: send whole audio, no split — accuracy first.
|
||||
segs = [(audio, duration)]
|
||||
else:
|
||||
# > 2min: safe split at ≥3s silence gaps
|
||||
# > 1min: safe split at silence gaps
|
||||
segs = self._safe_split(audio)
|
||||
# Merge segments with very low speech ratio into adjacent segments
|
||||
# to avoid batch inference failure from near-silent segments.
|
||||
if len(segs) > 1:
|
||||
segs = self._merge_silent_segments(segs)
|
||||
logger.info("Safe split %.1fs -> %d segs %s",
|
||||
duration, len(segs), [f"{d:.1f}s" for _, d in segs])
|
||||
job_vad_ms[idx] = int((time.monotonic() - vad_t0) * 1000)
|
||||
@@ -408,7 +409,55 @@ class ASREngine:
|
||||
logger.exception("VAD failed")
|
||||
return [(audio, len(audio) / 16000.0)] if fallback_to_full else []
|
||||
|
||||
def _safe_split(self, audio: np.ndarray, min_gap: float = 3.0, min_seg: float = 15.0, max_seg: float = 240.0) -> list[tuple[np.ndarray, float]]:
|
||||
def _merge_silent_segments(self, segs: list[tuple[np.ndarray, float]], min_speech_ratio: float = 0.15) -> list[tuple[np.ndarray, float]]:
|
||||
"""Merge segments with very low speech ratio into adjacent segments.
|
||||
|
||||
Near-silent segments in a batch can cause vLLM to return empty results
|
||||
for the entire batch. Merging them into neighbors avoids this while
|
||||
preserving any speech content they may contain.
|
||||
"""
|
||||
if not self._vad or len(segs) <= 1:
|
||||
return segs
|
||||
import torch
|
||||
# Identify which segments have low speech ratio
|
||||
speech_ok = []
|
||||
for seg_audio, seg_dur in segs:
|
||||
try:
|
||||
res = self._vad.generate(input=torch.tensor(seg_audio, dtype=torch.float32), cache={}, is_final=True)
|
||||
vad_segs = res[0].get("value", []) if res else []
|
||||
speech_time = sum((e - s) for s, e in vad_segs) / 1000.0 if vad_segs else 0
|
||||
speech_ok.append(speech_time / seg_dur >= min_speech_ratio if seg_dur > 0 else True)
|
||||
except Exception:
|
||||
speech_ok.append(True) # on error, assume it's fine
|
||||
|
||||
if all(speech_ok):
|
||||
return segs
|
||||
|
||||
# Merge low-speech segments into adjacent (prefer previous, fallback to next)
|
||||
merged: list[tuple[np.ndarray, float]] = []
|
||||
i = 0
|
||||
while i < len(segs):
|
||||
if speech_ok[i]:
|
||||
merged.append(segs[i])
|
||||
i += 1
|
||||
else:
|
||||
# Merge into previous segment if available, otherwise next
|
||||
if merged:
|
||||
prev_audio, prev_dur = merged[-1]
|
||||
cur_audio, cur_dur = segs[i]
|
||||
merged[-1] = (np.concatenate([prev_audio, cur_audio]), prev_dur + cur_dur)
|
||||
elif i + 1 < len(segs):
|
||||
cur_audio, cur_dur = segs[i]
|
||||
nxt_audio, nxt_dur = segs[i + 1]
|
||||
merged.append((np.concatenate([cur_audio, nxt_audio]), cur_dur + nxt_dur))
|
||||
speech_ok[i + 1] = True # skip next since we consumed it
|
||||
i += 1
|
||||
else:
|
||||
merged.append(segs[i])
|
||||
i += 1
|
||||
return merged
|
||||
|
||||
def _safe_split(self, audio: np.ndarray, min_gap: float = 1.0, min_seg: float = 10.0, max_seg: float = 100.0) -> list[tuple[np.ndarray, float]]:
|
||||
"""Split audio at long silence gaps (≥ min_gap seconds), keeping ALL audio.
|
||||
|
||||
Unlike _vad_split which discards silence, this preserves the full audio
|
||||
@@ -443,7 +492,41 @@ class ASREngine:
|
||||
if all(d >= min_seg for d in seg_durs) and all(d <= max_seg for d in seg_durs):
|
||||
cuts.append(mid)
|
||||
|
||||
if not cuts:
|
||||
if not cuts and dur > 100.0:
|
||||
# Fallback: force split at lowest energy point near ideal boundaries
|
||||
ideal_seg = 60.0 # target segment length
|
||||
n_cuts_needed = max(1, int(dur / ideal_seg) - 1)
|
||||
frame_len = int(0.05 * 16000) # 50ms frames
|
||||
hop = frame_len // 2
|
||||
n_frames = (len(audio) - frame_len) // hop
|
||||
if n_frames > 0:
|
||||
energy = np.array([np.sum(audio[i*hop:i*hop+frame_len]**2) for i in range(n_frames)])
|
||||
# Smooth energy
|
||||
kernel = np.ones(20) / 20
|
||||
if len(energy) > len(kernel):
|
||||
energy = np.convolve(energy, kernel, mode='same')
|
||||
forced_cuts = []
|
||||
for c in range(1, n_cuts_needed + 1):
|
||||
target_time = dur * c / (n_cuts_needed + 1)
|
||||
# Search window: +/- 10s around target
|
||||
win_start = max(0, int((target_time - 10) * 16000 / hop))
|
||||
win_end = min(n_frames, int((target_time + 10) * 16000 / hop))
|
||||
if win_start < win_end:
|
||||
min_idx = win_start + np.argmin(energy[win_start:win_end])
|
||||
cut_time = min_idx * hop / 16000.0
|
||||
forced_cuts.append(cut_time)
|
||||
if forced_cuts:
|
||||
forced_cuts.sort()
|
||||
points = [0.0] + forced_cuts + [dur]
|
||||
segments = []
|
||||
for i in range(len(points) - 1):
|
||||
s, e = int(points[i]*16000), int(points[i+1]*16000)
|
||||
seg = audio[s:e]
|
||||
segments.append((seg, len(seg)/16000.0))
|
||||
logger.info("Energy fallback split %.1fs -> %d segs %s", dur, len(segments), [f"{d:.1f}s" for _,d in segments])
|
||||
return segments
|
||||
return [(audio, dur)]
|
||||
elif not cuts:
|
||||
return [(audio, dur)]
|
||||
|
||||
cuts.sort()
|
||||
@@ -467,7 +550,7 @@ class ASREngine:
|
||||
debug = {"duration_sec": round(duration, 1), "vad_segments": [], "context": ctx}
|
||||
|
||||
if self._engine_type == "firered":
|
||||
if duration <= 15.0:
|
||||
if duration <= 120.0:
|
||||
debug["vad_segments"] = [{"index": 0, "duration_sec": round(duration, 1)}]
|
||||
return self._firered_backend.transcribe_audio(audio), debug
|
||||
segments = self._vad_split(audio)
|
||||
|
||||
@@ -220,15 +220,15 @@ def _is_hotword_hallucination(text: str, context: str, audio_peak: float = 1.0,
|
||||
audio_dur: float = 0.0, voice_score: float = 1.0) -> bool:
|
||||
if not context or not text:
|
||||
return False
|
||||
words = set(context.split())
|
||||
words = set(w.lower() for w in context.split())
|
||||
cleaned = re.sub(r"[\s銆傦紝銆侊紵锛?,?!\u3000。]+", " ", text).strip()
|
||||
tokens = cleaned.split()
|
||||
if not tokens:
|
||||
return False
|
||||
matched = sum(1 for token in tokens if token in words)
|
||||
matched = sum(1 for token in tokens if token.lower() in words)
|
||||
consec = max_consec = 0
|
||||
for token in tokens:
|
||||
if token in words:
|
||||
if token.lower() in words:
|
||||
consec += 1
|
||||
max_consec = max(max_consec, consec)
|
||||
else:
|
||||
@@ -239,7 +239,7 @@ def _is_hotword_hallucination(text: str, context: str, audio_peak: float = 1.0,
|
||||
return True
|
||||
# Short result composed entirely of hotwords → almost certainly hallucination.
|
||||
# Real users don't speak just a single hotword and stop.
|
||||
if len(cleaned) <= 6 and all(t in words for t in tokens):
|
||||
if len(cleaned) <= 6 and all(t.lower() in words for t in tokens):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -372,11 +372,28 @@ async def _run_asr_llm(
|
||||
|
||||
# Strip hotword context when audio has no detectable speech energy,
|
||||
# preventing the ASR model from hallucinating hotwords on noise.
|
||||
# Whispered speech has voice_score > 0.50; noise/silence < 0.15.
|
||||
# Three-layer detection: voice_score, energy level, and VAD.
|
||||
audio_peak = float(np.max(np.abs(audio)))
|
||||
audio_rms = float(np.sqrt(np.mean(audio**2)))
|
||||
vs = _voice_score(audio)
|
||||
if vs < _VS_THRESHOLD and (context is None or context):
|
||||
logger.info("Stripping hotword context: voice_score=%.3f peak=%.4f", vs, audio_peak)
|
||||
strip_context = False
|
||||
if vs < _VS_THRESHOLD or (audio_peak < 0.02 and audio_rms < 0.002):
|
||||
strip_context = True
|
||||
elif context and duration < 10.0 and asr_engine and asr_engine._vad is not None:
|
||||
# For short audio with hotwords: use VAD to detect if there's actual speech.
|
||||
# This catches non-speech noise (keyboard, clicks) that has high energy.
|
||||
import torch
|
||||
try:
|
||||
vad_res = asr_engine._vad.generate(
|
||||
input=torch.tensor(audio, dtype=torch.float32), cache={}, is_final=True)
|
||||
vad_segs = vad_res[0].get("value", []) if vad_res else []
|
||||
logger.debug("VAD check: segs=%d, duration=%.1f", len(vad_segs), duration)
|
||||
if not vad_segs:
|
||||
strip_context = True
|
||||
except Exception as e:
|
||||
logger.warning("VAD check failed: %s", e)
|
||||
if strip_context and (context is None or context):
|
||||
logger.info("Stripping hotword context: voice_score=%.3f peak=%.4f rms=%.5f", vs, audio_peak, audio_rms)
|
||||
context = ""
|
||||
|
||||
raw_text, asr_ms, asr_debug = await asr_engine.transcribe(audio, context=context, language=language)
|
||||
@@ -384,6 +401,12 @@ async def _run_asr_llm(
|
||||
asr_debug["voice_score"] = round(vs, 3)
|
||||
if _is_noise(asr_text, duration, voice_score=vs) or _is_hotword_hallucination(asr_text, asr_debug.get("context", ""), audio_peak=audio_peak, audio_dur=duration, voice_score=vs):
|
||||
asr_text = ""
|
||||
# Low energy audio producing short text is almost certainly noise/hallucination
|
||||
if asr_text and audio_peak < 0.02 and audio_rms < 0.002 and len(asr_text) <= 10:
|
||||
asr_text = ""
|
||||
# VAD detected no speech but ASR produced short text → noise hallucination
|
||||
if asr_text and strip_context and len(asr_text) <= 10:
|
||||
asr_text = ""
|
||||
|
||||
if cfg.logging.slow_asr_ms and asr_ms >= cfg.logging.slow_asr_ms:
|
||||
logger.warning("Slow ASR detected duration_sec=%.2f asr_ms=%d", duration, asr_ms)
|
||||
|
||||
@@ -31,7 +31,7 @@ asr:
|
||||
language: "Chinese" # Chinese / English / Cantonese
|
||||
itn: true # Inverse text normalization
|
||||
vllm_gpu_util: 0.5
|
||||
vllm_max_model_len: 8192
|
||||
vllm_max_model_len: 16384
|
||||
max_concurrency: 1
|
||||
batch_max_size: 32
|
||||
batch_wait_ms: 60
|
||||
|
||||
Reference in new issue
Block a user