chore: 整合后端更新与文档,准备 0.0.4 发布

- 后端: ASR 性能优化与热词幻觉修复 (asr.py / main.py),调整 config.example.yaml vllm_max_model_len
- 补充 docs AI/ASR 服务提供商文档
- 同步客户端原生层(src-tauri)、overlay、recorder 等既有改动
- .gitignore 忽略 we-media/ 运营资料
This commit is contained in:
crosswk committed 2026-06-02 15:22:30 +08:00
1 parent be137722ca
commit a7e2dd6590
19 files changed
+691 -106

No files matched your search

+2
View File
@@ -7,6 +7,8 @@ config.yaml
# ── Development-only (not for open source) ──
dev-docs/
dev-scripts/
dev-notes/
we-media/
sayit说话波形图效果.html
# ── Server dev artifacts ──
+1 -1
View File
@@ -4,7 +4,7 @@
# SayIt
**开口即成稿 — 用说话代替打字,AI 实时把口语变成书面表达。**
**随口说,出色写 — 用说话代替打字,AI 实时把口语变成书面表达。**
按下快捷键开始说话,再按一次,润色后的文字自动输入到光标位置。
+7
View File
@@ -23,3 +23,10 @@ pub async fn update_overlay_state(data: Value, app: AppHandle, window_state: Sta
window_state.update_overlay_state(&app, &data);
Ok(())
}
/// Reported by the overlay webview in response to an `overlay-ping` event.
/// Used to detect WebView2 unresponsiveness.
#[tauri::command]
pub fn overlay_pong(seq: u64) {
crate::window::record_overlay_pong(seq);
}
+1
View File
@@ -302,6 +302,7 @@ fn main() {
commands::window::show_overlay,
commands::window::hide_overlay,
commands::window::update_overlay_state,
commands::window::overlay_pong,
// Paste / Context
commands::paste::paste_text,
commands::paste::get_probe_result,
+115 -8
View File
@@ -1,12 +1,30 @@
use serde_json::Value;
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
use std::sync::Mutex;
use tauri::{AppHandle, Emitter, Manager, WebviewUrl, WebviewWindowBuilder};
use crate::commands::system::write_log_line;
const OVERLAY_DEFAULT_BASE_WIDTH: f64 = 360.0;
const OVERLAY_BASE_HEIGHT: f64 = 56.0;
const OVERLAY_FALLBACK_WIDTH: f64 = 520.0;
const OVERLAY_FALLBACK_HEIGHT: f64 = 224.0;
/// Time of last received pong from overlay (epoch millis). 0 = never.
/// Used to detect WebView2 unresponsiveness in `show_overlay`.
static LAST_PONG_MS: AtomicI64 = AtomicI64::new(0);
/// Monotonic id for ping/pong correlation
static PING_SEQ: AtomicU64 = AtomicU64::new(0);
pub fn record_overlay_pong(_seq: u64) {
let now = chrono::Utc::now().timestamp_millis();
LAST_PONG_MS.store(now, Ordering::SeqCst);
}
fn now_ms() -> i64 {
chrono::Utc::now().timestamp_millis()
}
#[derive(Debug, Clone, PartialEq)]
enum OverlayLayout {
Base,
@@ -31,25 +49,93 @@ impl WindowState {
let is_fallback = layout == OverlayLayout::Fallback;
let base_w = *self.overlay_base_width.lock().unwrap();
let bounds = calc_overlay_bounds(app, is_fallback, base_w);
let (mon_w, mon_h, mon_scale) = monitor_info(app);
if let Some(overlay) = app.get_webview_window("overlay") {
let bounds = calc_overlay_bounds(app, is_fallback, base_w);
let _ = overlay.set_position(tauri::Position::Logical(tauri::LogicalPosition::new(bounds.0, bounds.1)));
let _ = overlay.set_size(tauri::Size::Logical(tauri::LogicalSize::new(bounds.2, bounds.3)));
let _ = overlay.show();
let _ = overlay.set_always_on_top(true);
// ── Diagnostic snapshot before any state change ──
let was_visible = overlay.is_visible().unwrap_or(false);
let outer_pos = overlay.outer_position()
.map(|p| format!("{},{}", p.x, p.y))
.unwrap_or_else(|_| "err".to_string());
let outer_size = overlay.outer_size()
.map(|s| format!("{}x{}", s.width, s.height))
.unwrap_or_else(|_| "err".to_string());
// Probe webview responsiveness — send a ping; pong arrival is logged async.
let seq = PING_SEQ.fetch_add(1, Ordering::SeqCst) + 1;
let last_pong = LAST_PONG_MS.load(Ordering::SeqCst);
let pong_age_ms = if last_pong == 0 { -1 } else { now_ms() - last_pong };
let ping_emit_err = overlay.emit("overlay-ping", seq).err()
.map(|e| format!("{:?}", e));
write_log_line(&format!(
"[overlay-diag] show begin handle=present layout={:?} fallback={} was_visible={} \
outer_pos={} outer_size={} target_bounds={:?} monitor={}x{}@{:.2} \
ping_seq={} last_pong_age_ms={} ping_emit_err={:?}",
layout, is_fallback, was_visible,
outer_pos, outer_size, bounds, mon_w, mon_h, mon_scale,
seq, pong_age_ms, ping_emit_err,
));
let pos_err = overlay.set_position(tauri::Position::Logical(
tauri::LogicalPosition::new(bounds.0, bounds.1)
)).err().map(|e| format!("{:?}", e));
let size_err = overlay.set_size(tauri::Size::Logical(
tauri::LogicalSize::new(bounds.2, bounds.3)
)).err().map(|e| format!("{:?}", e));
let show_err = overlay.show().err().map(|e| format!("{:?}", e));
let aot_err = overlay.set_always_on_top(true).err().map(|e| format!("{:?}", e));
set_overlay_interactivity(&overlay, is_fallback);
// Re-snapshot to confirm visibility actually changed
let now_visible = overlay.is_visible().unwrap_or(false);
let now_outer_pos = overlay.outer_position()
.map(|p| format!("{},{}", p.x, p.y))
.unwrap_or_else(|_| "err".to_string());
let any_err = pos_err.is_some() || size_err.is_some()
|| show_err.is_some() || aot_err.is_some();
let level = if any_err { "WARN" } else { "INFO" };
write_log_line(&format!(
"[overlay-diag] show end [{}] now_visible={} now_outer_pos={} \
pos_err={:?} size_err={:?} show_err={:?} aot_err={:?}",
level, now_visible, now_outer_pos,
pos_err, size_err, show_err, aot_err,
));
} else {
write_log_line(&format!(
"[overlay-diag] show begin handle=MISSING layout={:?} fallback={} \
target_bounds={:?} monitor={}x{}@{:.2} — recreating",
layout, is_fallback, bounds, mon_w, mon_h, mon_scale,
));
self.create_overlay(app, is_fallback, base_w);
}
}
pub fn hide_overlay(&self, app: &AppHandle) {
// Reset to base layout when hiding
*self.overlay_layout.lock().unwrap() = OverlayLayout::Base;
let prev_layout = {
let mut g = self.overlay_layout.lock().unwrap();
let prev = g.clone();
*g = OverlayLayout::Base;
prev
};
if let Some(overlay) = app.get_webview_window("overlay") {
set_overlay_interactivity(&overlay, false);
let _ = overlay.hide();
let was_visible = overlay.is_visible().unwrap_or(false);
let hide_err = overlay.hide().err().map(|e| format!("{:?}", e));
write_log_line(&format!(
"[overlay-diag] hide prev_layout={:?} was_visible={} hide_err={:?}",
prev_layout, was_visible, hide_err,
));
} else {
write_log_line(&format!(
"[overlay-diag] hide handle=MISSING prev_layout={:?}",
prev_layout,
));
}
}
@@ -120,11 +206,21 @@ impl WindowState {
match builder.build() {
Ok(overlay) => {
set_overlay_interactivity(&overlay, is_fallback);
let _ = overlay.show();
let show_err = overlay.show().err().map(|e| format!("{:?}", e));
let now_visible = overlay.is_visible().unwrap_or(false);
write_log_line(&format!(
"[overlay-diag] create_overlay OK is_fallback={} bounds={:?} \
show_err={:?} now_visible={}",
is_fallback, bounds, show_err, now_visible,
));
}
Err(e) => {
eprintln!("[overlay] create_overlay: FAILED: {}", e);
log::error!("Failed to create overlay window: {}", e);
write_log_line(&format!(
"[overlay-diag] create_overlay FAILED is_fallback={} bounds={:?} err={:?}",
is_fallback, bounds, e,
));
}
}
}
@@ -134,6 +230,17 @@ fn set_overlay_interactivity(overlay: &tauri::WebviewWindow, interactive: bool)
let _ = overlay.set_ignore_cursor_events(!interactive);
}
/// Returns (logical_width, logical_height, scale_factor) of primary monitor.
fn monitor_info(app: &AppHandle) -> (f64, f64, f64) {
if let Some(monitor) = app.primary_monitor().ok().flatten() {
let size = monitor.size();
let scale = monitor.scale_factor();
(size.width as f64 / scale, size.height as f64 / scale, scale)
} else {
(1920.0, 1080.0, 1.0)
}
}
/// Returns (x, y, width, height)
fn calc_overlay_bounds(app: &AppHandle, is_fallback: bool, base_w: f64) -> (f64, f64, f64, f64) {
let (desired_w, desired_h) = if is_fallback {
+35 -24
View File
@@ -1,12 +1,12 @@
// 外观设置页面 — 主题 + 悬浮窗样式 + 预览
import { useEffect, useState } from 'react'
import { useEffect, useRef, useState } from 'react'
import { Card, CardContent } from '@/components/ui/card'
import { Switch } from '@/components/ui/switch'
import { themeList } from '@/themes'
import { switchTheme, getActiveThemeId } from '@/stores/theme'
import { getSetting, setSetting } from '@/services/store'
import { refreshRecorderSettings } from '@/services/recorder'
import { refreshOverlaySettings } from '@/services/recorder'
import { OVERLAY_WIDTH_PRESETS, type OverlayWidthPreset } from '@/services/recorder/types'
import { type OverlayWaveTheme } from './utils'
@@ -46,26 +46,37 @@ function getTimerColor(theme: OverlayWaveTheme): string {
}
function OverlayPreview({ theme, showDuration, barCount }: { theme: OverlayWaveTheme; showDuration: boolean; barCount: number }) {
const [bars, setBars] = useState<number[]>(Array.from({ length: barCount }, () => 3))
useEffect(() => {
setBars(Array.from({ length: barCount }, () => 3))
}, [barCount])
const barRefs = useRef<Array<HTMLDivElement | null>>([])
useEffect(() => {
const heights = new Array(barCount).fill(3)
let running = true
const animate = () => {
let rafId = 0
let lastFrame = 0
const FRAME_INTERVAL = 1000 / 30 // 30fps 足够流畅,且不占满主线程
const animate = (now: number) => {
if (!running) return
setBars((prev) =>
prev.map((v) => {
if (now - lastFrame >= FRAME_INTERVAL) {
lastFrame = now
for (let i = 0; i < heights.length; i++) {
const target = 3 + Math.random() * 15
return v + (target - v) * 0.15
}),
)
requestAnimationFrame(animate)
heights[i] = heights[i] + (target - heights[i]) * 0.15
const el = barRefs.current[i]
if (el) {
const h = Math.min(18, Math.max(3, heights[i]))
el.style.height = `${h}px`
el.style.opacity = String(0.7 + (h / 18) * 0.3)
}
}
}
rafId = requestAnimationFrame(animate)
}
rafId = requestAnimationFrame(animate)
return () => {
running = false
cancelAnimationFrame(rafId)
}
requestAnimationFrame(animate)
return () => { running = false }
}, [barCount])
return (
@@ -73,17 +84,17 @@ function OverlayPreview({ theme, showDuration, barCount }: { theme: OverlayWaveT
{/* 1:1 还原真实悬浮窗样式 */}
<div className="flex items-center rounded-full border border-slate-600 bg-black px-4 py-2 shadow-[0_6px_16px_rgba(0,0,0,0.35)]">
<div className="flex items-center gap-[2px]" style={{ height: '20px' }}>
{bars.map((height, index) => {
const h = Math.min(18, Math.max(3, height))
const color = getBarColor(index, bars.length, theme)
{Array.from({ length: barCount }, (_, index) => {
const color = getBarColor(index, barCount, theme)
return (
<div
key={index}
ref={(el) => { barRefs.current[index] = el }}
className="w-[2.5px] rounded-full"
style={{
backgroundColor: color,
height: `${h}px`,
opacity: 0.7 + (h / 18) * 0.3,
height: '3px',
opacity: 0.7,
transition: 'height 50ms ease-out, opacity 50ms ease-out',
}}
/>
@@ -130,20 +141,20 @@ export default function AppearancePage() {
const handleOverlayThemeChange = async (theme: OverlayWaveTheme) => {
setOverlayWaveTheme(theme)
await setSetting('overlayWaveTheme', theme)
await refreshRecorderSettings()
await refreshOverlaySettings()
}
const handleToggleDuration = async () => {
const next = !overlayShowDuration
setOverlayShowDuration(next)
await setSetting('overlayShowDuration', next)
await refreshRecorderSettings()
await refreshOverlaySettings()
}
const handleOverlayWidthChange = async (preset: OverlayWidthPreset) => {
setOverlayWidth(preset)
await setSetting('overlayWidth', preset)
await refreshRecorderSettings()
await refreshOverlaySettings()
}
return (
+9
View File
@@ -1,6 +1,8 @@
import { startWebviewKeyboardFallback } from '../services/webviewKeyboardFallback'
import React from 'react'
import ReactDOM from 'react-dom/client'
import { listen } from '@tauri-apps/api/event'
import { invoke } from '@tauri-apps/api/core'
import Overlay from './Overlay'
import '../index.css'
@@ -11,6 +13,13 @@ document.head.appendChild(style)
void startWebviewKeyboardFallback()
// Health check: respond to ping from main process so it can detect
// WebView2 unresponsiveness. Bug 003 diagnostic.
void listen<number>('overlay-ping', (event) => {
const seq = typeof event.payload === 'number' ? event.payload : 0
void invoke('overlay_pong', { seq }).catch(() => {})
})
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<Overlay />
+1 -1
View File
@@ -44,7 +44,7 @@ export default function About() {
<span className="rounded-full bg-muted/50 px-2.5 py-0.5 text-xs text-muted-foreground">v{currentVersion}</span>
<button
type="button"
onClick={() => void shellOpen('https://github.com/crosswk/')}
onClick={() => void shellOpen('https://github.com/crosswk/SayIt')}
className="flex h-6 w-6 items-center justify-center rounded-full bg-muted/50 text-muted-foreground transition-colors hover:bg-foreground/15 hover:text-foreground"
title="GitHub"
>
+147 -46
View File
@@ -21,8 +21,17 @@ import {
import { loadAudioAsDataUrl } from '@/services/audioFileService'
import { segmentAsrText } from '@/services/textSegmenter'
import { applyTextReplacements } from '@/services/textReplacement'
import { getProvider } from '@/services/transcription'
import { reconnectProvider } from '@/services/recorder'
import {
BUILTIN_SET_WORDS_KEY,
BUILTIN_SET_ACTIVE_KEY,
CUSTOM_THEMES_KEY,
CUSTOM_THEME_ACTIVE_KEY,
composeHotwords,
normalizeBuiltinSetActive,
normalizeBuiltinSetWords,
normalizeCustomThemeActive,
normalizeCustomThemes,
} from '@/services/hotwords/model'
const HISTORY_PAGE_SIZE = 100
@@ -102,50 +111,146 @@ export default function History() {
const pcmData = bytes.slice(44)
const chunk = pcmData.buffer.slice(pcmData.byteOffset, pcmData.byteOffset + pcmData.byteLength)
// Diagnostic: compute peak amplitude of the PCM data being sent
const pcmInt16 = new Int16Array(chunk)
let reprocessPeak = 0
for (let i = 0; i < pcmInt16.length; i++) {
const v = Math.abs(pcmInt16[i])
if (v > reprocessPeak) reprocessPeak = v
}
const reprocessPeakNorm = reprocessPeak / 32768
const reprocessDurSec = pcmInt16.length / 16000
console.log('[reprocess-diag] PCM stats', {
byteLength: chunk.byteLength,
samples: pcmInt16.length,
durationSec: reprocessDurSec.toFixed(2),
peakInt16: reprocessPeak,
peakNormalized: reprocessPeakNorm.toFixed(4),
wouldBeSilent: reprocessPeakNorm < 0.01,
})
const preset = await getActivePreset()
const aiEnabled = await getSetting('aiEnabled', false)
// 使用当前 Provider 重新识别
const provider = getProvider()
let reprocessDone = false
// 加载热词
let hotwords: string[] = []
try {
const [rawSetWords, rawSetActive, rawCustomThemes, rawCustomThemeActive] = await Promise.all([
getSetting(BUILTIN_SET_WORDS_KEY, {}),
getSetting(BUILTIN_SET_ACTIVE_KEY, {}),
getSetting(CUSTOM_THEMES_KEY, []),
getSetting(CUSTOM_THEME_ACTIVE_KEY, {}),
])
const setWords = normalizeBuiltinSetWords(rawSetWords as Record<string, unknown>)
const setActive = normalizeBuiltinSetActive(rawSetActive as Record<string, unknown>)
const themes = normalizeCustomThemes(rawCustomThemes)
const themeActive = normalizeCustomThemeActive(rawCustomThemeActive as Record<string, unknown>, themes)
hotwords = composeHotwords([], setWords, setActive, themes, themeActive)
} catch { /* ignore */ }
const clientMeta = await bridge.getClientRuntimeInfo().catch(() => null)
// 使用独立的 WebSocket 连接进行重新识别,避免干扰 RecorderOrchestrator 的全局连接。
const { getWSUrl } = await import('@/services/runtimeConfig')
const wsUrl = getWSUrl()
const result = await new Promise<{ asrText: string; llmText: string; asrMs: number; llmMs: number; durationSec: number; asrEngine?: string; asrModel?: string }>((resolve, reject) => {
const timeout = window.setTimeout(() => {
if (!reprocessDone) { reprocessDone = true; provider.disconnect() }
reconnectProvider()
try { socket.close() } catch { /* ignore */ }
reject(new Error('重新识别超时'))
}, 60000)
}, 30_000) // ASR 最多 30 秒
void provider.connect({
onFinal: (r) => {
clearTimeout(timeout)
reprocessDone = true
provider.disconnect()
reconnectProvider()
resolve(r)
},
onError: (msg) => {
clearTimeout(timeout)
if (!reprocessDone) { reprocessDone = true; provider.disconnect() }
reconnectProvider()
reject(new Error(msg))
},
onDone: () => {
// 如果没有 final 就 done 了,说明没有结果
},
}).then(async () => {
const clientMeta = await bridge.getClientRuntimeInfo().catch(() => null)
provider.start({
systemPrompt: aiEnabled ? preset.systemPrompt : undefined,
disableAi: !aiEnabled,
clientMeta,
const socket = new WebSocket(wsUrl)
socket.binaryType = 'arraybuffer'
let resolved = false
socket.onopen = () => {
// 发送 start
const startMsg: Record<string, unknown> = {
cmd: 'start',
source: 'history_reprocess',
})
provider.sendAudio(chunk)
provider.stop()
}).catch((err) => {
clearTimeout(timeout)
reject(err)
})
disable_ai: !aiEnabled,
}
if (aiEnabled && preset.systemPrompt) startMsg.system_prompt = preset.systemPrompt
if (clientMeta) {
startMsg.client_meta = {
user_id: clientMeta.userId,
device_id: clientMeta.deviceId,
hostname: clientMeta.hostname,
client_version: clientMeta.clientVersion,
platform: clientMeta.platform,
os_version: clientMeta.osVersion,
local_ip: clientMeta.localIp,
system_locale: clientMeta.systemLocale,
cpu_cores: clientMeta.cpuCores,
memory_mb: clientMeta.memoryMb,
}
}
if (hotwords.length > 0) startMsg.hotwords = hotwords
socket.send(JSON.stringify(startMsg))
// 分片发送 PCM 数据
const CHUNK_SIZE = 32000
const totalBytes = chunk.byteLength
for (let offset = 0; offset < totalBytes; offset += CHUNK_SIZE) {
const end = Math.min(offset + CHUNK_SIZE, totalBytes)
socket.send(chunk.slice(offset, end))
}
// 发送 stop
socket.send(JSON.stringify({ cmd: 'stop' }))
}
socket.onmessage = (ev) => {
if (typeof ev.data !== 'string') return
try {
const msg = JSON.parse(ev.data)
console.log('[reprocess-diag] ws message:', msg.type, msg)
if (msg.type === 'final') {
resolved = true
clearTimeout(timeout)
socket.close()
resolve({
asrText: msg.asr_text || '',
llmText: msg.llm_text || '',
asrMs: msg.asr_ms || 0,
llmMs: msg.llm_ms || 0,
durationSec: Number(msg.duration_sec || 0),
asrEngine: msg.asr_engine || undefined,
asrModel: msg.asr_model || undefined,
})
} else if (msg.type === 'done' && !resolved) {
// 没有 final 就 done 了(后端判定为静音/无结果)
resolved = true
clearTimeout(timeout)
socket.close()
resolve({ asrText: '', llmText: '', asrMs: 0, llmMs: 0, durationSec: 0 })
} else if (msg.type === 'error') {
resolved = true
clearTimeout(timeout)
socket.close()
reject(new Error(msg.message || 'backend error'))
}
} catch { /* ignore parse errors */ }
}
socket.onerror = () => {
if (!resolved) {
resolved = true
clearTimeout(timeout)
reject(new Error('WebSocket 连接错误'))
}
}
socket.onclose = (ev) => {
console.log('[reprocess-diag] ws closed', { code: ev.code, reason: ev.reason, resolved })
if (!resolved) {
resolved = true
clearTimeout(timeout)
reject(new Error(`WebSocket 连接意外关闭 code=${ev.code}`))
}
}
})
// 极速模式下 llmText === asrText(后端未经 LLM 处理时直接复制 asrText)
@@ -161,14 +266,10 @@ export default function History() {
llmMs: result.llmMs,
charCount: (result.llmText || result.asrText).length,
isEmpty: !(result.llmText || result.asrText).trim(),
workMode: provider.mode,
aiProvider: provider.mode === 'server' ? 'server' : undefined,
aiModel: provider.mode === 'server' ? undefined : undefined,
asrProvider: provider.mode === 'local'
? String(await getSetting('localAsr.modelId', 'sensevoice-small'))
: provider.mode === 'cloud_api'
? resolveAsrDisplayModel(String(await getSetting('cloudAsr.provider', '')))
: (result.asrModel || result.asrEngine || 'server').replace(/^.*\//, ''),
workMode: 'server',
aiProvider: 'server',
aiModel: undefined,
asrProvider: (result.asrModel || result.asrEngine || 'server').replace(/^.*\//, ''),
})
void loadRecords(debouncedKeyword, visibleCount, favoriteOnly)
+18 -3
View File
@@ -28,16 +28,31 @@ export function close() {
// ─── Overlay ───
/**
* Bug 003 diagnostic helper — log overlay IPC failures to runtime events
* (which mirror to sayit.log via appendDebugLog).
*/
function logOverlayIpcError(op: string, err: unknown) {
try {
// Lazy import to avoid circular dep with debugLog
void import('./debugLog').then(({ addRuntimeEvent }) => {
addRuntimeEvent('error', 'overlay-ipc', `${op} failed`, { error: String(err) })
})
} catch {
console.error(`[overlay-ipc] ${op} failed:`, err)
}
}
export function showOverlay() {
invoke('show_overlay')
invoke('show_overlay').catch((err) => logOverlayIpcError('show_overlay', err))
}
export function hideOverlay() {
invoke('hide_overlay')
invoke('hide_overlay').catch((err) => logOverlayIpcError('hide_overlay', err))
}
export function updateOverlay(data: unknown) {
invoke('update_overlay_state', { data })
invoke('update_overlay_state', { data }).catch((err) => logOverlayIpcError('update_overlay_state', err))
}
// ─── Paste / Context ───
+5
View File
@@ -33,6 +33,11 @@ export async function refreshRecorderSettings() {
await orchestrator.refreshRuntimeSettings()
}
/** 仅刷新 overlay 显示设置(主题/长度/时长)— 轻量,避免触发全量录音缓存刷新 */
export async function refreshOverlaySettings() {
await orchestrator.refreshOverlaySettings()
}
/** 工作模式切换后重新连接 */
export function reconnectProvider() {
orchestrator.reconnectProvider()
@@ -316,6 +316,12 @@ export class RecorderOrchestrator {
this.ensureConnection()
}
/** 仅刷新 overlay 显示设置(主题/长度/时长),不触碰录音相关缓存。
* 用于外观设置页面,避免改个颜色就触发十几个 IPC 调用。 */
async refreshOverlaySettings() {
await this.overlayService.refreshSettings()
}
// ── State machine ──
private transition(to: RecorderState): boolean {
@@ -44,6 +44,9 @@ const PTT_LAB_VK = 0xA3
let pttCode = ''
let pttSetting = ''
let pttKeyDown = false
let hfCode = ''
let hfSetting = ''
let hfKeyDown = false
let labKeyDown = false
let labEnabled = false
let started = false
@@ -75,6 +78,15 @@ function handleKeyDown(e: KeyboardEvent) {
return
}
// 免提键(HF):在 keydown 时阻止默认行为,keyup 时触发 toggle
if (hfCode && e.code === hfCode && pttCode !== hfCode) {
hfKeyDown = true
if (isModifierSetting(hfSetting)) {
e.preventDefault()
}
return
}
// PTT Lab 键(右 Ctrl)
if (labEnabled && e.code === PTT_LAB_CODE && !labKeyDown) {
// 如果 PTT 主键也是右 Ctrl,不重复处理 lab
@@ -112,6 +124,20 @@ function handleKeyUp(e: KeyboardEvent) {
return
}
// 免提键:keyup 时 emit toggle-hands-free
if (hfCode && e.code === hfCode && hfKeyDown && pttCode !== hfCode) {
hfKeyDown = false
if (isModifierSetting(hfSetting)) {
e.preventDefault()
}
console.log('[webview-kb] toggle-hands-free (webview fallback)', { code: e.code, hfSetting })
emit('toggle-hands-free', {
source: 'webview_fallback',
vk: SETTING_TO_VK[hfSetting] || 0,
})
return
}
// PTT Lab 键
if (labEnabled && e.code === PTT_LAB_CODE && labKeyDown) {
if (pttCode === PTT_LAB_CODE) return
@@ -137,7 +163,18 @@ export async function refreshPTTSetting() {
}
pttCode = SETTING_TO_CODE[pttSetting] || ''
pttKeyDown = false
try {
const hfSettingVal = await getSetting('shortcutHandsFree', 'AltRight')
hfSetting = String(hfSettingVal || 'AltRight')
} catch {
hfSetting = 'AltRight'
}
hfCode = SETTING_TO_CODE[hfSetting] || ''
hfKeyDown = false
console.log('[webview-kb] PTT setting refreshed:', pttSetting, '→ code:', pttCode)
console.log('[webview-kb] HF setting refreshed:', hfSetting, '→ code:', hfCode)
}
/** PTT Lab 启用/禁用 */
+83
View File
@@ -0,0 +1,83 @@
# SayIt AI 润色供应商
SayIt 有两个核心功能:语音转文本(ASR)和 AI 文本润色。ASR 部分可以参考 [豆包语音识别凭证获取](豆包ASR凭据获取.md) 和 [阿里云千问语音识别](aliyun.md)。
AI 润色功能可以去除口癖词(嗯、啊、那个)、修正语音识别错误、自动分段排版等,都可以通过 Prompt 自定义控制。
SayIt 已对接国内常用的 AI 服务商(通义千问、DeepSeek、豆包等),同时支持 OpenAI 兼容格式,可以接入其他供应商或第三方中转。
## 1. 最佳实践
**推荐 DeepSeek deepseek-v4-flash 作为默认选择。** 质量最好、价格极低、速度可接受。
DeepSeek V4 Flash(2026 年 4 月发布)是 284B 参数的 MoE 模型,13B 激活参数,支持 1M 上下文。关闭思考模式后速度接近旧版 deepseek-chat,但质量明显更好,价格还更便宜(输入 1 元/百万 Token,输出 2 元/百万 Token)。
如果对延迟极度敏感(需要 <500ms),可以用通义千问 qwen3.6-flash,速度最快但质量略逊。
| 供应商    | 模型       | 短文本  | 长文本   | 每月成本 |
| --------------| -------------------| -----------| ------------| ----------|
| **DeepSeek** | deepseek-v4-flash | **803ms** | **2089ms** | ~1.5 元 |
| 通义千问   | qwen3.6-flash   | 393ms   | 1308ms   | ~3.8 元 |
> 月成本按每天 100 次、每次约 100 字估算。V4 Flash 默认开启 thinking,会导致延迟飙升到 2~7 秒。SayIt 已在代码中自动关闭思考模式,无需用户手动配置。
## 2. 如何选择AI模型
- **速度**:总耗时由 TTFT(Time To First Token,首个字生成时间)和 TPS(每秒 Token 数)决定。语音转写场景下,总耗时超过 1~2 秒会有明显等待感。
- **质量**:能否准确执行 Prompt 指令——去口癖、识别自我修正、格式化列表、纠正专有名词大小写等。
- **价格**:V4 Flash 的自动 prompt 缓存机制让重复 system prompt 的场景成本极低。
## 3. 实测数据
以下数据是 2026 年 5 月实测的,使用非流式请求(和 SayIt 实际调用方式一致),每条用例跑了 3 次取中位数。
### 速度
| 供应商  | 模型             | 短文本 | 中等文本 | 长文本(~200字) |
| ----------| -------------------------------| --------| ----------| ------------------|
| DeepSeek | deepseek-v4-flash(关闭思考) | 803ms | 971ms  | 2089ms      |
| 通义千问 | qwen3.6-flash         | 393ms | 489ms  | 1308ms      |
### 质量
两家都能正确去口癖、识别列表结构、纠正专有名词大小写(DeepSeek、ChatGPT、OpenAI 等)。V4 Flash 在专有名词纠正和格式化方面更稳定,输出更简洁。
## 4. 关于豆包
豆包的语音识别(ASR)是目前中文场景下最好的,SayIt 推荐使用豆包 ASR。但豆包的 AI 文本模型(lite 和 mini 都测了)延迟极不稳定,中长文本经常超过 10 秒,不适合语音转写的润色场景。建议 ASR 用豆包,AI 润色用 DeepSeek 或通义千问。
## 5. 价格参考
| 供应商  | 模型         | 输入(百万Token) | 输出(百万Token) |
| ----------| ----------------------| -------------------| -------------------|
| DeepSeek | deepseek-v4-flash  | 1 元       | 2 元       |
| 通义千问 | qwen3.6-flash    | 1.8 元      | 10.8 元      |
| 豆包   | doubao-seed-2-0-mini | 0.2 元      | 2 元       |
## 6. 配置指南
> 建议直接使用官方 API。第三方中转站延迟普遍更高。
### DeepSeek(推荐)
- API 地址:`https://api.deepseek.com`
- 模型:`deepseek-v4-flash`
- 获取 API Key:[platform.deepseek.com](https://platform.deepseek.com/api_keys)
- **注意**:SayIt 会自动在请求中添加 `thinking: {type: "disabled"}` 关闭思考模式以降低延迟
![alt text](DeepSeek获取API.png)
### 通义千问
- API 地址:`https://dashscope.aliyuncs.com/compatible-mode`
- 模型:`qwen3.6-flash`
- 获取 API Key:[百炼平台](https://bailian.console.aliyun.com/?spm=a2c4g.11186623.0.0.60905ec6iyaRqr&tab=model#/api-key)
![alt text](百炼获取API.png)
### 豆包(火山方舟)
- API 地址:`https://ark.cn-beijing.volces.com/api/v3`
- 模型:`doubao-seed-2-0-mini-260215`
![alt text](豆包获取API.png)
+4 -4
View File
@@ -12,10 +12,10 @@ AI 润色模型推荐使用 **deepseek-v4-flash** 质量好、价格低、速度
如果对延迟极度敏感(需要 <500ms),可以用通义千问 qwen3.6-flash,速度最快但准确度稍微差点。
| 供应商 | 模型 | 短文本 | 长文本(~200字) | 输入价格 | 输出价格 |
| ------------ | ----------------- | ------ | ---------------- | ---------------- | ----------------- |
| **DeepSeek** | deepseek-v4-flash | 803ms | 2089ms | 1 元/百万Token | 2 元/百万Token |
| 通义千问 | qwen3.6-flash | 393ms | 1308ms | 1.8 元/百万Token | 10.8 元/百万Token |
| 供应商    | 模型       | 短文本 | 长文本(~200字) | 输入价格     | 输出价格     |
| --------------| -------------------| --------| ------------------| ------------------| -------------------|
| **DeepSeek** | deepseek-v4-flash | 803ms | 2089ms      | 1 元/百万Token  | 2 元/百万Token  |
| 通义千问   | qwen3.6-flash   | 393ms | 1308ms      | 1.8 元/百万Token | 10.8 元/百万Token |
## 2. 配置指南
+95
View File
@@ -0,0 +1,95 @@
# SayIt ASR 语音识别配置
SayIt 支持三种语音识别模式:
- **云端 API**:调用云服务商的语音识别接口,准确率最高,适合大多数个人用户
- **本地推理**:使用本地模型离线识别,不需要网络,但准确率和速度不如云端
- **服务器模式**:自己部署后端服务,适合团队或有定制需求的场景
对个人用户来说,**云端 API 是最推荐的方式**,下面主要介绍这个。
## 1. 最佳实践
中文语音识别准确率最高的是**豆包流式语音识别 2.0**(Seed-ASR-2.0),其次是阿里的**千问 ASR**(qwen3-asr-flash)。
SayIt 对豆包和千问 ASR 都做了流式优化——按住说话时,音频会实时发送给服务端,松手后只需要等待最后一小段音频的处理时间,通常几百毫秒就能拿到结果。
另外 SayIt 还支持**千问 3.5 Omni** 多模态模型,这类模型原生支持音频输入、文本输出,可以在转录的同时执行 Prompt 指令(比如去口癖、翻译等),相当于 ASR + AI 润色一步到位。
| 供应商    | 模型           | 特点           | 价格          |
| --------------| --------------------------| --------------------------| -------------------------|
| **豆包**   | Seed-ASR-2.0       | 中文准确率最高,推荐首选 | 1.00 元/小时      |
| **阿里千问** | qwen3-asr-flash-realtime | 流式识别,速度极快    | 1.19 元/小时      |
| **阿里千问** | qwen3.5-omni-flash    | ASR + AI 一步到位    | 按 Token 计费(见下方) |
## 2. 使用豆包语音识别
SayIt 支持使用字节跳动「豆包流式语音识别模型 2.0」作为语音识别引擎。豆包 ASR 是目前中文语音识别准确率最高的模型,推荐使用。
1. 注册 / 登录火山引擎
打开[火山引擎语音服务控制台](https://console.volcengine.com/speech/service/10038),用手机号注册或登录即可。已有账号的直接登录,没有账号会引导你先注册。
[图片]
2. 完成实名认证
火山引擎的 API 服务需要先做实名认证,没认证的话后面创建应用会卡住。进入[实名认证页面](https://console.volcengine.com/user/authentication/detail/),按页面提示走完就行。
[图片]
3. 创建应用并获取 APP ID
进入[应用管理页面](https://console.volcengine.com/speech/app1),点击「创建应用」。填写应用名称和简介(随便写就行),关键一步:一定要勾选「豆包流式语音识别模型 2.0 小时版」,不然后面用不了。
[图片]
4. 获取APP ID 和 Access Token
进入 [API 服务中心页面](https://console.volcengine.com/speech/service/10038),点击「豆包流式语音识别模型2.0」(配图是旧版界面)。页面上会显示你的 APP ID,鼠标放到 Access Token 上面,点击小眼睛图标,可以看到 Access Token 信息。
[图片]
记录下来这两个信息填到 SayIt 软件中。选择 语音引擎 - 云 API 模式 - 选择ASR供应商为豆包 ASR,填写凭证信息后可以保存测试。
[图片]
豆包开通试用后,半年内有 20 小时的[免费额度](https://www.volcengine.com/docs/6561/1359369?lang=zh)。超出免费额度后,按量计费:豆包流式语音识别模型 2.0 — 1 元/小时。[后付费价格详情](https://www.volcengine.com/docs/6561/1359370?lang=zh)
## 3. 阿里千问 ASR
阿里百炼平台提供了两个语音识别模型,都可以在 SayIt 中使用。
**qwen3-asr-flash / qwen3-asr-flash-realtime**
纯语音识别模型,中文准确率仅次于豆包。有两个版本:
- **qwen3-asr-flash**:非流式,录完再发,速度也很快
- **qwen3-asr-flash-realtime**:流式,边录边发,松手后几乎立刻出结果,体验极佳
两者准确率一样,推荐使用流式版本。适合搭配独立的 AI 润色供应商使用(比如通义千问或 DeepSeek 做文本校对)。
**qwen3.5-omni-flash**
多模态模型,可以直接接收音频输入并输出文本。和普通 ASR + AI 润色的两步流程不同,Omni 模型一步完成,可以在 System Prompt 中直接控制输出格式(比如去口癖、翻译、列表排版等)。
适合不想分别配置 ASR 和 AI 润色的用户,配置更简单。
**配置方式**
在 SayIt 设置中选择千问 ASR 系列,需要填写 API Key。
- 获取 API Key:[阿里百炼平台](https://bailian.console.aliyun.com/cn-beijing?tab=model#/api-key)
## 4. 价格
下面是各个模型按需使用的价格参考。
| 模型                 | 价格      | 免费额度        |
| ---------------------------------------| -----------------| ------------------------|
| 豆包 Seed-ASR-2.0           | 1.00 元/小时  | 20 小时(半年内)   |
| 千问 qwen3-asr-flash(非流式)    | 0.79 元/小时  | 10 小时(90天内)   |
| 千问 qwen3-asr-flash-realtime(流式) | 1.19 元/小时  | 10 小时(90天内)   |
| 千问 qwen3.5-omni-flash        | 约 2.3 元/小时* | 100 万 Token(90天内) |
*Omni 按 Token 计费(每秒音频 ≈ 25 token),2.3 元为粗略估算。详见[千问价格文档](https://help.aliyun.com/zh/model-studio/model-pricing#95f0464a10q5c)。
+94 -11
View File
@@ -61,7 +61,7 @@ class ASREngine:
self._model = Qwen3ASRModel.LLM(
model=config.asr.model,
gpu_memory_utilization=config.asr.vllm_gpu_util,
max_new_tokens=512,
max_new_tokens=2048,
max_model_len=config.asr.vllm_max_model_len,
)
self._backend = "vllm"
@@ -70,7 +70,7 @@ class ASREngine:
import torch
self._model = Qwen3ASRModel.from_pretrained(
config.asr.model, dtype=torch.bfloat16,
device_map=config.asr.device, max_new_tokens=512,
device_map=config.asr.device, max_new_tokens=2048,
)
self._backend = "transformers"
logger.info("Qwen3-ASR ready (%s)", self._backend)
@@ -235,15 +235,16 @@ class ASREngine:
job_text_parts[idx] = []
continue
if duration <= 120.0:
# ≤ 2min: send whole audio, no split — accuracy first.
# VAD-based filtering was removed because it aggressively
# discards whispered speech (low-energy audio misclassified
# as silence), causing severe recognition loss.
if duration <= 60.0:
# ≤ 1min: send whole audio, no split — accuracy first.
segs = [(audio, duration)]
else:
# > 2min: safe split at ≥3s silence gaps
# > 1min: safe split at silence gaps
segs = self._safe_split(audio)
# Merge segments with very low speech ratio into adjacent segments
# to avoid batch inference failure from near-silent segments.
if len(segs) > 1:
segs = self._merge_silent_segments(segs)
logger.info("Safe split %.1fs -> %d segs %s",
duration, len(segs), [f"{d:.1f}s" for _, d in segs])
job_vad_ms[idx] = int((time.monotonic() - vad_t0) * 1000)
@@ -408,7 +409,55 @@ class ASREngine:
logger.exception("VAD failed")
return [(audio, len(audio) / 16000.0)] if fallback_to_full else []
def _safe_split(self, audio: np.ndarray, min_gap: float = 3.0, min_seg: float = 15.0, max_seg: float = 240.0) -> list[tuple[np.ndarray, float]]:
def _merge_silent_segments(self, segs: list[tuple[np.ndarray, float]], min_speech_ratio: float = 0.15) -> list[tuple[np.ndarray, float]]:
"""Merge segments with very low speech ratio into adjacent segments.
Near-silent segments in a batch can cause vLLM to return empty results
for the entire batch. Merging them into neighbors avoids this while
preserving any speech content they may contain.
"""
if not self._vad or len(segs) <= 1:
return segs
import torch
# Identify which segments have low speech ratio
speech_ok = []
for seg_audio, seg_dur in segs:
try:
res = self._vad.generate(input=torch.tensor(seg_audio, dtype=torch.float32), cache={}, is_final=True)
vad_segs = res[0].get("value", []) if res else []
speech_time = sum((e - s) for s, e in vad_segs) / 1000.0 if vad_segs else 0
speech_ok.append(speech_time / seg_dur >= min_speech_ratio if seg_dur > 0 else True)
except Exception:
speech_ok.append(True) # on error, assume it's fine
if all(speech_ok):
return segs
# Merge low-speech segments into adjacent (prefer previous, fallback to next)
merged: list[tuple[np.ndarray, float]] = []
i = 0
while i < len(segs):
if speech_ok[i]:
merged.append(segs[i])
i += 1
else:
# Merge into previous segment if available, otherwise next
if merged:
prev_audio, prev_dur = merged[-1]
cur_audio, cur_dur = segs[i]
merged[-1] = (np.concatenate([prev_audio, cur_audio]), prev_dur + cur_dur)
elif i + 1 < len(segs):
cur_audio, cur_dur = segs[i]
nxt_audio, nxt_dur = segs[i + 1]
merged.append((np.concatenate([cur_audio, nxt_audio]), cur_dur + nxt_dur))
speech_ok[i + 1] = True # skip next since we consumed it
i += 1
else:
merged.append(segs[i])
i += 1
return merged
def _safe_split(self, audio: np.ndarray, min_gap: float = 1.0, min_seg: float = 10.0, max_seg: float = 100.0) -> list[tuple[np.ndarray, float]]:
"""Split audio at long silence gaps (≥ min_gap seconds), keeping ALL audio.
Unlike _vad_split which discards silence, this preserves the full audio
@@ -443,7 +492,41 @@ class ASREngine:
if all(d >= min_seg for d in seg_durs) and all(d <= max_seg for d in seg_durs):
cuts.append(mid)
if not cuts:
if not cuts and dur > 100.0:
# Fallback: force split at lowest energy point near ideal boundaries
ideal_seg = 60.0 # target segment length
n_cuts_needed = max(1, int(dur / ideal_seg) - 1)
frame_len = int(0.05 * 16000) # 50ms frames
hop = frame_len // 2
n_frames = (len(audio) - frame_len) // hop
if n_frames > 0:
energy = np.array([np.sum(audio[i*hop:i*hop+frame_len]**2) for i in range(n_frames)])
# Smooth energy
kernel = np.ones(20) / 20
if len(energy) > len(kernel):
energy = np.convolve(energy, kernel, mode='same')
forced_cuts = []
for c in range(1, n_cuts_needed + 1):
target_time = dur * c / (n_cuts_needed + 1)
# Search window: +/- 10s around target
win_start = max(0, int((target_time - 10) * 16000 / hop))
win_end = min(n_frames, int((target_time + 10) * 16000 / hop))
if win_start < win_end:
min_idx = win_start + np.argmin(energy[win_start:win_end])
cut_time = min_idx * hop / 16000.0
forced_cuts.append(cut_time)
if forced_cuts:
forced_cuts.sort()
points = [0.0] + forced_cuts + [dur]
segments = []
for i in range(len(points) - 1):
s, e = int(points[i]*16000), int(points[i+1]*16000)
seg = audio[s:e]
segments.append((seg, len(seg)/16000.0))
logger.info("Energy fallback split %.1fs -> %d segs %s", dur, len(segments), [f"{d:.1f}s" for _,d in segments])
return segments
return [(audio, dur)]
elif not cuts:
return [(audio, dur)]
cuts.sort()
@@ -467,7 +550,7 @@ class ASREngine:
debug = {"duration_sec": round(duration, 1), "vad_segments": [], "context": ctx}
if self._engine_type == "firered":
if duration <= 15.0:
if duration <= 120.0:
debug["vad_segments"] = [{"index": 0, "duration_sec": round(duration, 1)}]
return self._firered_backend.transcribe_audio(audio), debug
segments = self._vad_split(audio)
+30 -7
View File
@@ -220,15 +220,15 @@ def _is_hotword_hallucination(text: str, context: str, audio_peak: float = 1.0,
audio_dur: float = 0.0, voice_score: float = 1.0) -> bool:
if not context or not text:
return False
words = set(context.split())
words = set(w.lower() for w in context.split())
cleaned = re.sub(r"[\s銆傦紝銆侊紵锛?,?!\u3000。]+", " ", text).strip()
tokens = cleaned.split()
if not tokens:
return False
matched = sum(1 for token in tokens if token in words)
matched = sum(1 for token in tokens if token.lower() in words)
consec = max_consec = 0
for token in tokens:
if token in words:
if token.lower() in words:
consec += 1
max_consec = max(max_consec, consec)
else:
@@ -239,7 +239,7 @@ def _is_hotword_hallucination(text: str, context: str, audio_peak: float = 1.0,
return True
# Short result composed entirely of hotwords → almost certainly hallucination.
# Real users don't speak just a single hotword and stop.
if len(cleaned) <= 6 and all(t in words for t in tokens):
if len(cleaned) <= 6 and all(t.lower() in words for t in tokens):
return True
return False
@@ -372,11 +372,28 @@ async def _run_asr_llm(
# Strip hotword context when audio has no detectable speech energy,
# preventing the ASR model from hallucinating hotwords on noise.
# Whispered speech has voice_score > 0.50; noise/silence < 0.15.
# Three-layer detection: voice_score, energy level, and VAD.
audio_peak = float(np.max(np.abs(audio)))
audio_rms = float(np.sqrt(np.mean(audio**2)))
vs = _voice_score(audio)
if vs < _VS_THRESHOLD and (context is None or context):
logger.info("Stripping hotword context: voice_score=%.3f peak=%.4f", vs, audio_peak)
strip_context = False
if vs < _VS_THRESHOLD or (audio_peak < 0.02 and audio_rms < 0.002):
strip_context = True
elif context and duration < 10.0 and asr_engine and asr_engine._vad is not None:
# For short audio with hotwords: use VAD to detect if there's actual speech.
# This catches non-speech noise (keyboard, clicks) that has high energy.
import torch
try:
vad_res = asr_engine._vad.generate(
input=torch.tensor(audio, dtype=torch.float32), cache={}, is_final=True)
vad_segs = vad_res[0].get("value", []) if vad_res else []
logger.debug("VAD check: segs=%d, duration=%.1f", len(vad_segs), duration)
if not vad_segs:
strip_context = True
except Exception as e:
logger.warning("VAD check failed: %s", e)
if strip_context and (context is None or context):
logger.info("Stripping hotword context: voice_score=%.3f peak=%.4f rms=%.5f", vs, audio_peak, audio_rms)
context = ""
raw_text, asr_ms, asr_debug = await asr_engine.transcribe(audio, context=context, language=language)
@@ -384,6 +401,12 @@ async def _run_asr_llm(
asr_debug["voice_score"] = round(vs, 3)
if _is_noise(asr_text, duration, voice_score=vs) or _is_hotword_hallucination(asr_text, asr_debug.get("context", ""), audio_peak=audio_peak, audio_dur=duration, voice_score=vs):
asr_text = ""
# Low energy audio producing short text is almost certainly noise/hallucination
if asr_text and audio_peak < 0.02 and audio_rms < 0.002 and len(asr_text) <= 10:
asr_text = ""
# VAD detected no speech but ASR produced short text → noise hallucination
if asr_text and strip_context and len(asr_text) <= 10:
asr_text = ""
if cfg.logging.slow_asr_ms and asr_ms >= cfg.logging.slow_asr_ms:
logger.warning("Slow ASR detected duration_sec=%.2f asr_ms=%d", duration, asr_ms)
+1 -1
View File
@@ -31,7 +31,7 @@ asr:
language: "Chinese" # Chinese / English / Cantonese
itn: true # Inverse text normalization
vllm_gpu_util: 0.5
vllm_max_model_len: 8192
vllm_max_model_len: 16384
max_concurrency: 1
batch_max_size: 32
batch_wait_ms: 60