diff --git a/openless-all/app/src-tauri/src/asr/mod.rs b/openless-all/app/src-tauri/src/asr/mod.rs index ce1a5d48..41e49308 100644 --- a/openless-all/app/src-tauri/src/asr/mod.rs +++ b/openless-all/app/src-tauri/src/asr/mod.rs @@ -13,6 +13,7 @@ pub mod local; pub mod mimo; pub mod pcm; pub mod qwen_realtime; +pub mod stepfun_realtime; pub mod volcengine; pub mod wav; pub mod whisper; @@ -22,6 +23,7 @@ pub use dashscope_multimodal::DashScopeMultimodalASR; pub use elevenlabs::ElevenLabsBatchASR; pub use mimo::MimoBatchASR; pub use qwen_realtime::{Qwen3RealtimeASR, Qwen3RealtimeCredentials}; +pub use stepfun_realtime::{StepfunRealtimeASR, StepfunRealtimeCredentials}; pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR}; pub use whisper::WhisperBatchASR; diff --git a/openless-all/app/src-tauri/src/asr/qwen_realtime.rs b/openless-all/app/src-tauri/src/asr/qwen_realtime.rs index 577f61cd..8920b40d 100644 --- a/openless-all/app/src-tauri/src/asr/qwen_realtime.rs +++ b/openless-all/app/src-tauri/src/asr/qwen_realtime.rs @@ -576,7 +576,8 @@ fn drain_audio_chunks(buffer: &mut Vec) -> Vec> { } /// VAD 句段拼接:CJK 之间直接相连;拉丁词之间补空格,避免英文句段黏连。 -fn join_segments(segments: &[String]) -> String { +/// `stepfun_realtime` 的多句段收尾复用同一套拼接逻辑,故 `pub(crate)`。 +pub(crate) fn join_segments(segments: &[String]) -> String { let mut joined = String::new(); for seg in segments.iter().map(|s| s.trim()) { if seg.is_empty() { diff --git a/openless-all/app/src-tauri/src/asr/stepfun_realtime.rs b/openless-all/app/src-tauri/src/asr/stepfun_realtime.rs new file mode 100644 index 00000000..3f336698 --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/stepfun_realtime.rs @@ -0,0 +1,1089 @@ +//! 阶跃星辰 StepAudio 实时 ASR 客户端(`wss://api.stepfun.com/v1/realtime/asr/stream`)。 +//! +//! 与 `qwen_realtime.rs` 同为 OpenAI Realtime 风格 WS,但四处关键差异 +//! (2026-07-16 真实接口逐项实测确认): +//! +//! - **模型在 `session.update` 里传**(`session.audio.input.transcription.model`), +//! 不是 URL query;session 配置是 `audio.input.{format,transcription,turn_detection}` +//! 的嵌套形状。 +//! - **`delta` 的 `text`/`stash` 是拼接关系**:`text` 是已确定前缀、`stash` 是 +//! 未定尾巴,当前句段全文 = `text + stash`(Qwen 是两者互斥取非空)。 +//! - **没有服务端结束事件**:`session.finish` 回 `transcript.response.error`(不支持); +//! server_vad 模式下 `input_audio_buffer.commit` 被静默忽略。唯一可靠的收尾是 +//! **补送 ≥silence_duration_ms 的静音帧逼 VAD 关段**——speech_stopped 后 ~0.4s +//! 吐出该句段的 `completed`,随后客户端自行断开。 +//! - `prompt` 字段在 transcription 配置里被接受(批式 /audio/transcriptions 则 +//! 静默忽略 prompt、只认 hotwords——两条通道词汇偏置方式相反)。 + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use base64::Engine; +use futures_util::{SinkExt, StreamExt}; +use parking_lot::Mutex as ParkingMutex; +use serde_json::{json, Value}; +use tokio::net::TcpStream; +use tokio::runtime::Handle; +use tokio::sync::{mpsc, oneshot, Mutex as AsyncMutex, Notify}; +use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tokio_tungstenite::tungstenite::http::header::HeaderValue; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use uuid::Uuid; + +use super::qwen_realtime::join_segments; +use super::{AudioConsumer, RawTranscript}; + +/// 内部 effective id(`resolve_effective_asr_provider` 按模型名从 `stepfun` +/// 路由到这里),不出现在设置页 preset 列表里。 +pub const PROVIDER_ID: &str = "stepfun-realtime"; +pub const DEFAULT_ENDPOINT: &str = "wss://api.stepfun.com/v1/realtime/asr/stream"; +pub const DEFAULT_MODEL: &str = "stepaudio-2.5-asr-stream"; +/// 实时 WS 在 base URL 下的固定路径(从批式共用的 https base 派生 wss URL 用)。 +const REALTIME_PATH: &str = "/realtime/asr/stream"; + +/// 100 ms of 16 kHz / 16-bit / mono PCM,与 recorder 输出一致。 +pub const TARGET_AUDIO_CHUNK_BYTES: usize = 3_200; +const BYTES_PER_MS: u64 = 32; +const FINAL_RESULT_TIMEOUT: Duration = Duration::from_secs(12); +const SESSION_READY_TIMEOUT: Duration = Duration::from_secs(5); +const WRITE_TIMEOUT: Duration = Duration::from_secs(5); +/// server_vad 断句静默阈值,与 qwen_realtime 取齐(500ms 降低换气误切概率)。 +const VAD_SILENCE_DURATION_MS: u32 = 500; +/// 收尾补送的静音时长:必须 > VAD_SILENCE_DURATION_MS 并留网络余量, +/// 否则 VAD 不关段、最后一句永远等不到 completed(协议无 finish 事件)。 +const SILENCE_TAIL_MS: u64 = 700; +/// 静音送出后等待「无未关句段」的宽限期:纯静音会话(连接检查、误触)没有任何 +/// speech_started,宽限到点即以空文本成功返回。 +const FINISH_GRACE: Duration = Duration::from_millis(1_200); + +type WsStream = WebSocketStream>; +type WsSink = futures_util::stream::SplitSink; +type SharedWriter = Arc>>; + +#[derive(Clone, Debug)] +pub struct StepfunRealtimeCredentials { + pub api_key: String, + /// 允许三种形态:空(默认网关)、批式共用的 `https://api.stepfun.com/v1` + /// (自动派生 wss 路径)、完整 `wss://` URL(原样使用)。 + pub endpoint: String, + pub model: String, + /// 用户词典拼成的 prompt(实时协议接受 transcription.prompt;批式则相反, + /// 只认 hotwords)。None = 不发。 + pub prompt: Option, +} + +impl StepfunRealtimeCredentials { + pub fn normalized_model(&self) -> String { + let model = self.model.trim(); + if model.is_empty() { + DEFAULT_MODEL.to_string() + } else { + model.to_string() + } + } + + /// 连接 URL:`wss://` 开头原样用;`http(s)://` base(与批式共用的凭据槽) + /// 换 scheme 并补 `/realtime/asr/stream` 路径;解析失败/空值回默认网关。 + pub fn connect_url(&self) -> String { + let endpoint = self.endpoint.trim(); + if endpoint.is_empty() { + return DEFAULT_ENDPOINT.to_string(); + } + if endpoint.starts_with("wss://") || endpoint.starts_with("ws://") { + return endpoint.trim_end_matches('/').to_string(); + } + let Ok(mut url) = url::Url::parse(endpoint) else { + return DEFAULT_ENDPOINT.to_string(); + }; + if url.set_scheme("wss").is_err() { + return DEFAULT_ENDPOINT.to_string(); + } + let path = url.path().trim_end_matches('/').to_string(); + if !path.ends_with(REALTIME_PATH) { + url.set_path(&format!("{path}{REALTIME_PATH}")); + } + url.set_query(None); + url.set_fragment(None); + url.to_string() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum StepfunASRError { + #[error("credentials missing")] + CredentialsMissing, + #[error("connection failed: {0}")] + ConnectionFailed(String), + #[error("send failed: {0}")] + SendFailed(String), + #[error("task failed: {0}")] + TaskFailed(String), + #[error("no final result")] + NoFinalResult, + #[error("final result timed out")] + FinalResultTimeout, +} + +enum SendItem { + Audio { + chunk: Vec, + contains_non_silent_audio: bool, + /// `send_last_frame` 用它确认尾帧已由写 worker 实际写入 WebSocket。 + written_tx: Option>>, + }, +} + +#[derive(Default)] +struct SyncState { + pending_audio: Vec, + audio_scratch: Vec, + bytes_received: u64, + /// 最近一次非静音 PCM 由写 worker 成功写入 WebSocket 的时刻。 + last_non_silent_audio_written_at: Option, + /// 最近一次服务端 completed 事件到达的时刻。 + last_completed_at: Option, + session_started: bool, + session_finished: bool, + session_start_error: Option, + runtime: Option, + start: Option, + final_tx: Option>>, + send_tx: Option>, + /// VAD 断句后按到达顺序累积的已完成句段(completed.transcript)。 + completed_segments: Vec, + /// 当前开放句段的 interim 全文 = delta.text(已确定前缀)+ delta.stash + /// (未定尾巴);completed 到达后清空。 + partial_text: String, + /// speech_started 开、completed 关的未收尾句段计数。收尾判据:finishing + /// 且归零(静音尾帧已逼 VAD 关掉所有段)。 + open_segments: u32, + /// send_last_frame 已冲刷尾音频 + 静音帧,进入等待句段归零阶段。 + finishing: bool, + /// 尾帧尚未由写 worker 确认写入 WebSocket,不能被旧句段的 completed 抢先收尾。 + tail_write_pending: bool, +} + +pub struct StepfunRealtimeASR { + credentials: StepfunRealtimeCredentials, + state: ParkingMutex, + writer: SharedWriter, + final_rx: ParkingMutex>>>, + session_started: Arc, + session_finished: Arc, +} + +impl StepfunRealtimeASR { + pub fn new(credentials: StepfunRealtimeCredentials) -> Self { + Self { + credentials, + state: ParkingMutex::new(SyncState::default()), + writer: Arc::new(AsyncMutex::new(None)), + final_rx: ParkingMutex::new(None), + session_started: Arc::new(Notify::new()), + session_finished: Arc::new(Notify::new()), + } + } + + pub async fn open_session(self: &Arc) -> Result<(), StepfunASRError> { + if self.credentials.api_key.trim().is_empty() { + return Err(StepfunASRError::CredentialsMissing); + } + + let url = self.credentials.connect_url(); + let mut request = url + .into_client_request() + .map_err(|e| StepfunASRError::ConnectionFailed(e.to_string()))?; + request.headers_mut().insert( + "Authorization", + HeaderValue::from_str(&format!("Bearer {}", self.credentials.api_key.trim())) + .map_err(|e| StepfunASRError::ConnectionFailed(e.to_string()))?, + ); + + let (ws, _resp) = connect_async(request) + .await + .map_err(|e| StepfunASRError::ConnectionFailed(e.to_string()))?; + let (write, read) = ws.split(); + *self.writer.lock().await = Some(write); + + let (final_tx, final_rx) = oneshot::channel(); + let (send_tx, mut send_rx) = mpsc::unbounded_channel::(); + { + let mut st = self.state.lock(); + *st = SyncState::default(); + st.runtime = Some(Handle::current()); + st.start = Some(Instant::now()); + st.final_tx = Some(final_tx); + st.send_tx = Some(send_tx); + } + *self.final_rx.lock() = Some(final_rx); + + let writer_for_worker = Arc::clone(&self.writer); + let weak_self_for_worker = Arc::downgrade(self); + tokio::spawn(async move { + while let Some(SendItem::Audio { + chunk, + contains_non_silent_audio, + written_tx, + }) = send_rx.recv().await + { + match send_text(&writer_for_worker, append_audio_message(&chunk)).await { + Ok(()) => { + if contains_non_silent_audio { + if let Some(this) = weak_self_for_worker.upgrade() { + this.mark_non_silent_audio_written(); + } + } + if let Some(tx) = written_tx { + let _ = tx.send(Ok(())); + } + } + Err(error) => { + if let Some(tx) = written_tx { + let _ = tx.send(Err(error.to_string())); + } + log::error!("[stepfun-asr] audio frame send failed: {error}"); + if let Some(this) = weak_self_for_worker.upgrade() { + this.finish_error(error); + } + break; + } + } + } + }); + + let weak_self = Arc::downgrade(self); + tokio::spawn(async move { + let mut read = read; + while let Some(msg) = read.next().await { + let Some(this) = weak_self.upgrade() else { + break; + }; + match msg { + Ok(Message::Text(text)) => { + if !this.handle_text_message(&text) { + break; + } + } + Ok(Message::Close(_)) => { + this.fail_session_start( + "websocket closed before session configuration completed", + ); + this.finish_with_partial_or_error(StepfunASRError::NoFinalResult); + break; + } + Ok(_) => {} + Err(e) => { + log::error!("[stepfun-asr] receive loop error: {e}"); + this.fail_session_start(&e.to_string()); + this.finish_with_partial_or_error(StepfunASRError::ConnectionFailed( + e.to_string(), + )); + break; + } + } + } + }); + + let started = self.session_started.notified(); + tokio::pin!(started); + started.as_mut().enable(); + let update = session_update_message( + &self.credentials.normalized_model(), + self.credentials.prompt.as_deref(), + ); + if let Err(error) = send_text(&self.writer, update).await { + self.cancel(); + return Err(error); + } + let ready_result = if !self.state.lock().session_started { + tokio::time::timeout(SESSION_READY_TIMEOUT, started) + .await + .map_err(|_| StepfunASRError::FinalResultTimeout) + } else { + Ok(()) + }; + if let Err(error) = ready_result { + self.cancel(); + return Err(error); + } + if let Some(error) = self.state.lock().session_start_error.clone() { + self.cancel(); + return Err(StepfunASRError::TaskFailed(error)); + } + + Ok(()) + } + + /// 冲刷尾音频 + 补送静音帧逼 VAD 关掉所有开放句段,等全部 completed 到齐。 + /// + /// 协议没有 finish 事件(见模块注释),完成判据由客户端状态机给出: + /// `finishing && open_segments == 0`。纯静音会话(无任何 speech_started) + /// 由 FINISH_GRACE 宽限兜底,以空文本成功返回。 + pub async fn send_last_frame(self: &Arc) -> Result<(), StepfunASRError> { + let result = tokio::time::timeout(FINAL_RESULT_TIMEOUT, async { + let finished = self.session_finished.notified(); + tokio::pin!(finished); + finished.as_mut().enable(); + let (send_tx, tail) = { + let mut st = self.state.lock(); + let send_tx = st.send_tx.clone(); + if !st.pending_audio.is_empty() { + let pending = std::mem::take(&mut st.pending_audio); + st.audio_scratch.extend_from_slice(&pending); + } + let mut tail = std::mem::take(&mut st.audio_scratch); + // 尾音频和静音帧合并成一次 append,减少一次写。 + tail.resize(tail.len() + (SILENCE_TAIL_MS * BYTES_PER_MS) as usize, 0); + st.finishing = true; + st.tail_write_pending = send_tx.is_some(); + (send_tx, tail) + }; + let Some(send_tx) = send_tx else { + return Ok(()); + }; + let (written_tx, written_rx) = oneshot::channel(); + let contains_non_silent_audio = contains_non_silent_pcm(&tail); + send_tx + .send(SendItem::Audio { + chunk: tail, + contains_non_silent_audio, + written_tx: Some(written_tx), + }) + .map_err(|_| { + StepfunASRError::SendFailed("audio writer is not available".to_string()) + })?; + written_rx + .await + .map_err(|_| { + StepfunASRError::SendFailed( + "audio writer stopped before the tail frame was sent".to_string(), + ) + })? + .map_err(StepfunASRError::SendFailed)?; + self.state.lock().tail_write_pending = false; + + // 宽限任务:仅当最后一个 completed 之后没有新的非静音帧写入时才可收尾。 + // 这既允许松手前已 completed 的正常会话快速返回,也避免异步写入或服务端 + // VAD 延迟把新的真实音频误判为空会话。 + let weak = Arc::downgrade(self); + tokio::spawn(async move { + tokio::time::sleep(FINISH_GRACE).await; + if let Some(this) = weak.upgrade() { + let should_finish = { + let st = this.state.lock(); + !st.session_finished + && !has_audio_after_last_completed(&st) + && st.open_segments == 0 + }; + if should_finish { + this.finish_success(); + } + } + }); + + if !self.state.lock().session_finished { + finished.await; + } + Ok(()) + }) + .await; + match result { + Ok(inner) => inner, + Err(_) => { + // 超时兜底:有部分结果就带出去,没有才报错——与断连路径一致。 + self.finish_with_partial_or_error(StepfunASRError::FinalResultTimeout); + Ok(()) + } + } + } + + pub async fn await_final_result(&self) -> Result { + let rx = self.final_rx.lock().take(); + let Some(rx) = rx else { + return Err(StepfunASRError::NoFinalResult); + }; + tokio::time::timeout(FINAL_RESULT_TIMEOUT, rx) + .await + .map_err(|_| StepfunASRError::FinalResultTimeout)? + .map_err(|_| StepfunASRError::NoFinalResult)? + } + + pub fn cancel(&self) { + let mut st = self.state.lock(); + st.pending_audio.clear(); + st.audio_scratch.clear(); + st.send_tx.take(); + st.final_tx.take(); + st.session_finished = true; + drop(st); + let writer = Arc::clone(&self.writer); + if let Ok(handle) = Handle::try_current() { + handle.spawn(async move { + let _ = close_writer(&writer).await; + }); + } else { + std::thread::spawn(move || { + if let Ok(rt) = tokio::runtime::Runtime::new() { + rt.block_on(async move { + let _ = close_writer(&writer).await; + }); + } + }); + } + } + + fn handle_text_message(&self, text: &str) -> bool { + let value: Value = match serde_json::from_str(text) { + Ok(v) => v, + Err(e) => { + log::warn!("[stepfun-asr] invalid json event: {e}"); + return true; + } + }; + let event = value + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + match event { + "session.updated" => { + self.mark_session_started(); + true + } + "input_audio_buffer.speech_started" => { + self.state.lock().open_segments += 1; + true + } + "conversation.item.input_audio_transcription.delta" => { + self.record_partial(&value); + true + } + "conversation.item.input_audio_transcription.completed" => { + self.record_completed(&value) + } + "conversation.item.input_audio_transcription.failed" => { + let item_id = value + .get("item_id") + .and_then(Value::as_str) + .unwrap_or("unknown item"); + let message = value + .get("error") + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .or_else(|| value.get("message").and_then(Value::as_str)) + .unwrap_or("audio transcription failed"); + self.finish_error(StepfunASRError::TaskFailed(format!("{item_id}: {message}"))); + false + } + // `transcript.response.error` 是 StepFun 特有的请求级错误事件 + // (实测发不支持的 session.finish 时返回),与通用 `error` 同处理。 + "error" | "transcript.response.error" => { + let message = value + .get("error") + .and_then(|e| e.get("message")) + .and_then(Value::as_str) + .or_else(|| value.get("message").and_then(Value::as_str)) + .unwrap_or("realtime session error") + .to_string(); + self.finish_with_partial_or_error(StepfunASRError::TaskFailed(message)); + false + } + _ => true, + } + } + + fn mark_session_started(&self) { + let (send_tx, chunks) = { + let mut st = self.state.lock(); + st.session_started = true; + if !st.pending_audio.is_empty() { + let pending = std::mem::take(&mut st.pending_audio); + st.audio_scratch.extend_from_slice(&pending); + } + let send_tx = st.send_tx.clone(); + let chunks = drain_audio_chunks(&mut st.audio_scratch); + (send_tx, chunks) + }; + if let Some(tx) = send_tx { + for chunk in chunks { + let _ = tx.send(SendItem::Audio { + contains_non_silent_audio: contains_non_silent_pcm(&chunk), + chunk, + written_tx: None, + }); + } + } + self.session_started.notify_waiters(); + } + + fn fail_session_start(&self, error: &str) { + let mut st = self.state.lock(); + if !st.session_started && st.session_start_error.is_none() { + st.session_start_error = Some(error.to_string()); + self.session_started.notify_waiters(); + } + } + + fn record_partial(&self, value: &Value) { + // `text` 是已确定前缀、`stash` 是未定尾巴,二者拼接即当前句段全文 + // (与 Qwen 的互斥语义不同,见模块注释)。两者皆空不覆盖已有 partial。 + let confirmed = value.get("text").and_then(Value::as_str).unwrap_or(""); + let stash = value.get("stash").and_then(Value::as_str).unwrap_or(""); + let combined = format!("{confirmed}{stash}"); + let combined = combined.trim(); + if !combined.is_empty() { + self.state.lock().partial_text = combined.to_string(); + } + } + + /// 返回 false 表示会话已收尾、读循环可退出。 + fn record_completed(&self, value: &Value) -> bool { + let transcript = value + .get("transcript") + .and_then(Value::as_str) + .unwrap_or(""); + let should_finish = { + let mut st = self.state.lock(); + let trimmed = transcript.trim(); + if !trimmed.is_empty() { + st.completed_segments.push(trimmed.to_string()); + } + st.partial_text.clear(); + st.open_segments = st.open_segments.saturating_sub(1); + st.last_completed_at = Some(Instant::now()); + st.finishing + && !st.tail_write_pending + && !has_audio_after_last_completed(&st) + && st.open_segments == 0 + }; + if should_finish { + self.finish_success(); + return false; + } + true + } + + fn finish_success(&self) { + let (tx, text, duration_ms) = { + let mut st = self.state.lock(); + if st.session_finished { + return; + } + st.session_finished = true; + st.send_tx.take(); + let mut segments = std::mem::take(&mut st.completed_segments); + // 收尾时若还有未 completed 的 interim 尾巴(静音帧理应冲出 completed, + // 防御性兜底),拼在最后。 + if !st.partial_text.is_empty() { + segments.push(std::mem::take(&mut st.partial_text)); + } + let text = join_segments(&segments); + let duration_ms = if st.bytes_received > 0 { + st.bytes_received / BYTES_PER_MS + } else { + st.start + .map(|start| start.elapsed().as_millis() as u64) + .unwrap_or_default() + }; + (st.final_tx.take(), text, duration_ms) + }; + if let Some(tx) = tx { + let _ = tx.send(Ok(RawTranscript { text, duration_ms })); + } + self.session_finished.notify_waiters(); + self.close_on_runtime(); + } + + fn finish_with_partial_or_error(&self, error: StepfunASRError) { + let has_partial = { + let st = self.state.lock(); + !st.completed_segments.is_empty() || !st.partial_text.trim().is_empty() + }; + if has_partial { + // 与 Bailian / Qwen / Volcengine 一致:异常但已有结果时兜底返回。 + self.finish_success(); + } else { + self.finish_error(error); + } + } + + fn finish_error(&self, error: StepfunASRError) { + self.fail_session_start(&error.to_string()); + let tx = { + let mut st = self.state.lock(); + if st.session_finished { + return; + } + st.session_finished = true; + st.send_tx.take(); + st.final_tx.take() + }; + if let Some(tx) = tx { + let _ = tx.send(Err(error)); + } + self.session_finished.notify_waiters(); + self.close_on_runtime(); + } + + fn close_on_runtime(&self) { + let writer = Arc::clone(&self.writer); + if let Some(handle) = self.state.lock().runtime.clone() { + handle.spawn(async move { + let _ = close_writer(&writer).await; + }); + } + } + + fn mark_non_silent_audio_written(&self) { + self.state.lock().last_non_silent_audio_written_at = Some(Instant::now()); + } +} + +impl AudioConsumer for StepfunRealtimeASR { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + if pcm.is_empty() { + return; + } + let (send_tx, chunks) = { + let mut st = self.state.lock(); + st.bytes_received = st.bytes_received.saturating_add(pcm.len() as u64); + if !st.session_started { + st.pending_audio.extend_from_slice(pcm); + return; + } + st.audio_scratch.extend_from_slice(pcm); + let chunks = drain_audio_chunks(&mut st.audio_scratch); + (st.send_tx.clone(), chunks) + }; + if let Some(tx) = send_tx { + for chunk in chunks { + let _ = tx.send(SendItem::Audio { + contains_non_silent_audio: contains_non_silent_pcm(&chunk), + chunk, + written_tx: None, + }); + } + } + } +} + +fn drain_audio_chunks(buffer: &mut Vec) -> Vec> { + let mut chunks = Vec::new(); + while buffer.len() >= TARGET_AUDIO_CHUNK_BYTES { + chunks.push(buffer.drain(..TARGET_AUDIO_CHUNK_BYTES).collect()); + } + chunks +} + +fn contains_non_silent_pcm(pcm: &[u8]) -> bool { + pcm.iter().any(|&byte| byte != 0) +} + +fn has_audio_after_last_completed(state: &SyncState) -> bool { + match ( + state.last_non_silent_audio_written_at, + state.last_completed_at, + ) { + (Some(last_audio), Some(last_completed)) => last_audio > last_completed, + (Some(_), None) => true, + (None, _) => false, + } +} + +fn session_update_message(model: &str, prompt: Option<&str>) -> String { + // language 省略 => 服务端自动检测语种。 + let mut transcription = json!({ "model": model }); + if let Some(prompt) = prompt { + let trimmed = prompt.trim(); + if !trimmed.is_empty() { + transcription["prompt"] = json!(trimmed); + } + } + json!({ + "type": "session.update", + "event_id": event_id(), + "session": { + "audio": { + "input": { + "format": { + "type": "pcm", + "codec": "pcm_s16le", + "rate": 16000, + "bits": 16, + "channel": 1, + }, + "transcription": transcription, + "turn_detection": { + "type": "server_vad", + "silence_duration_ms": VAD_SILENCE_DURATION_MS, + }, + }, + }, + }, + }) + .to_string() +} + +fn append_audio_message(pcm: &[u8]) -> String { + json!({ + "type": "input_audio_buffer.append", + "event_id": event_id(), + "audio": base64::engine::general_purpose::STANDARD.encode(pcm), + }) + .to_string() +} + +fn event_id() -> String { + format!("event_{}", Uuid::new_v4()) +} + +async fn send_text(writer: &SharedWriter, text: String) -> Result<(), StepfunASRError> { + tokio::time::timeout(WRITE_TIMEOUT, async { + let mut guard = writer.lock().await; + let Some(ws) = guard.as_mut() else { + return Err(StepfunASRError::ConnectionFailed( + "websocket writer not available".to_string(), + )); + }; + ws.send(Message::Text(text)) + .await + .map_err(|e| StepfunASRError::SendFailed(e.to_string())) + }) + .await + .map_err(|_| StepfunASRError::SendFailed("websocket write timed out".to_string()))? +} + +async fn close_writer(writer: &SharedWriter) -> Result<(), StepfunASRError> { + let mut guard = writer.lock().await; + if let Some(mut ws) = guard.take() { + let _ = ws.close().await; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::{SinkExt, StreamExt}; + + fn create_test_asr() -> StepfunRealtimeASR { + StepfunRealtimeASR::new(StepfunRealtimeCredentials { + api_key: "sk-test".to_string(), + endpoint: String::new(), + model: String::new(), + prompt: None, + }) + } + + // ---- credentials / URL ---- + + #[test] + fn connect_url_defaults_and_derives_from_https_base() { + let mut creds = StepfunRealtimeCredentials { + api_key: "k".to_string(), + endpoint: String::new(), + model: String::new(), + prompt: None, + }; + assert_eq!(creds.connect_url(), DEFAULT_ENDPOINT); + assert_eq!(creds.normalized_model(), DEFAULT_MODEL); + + // 批式共用的 https base(preset 默认值)→ 派生 wss 完整路径。 + creds.endpoint = "https://api.stepfun.com/v1".to_string(); + assert_eq!(creds.connect_url(), DEFAULT_ENDPOINT); + creds.endpoint = "https://api.stepfun.com/v1/".to_string(); + assert_eq!(creds.connect_url(), DEFAULT_ENDPOINT); + + // 完整 wss URL 原样使用。 + creds.endpoint = "wss://gateway.example.com/v1/realtime/asr/stream".to_string(); + assert_eq!( + creds.connect_url(), + "wss://gateway.example.com/v1/realtime/asr/stream" + ); + } + + // ---- message builders ---- + + #[test] + fn session_update_uses_stepfun_nested_shape() { + let value: Value = + serde_json::from_str(&session_update_message(DEFAULT_MODEL, Some("OpenLess."))) + .unwrap(); + assert_eq!(value["type"], "session.update"); + let input = &value["session"]["audio"]["input"]; + assert_eq!(input["format"]["codec"], "pcm_s16le"); + assert_eq!(input["format"]["rate"], 16000); + assert_eq!(input["transcription"]["model"], DEFAULT_MODEL); + assert_eq!(input["transcription"]["prompt"], "OpenLess."); + assert_eq!(input["turn_detection"]["type"], "server_vad"); + } + + #[test] + fn session_update_omits_blank_prompt() { + let value: Value = + serde_json::from_str(&session_update_message(DEFAULT_MODEL, Some(" "))).unwrap(); + assert!(value["session"]["audio"]["input"]["transcription"]["prompt"].is_null()); + let value: Value = + serde_json::from_str(&session_update_message(DEFAULT_MODEL, None)).unwrap(); + assert!(value["session"]["audio"]["input"]["transcription"]["prompt"].is_null()); + } + + // ---- event handling ---- + + fn delta_event(text: &str, stash: &str) -> String { + json!({ + "type": "conversation.item.input_audio_transcription.delta", + "text": text, + "stash": stash, + }) + .to_string() + } + + fn completed_event(transcript: &str) -> String { + json!({ + "type": "conversation.item.input_audio_transcription.completed", + "transcript": transcript, + }) + .to_string() + } + + fn speech_started_event() -> String { + json!({ "type": "input_audio_buffer.speech_started" }).to_string() + } + + #[test] + fn delta_concatenates_confirmed_text_with_stash() { + // StepFun 语义:text 前缀 + stash 尾巴(实测 2026-07),非 Qwen 的互斥取一。 + let asr = create_test_asr(); + asr.handle_text_message(&delta_event("", "今天。")); + assert_eq!(asr.state.lock().partial_text, "今天。"); + asr.handle_text_message(&delta_event("今天天气不错,", "我们来测试。")); + assert_eq!(asr.state.lock().partial_text, "今天天气不错,我们来测试。"); + // 两者皆空不覆盖已有 partial。 + asr.handle_text_message(&delta_event("", "")); + assert_eq!(asr.state.lock().partial_text, "今天天气不错,我们来测试。"); + } + + #[test] + fn completed_closes_open_segment_and_clears_partial() { + let asr = create_test_asr(); + asr.handle_text_message(&speech_started_event()); + assert_eq!(asr.state.lock().open_segments, 1); + asr.handle_text_message(&delta_event("第一句", "")); + let keep_going = asr.handle_text_message(&completed_event("第一句话说完了。")); + assert!(keep_going, "未进入 finishing 时读循环应继续"); + let st = asr.state.lock(); + assert_eq!(st.completed_segments, vec!["第一句话说完了。"]); + assert!(st.partial_text.is_empty()); + assert_eq!(st.open_segments, 0); + } + + #[test] + fn finishing_completes_when_last_open_segment_closes() { + let asr = create_test_asr(); + asr.handle_text_message(&speech_started_event()); + let (tx, mut rx) = oneshot::channel(); + { + let mut st = asr.state.lock(); + st.final_tx = Some(tx); + st.finishing = true; + st.bytes_received = 32_000; + } + let keep_going = asr.handle_text_message(&completed_event("最后一句。")); + assert!(!keep_going, "最后一段 completed 后读循环应退出"); + let result = rx.try_recv().unwrap().unwrap(); + assert_eq!(result.text, "最后一句。"); + assert_eq!(result.duration_ms, 1_000); + } + + #[test] + fn finishing_waits_while_segments_still_open() { + let asr = create_test_asr(); + asr.handle_text_message(&speech_started_event()); + asr.handle_text_message(&speech_started_event()); + asr.state.lock().finishing = true; + let keep_going = asr.handle_text_message(&completed_event("第一段。")); + assert!(keep_going, "还有开放句段时不能提前收尾"); + assert!(!asr.state.lock().session_finished); + } + + #[test] + fn stepfun_request_error_event_returns_partial_when_available() { + let asr = create_test_asr(); + asr.handle_text_message(&speech_started_event()); + asr.handle_text_message(&completed_event("已识别内容。")); + let (tx, mut rx) = oneshot::channel(); + asr.state.lock().final_tx = Some(tx); + let keep_going = asr.handle_text_message( + &json!({"type": "transcript.response.error", "error": {"message": "boom"}}) + .to_string(), + ); + assert!(!keep_going); + assert_eq!(rx.try_recv().unwrap().unwrap().text, "已识别内容。"); + } + + #[test] + fn error_event_without_partial_returns_error() { + let asr = create_test_asr(); + let (tx, mut rx) = oneshot::channel(); + asr.state.lock().final_tx = Some(tx); + asr.handle_text_message(&json!({"type": "error", "error": {"message": "boom"}}).to_string()); + let err = rx.try_recv().unwrap().unwrap_err(); + assert!(matches!(err, StepfunASRError::TaskFailed(m) if m == "boom")); + } + + #[test] + fn only_session_updated_marks_session_ready() { + let asr = create_test_asr(); + asr.handle_text_message(&json!({"type": "session.created"}).to_string()); + assert!(!asr.state.lock().session_started); + asr.handle_text_message(&json!({"type": "session.updated"}).to_string()); + assert!(asr.state.lock().session_started); + } + + #[test] + fn empty_finishing_session_yields_empty_text() { + // 连接检查 / 误触场景:无任何句段,finish_success 返回空文本成功。 + let asr = create_test_asr(); + let (tx, mut rx) = oneshot::channel(); + { + let mut st = asr.state.lock(); + st.final_tx = Some(tx); + st.finishing = true; + } + asr.finish_success(); + assert_eq!(rx.try_recv().unwrap().unwrap().text, ""); + } + + #[test] + fn multi_segment_transcripts_join_like_qwen() { + let asr = create_test_asr(); + asr.handle_text_message(&speech_started_event()); + asr.handle_text_message(&completed_event("第一句。")); + asr.handle_text_message(&speech_started_event()); + let (tx, mut rx) = oneshot::channel(); + { + let mut st = asr.state.lock(); + st.final_tx = Some(tx); + st.finishing = true; + } + asr.handle_text_message(&completed_event("第二句。")); + assert_eq!(rx.try_recv().unwrap().unwrap().text, "第一句。第二句。"); + } + + // ---- audio buffering ---- + + #[test] + fn audio_buffered_before_session_ready() { + let asr = create_test_asr(); + asr.consume_pcm_chunk(&[0u8; 100]); + let st = asr.state.lock(); + assert_eq!(st.pending_audio.len(), 100); + assert_eq!(st.bytes_received, 100); + } + + #[test] + fn only_audio_written_after_completed_remains_pending() { + let completed_at = Instant::now(); + let mut state = SyncState { + last_non_silent_audio_written_at: Some(completed_at), + last_completed_at: Some(completed_at + Duration::from_millis(1)), + ..SyncState::default() + }; + assert!(!has_audio_after_last_completed(&state)); + + state.last_non_silent_audio_written_at = Some(completed_at + Duration::from_millis(2)); + assert!(has_audio_after_last_completed(&state)); + } + + #[test] + fn completed_does_not_finish_while_tail_write_is_pending() { + let asr = create_test_asr(); + let (tx, mut rx) = oneshot::channel(); + { + let mut state = asr.state.lock(); + state.final_tx = Some(tx); + state.finishing = true; + state.tail_write_pending = true; + state.open_segments = 1; + state.last_non_silent_audio_written_at = Some(Instant::now()); + } + + let keep_going = asr.handle_text_message(&completed_event("上一段。")); + + assert!(keep_going, "尾帧未写入时,旧 completed 不能提前关闭会话"); + assert!(!asr.state.lock().session_finished); + assert!(rx.try_recv().is_err()); + } + + #[tokio::test] + async fn non_silent_audio_waits_for_delayed_vad_before_finishing() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!( + "ws://{}/v1/realtime/asr/stream", + listener.local_addr().unwrap() + ); + let (release_events_tx, release_events_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws = tokio_tungstenite::accept_async(stream).await.unwrap(); + assert!(matches!( + ws.next().await.unwrap().unwrap(), + Message::Text(_) + )); + ws.send(Message::Text( + json!({ "type": "session.updated" }).to_string(), + )) + .await + .unwrap(); + + release_events_rx.await.unwrap(); + ws.send(Message::Text(speech_started_event())) + .await + .unwrap(); + ws.send(Message::Text(completed_event("delayed speech"))) + .await + .unwrap(); + // 保持连接直到客户端处理 completed 并主动关闭,避免测试服务端析构抢先 + // 触发客户端的连接错误。 + let _ = tokio::time::timeout(Duration::from_secs(2), async { + while let Some(message) = ws.next().await { + match message { + Ok(Message::Close(_)) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; + }); + + let asr = Arc::new(StepfunRealtimeASR::new(StepfunRealtimeCredentials { + api_key: "sk-test".to_string(), + endpoint, + model: DEFAULT_MODEL.to_string(), + prompt: None, + })); + asr.open_session().await.unwrap(); + asr.consume_pcm_chunk(&[1u8; TARGET_AUDIO_CHUNK_BYTES]); + + let asr_for_finish = Arc::clone(&asr); + let finish = tokio::spawn(async move { asr_for_finish.send_last_frame().await }); + tokio::time::sleep(FINISH_GRACE + Duration::from_millis(100)).await; + assert!( + !finish.is_finished(), + "non-silent audio must not finish before the server observes its VAD segment" + ); + + release_events_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(2), finish) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + asr.await_final_result().await.unwrap().text, + "delayed speech" + ); + server.await.unwrap(); + } +} diff --git a/openless-all/app/src-tauri/src/asr/whisper.rs b/openless-all/app/src-tauri/src/asr/whisper.rs index 05744782..b5242d13 100644 --- a/openless-all/app/src-tauri/src/asr/whisper.rs +++ b/openless-all/app/src-tauri/src/asr/whisper.rs @@ -52,6 +52,9 @@ pub struct WhisperBatchASR { verbose_json: bool, /// 请求体编码方式。默认 `Multipart`,OpenRouter 走 `OpenRouterJson`。 request_format: AsrRequestFormat, + /// 一等 `hotwords` 参数(JSON 数组字符串)。StepFun 等厂商不认 `prompt` + /// (静默忽略),但提供专门的热词字段——用它词典才真正生效。空 = 不发。 + hotwords: Vec, buffer: Mutex>, } @@ -72,6 +75,7 @@ impl WhisperBatchASR { max_chunk_duration_ms, verbose_json, request_format: AsrRequestFormat::Multipart, + hotwords: Vec::new(), buffer: Mutex::new(Vec::new()), } } @@ -83,6 +87,13 @@ impl WhisperBatchASR { self } + /// 设置一等热词列表(默认空 = 不发)。仅 Multipart 编码下生效;与 `prompt` + /// 二选一由 wiring 决定(见 coordinator 的 `whisper_uses_hotwords`)。 + pub fn with_hotwords(mut self, hotwords: Vec) -> Self { + self.hotwords = hotwords; + self + } + /// Stop collecting audio, encode the buffer as WAV, and POST to the /// Whisper transcriptions endpoint. /// @@ -172,6 +183,20 @@ impl WhisperBatchASR { } } + // 一等热词(StepFun 形状:可解析的 JSON 数组字符串,如 + // `["热词1","热词2"]`)。空白词条过滤后再编码,全空则不发该字段。 + let hotwords: Vec<&str> = self + .hotwords + .iter() + .map(|w| w.trim()) + .filter(|w| !w.is_empty()) + .collect(); + if !hotwords.is_empty() { + if let Ok(encoded) = serde_json::to_string(&hotwords) { + form = form.text("hotwords", encoded); + } + } + client .post(&url) .header("Authorization", format!("Bearer {}", self.api_key)) @@ -833,6 +858,63 @@ mod tests { server.join().unwrap(); } + #[tokio::test] + async fn hotwords_sent_as_json_array_field_without_prompt() { + // StepFun 形状:词典走一等 `hotwords`(JSON 数组字符串)而非 `prompt`; + // 空白词条过滤后编码。 + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "timed out waiting for ASR test request" + ); + thread::sleep(Duration::from_millis(10)); + } + Err(err) => panic!("accept ASR test request failed: {err}"), + } + }; + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let request = read_http_request(&mut stream); + let request_text = String::from_utf8_lossy(&request); + assert!(request_text.starts_with("POST /audio/transcriptions HTTP/1.1")); + assert!(request_text.contains(r#"name="hotwords""#)); + assert!(request_text.contains(r#"["阶跃星辰","OpenLess"]"#)); + assert!(!request_text.contains(r#"name="prompt""#)); + write_json_response(&mut stream, r#"{"text":"hotwords ok"}"#); + }); + let base_url = format!("http://{}", addr); + + let asr = WhisperBatchASR::new( + "key".to_string(), + base_url, + "stepaudio-2.5-asr".to_string(), + None, + None, + false, + ) + .with_hotwords(vec![ + "阶跃星辰".to_string(), + " ".to_string(), + "OpenLess".to_string(), + ]); + let pcm = vec![0u8; 32_000 * 2]; + asr.consume_pcm_chunk(&pcm); + + let transcript = asr.transcribe().await.unwrap(); + assert_eq!(transcript.text, "hotwords ok"); + server.join().unwrap(); + } + fn start_whisper_test_server(texts: Vec<&'static str>) -> (String, thread::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); listener.set_nonblocking(true).unwrap(); diff --git a/openless-all/app/src-tauri/src/commands/providers.rs b/openless-all/app/src-tauri/src/commands/providers.rs index a8b35f74..22dae6bd 100644 --- a/openless-all/app/src-tauri/src/commands/providers.rs +++ b/openless-all/app/src-tauri/src/commands/providers.rs @@ -270,6 +270,18 @@ async fn validate_asr_provider() -> Result<(), String> { if active_asr == crate::asr::elevenlabs::PROVIDER_ID { return validate_elevenlabs_asr_provider().await; } + // StepFun 一入口双协议:`*-stream` 模型走实时 WS 验证,其余走批式 + // /audio/transcriptions(与 build 侧 resolve_effective_asr_provider 同判据)。 + if active_asr == "stepfun" || active_asr == crate::asr::stepfun_realtime::PROVIDER_ID { + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .map_err(|e| e.to_string())? + .unwrap_or_default(); + if active_asr == crate::asr::stepfun_realtime::PROVIDER_ID + || crate::coordinator::stepfun_model_is_stream(&model) + { + return validate_stepfun_realtime_asr_provider().await; + } + } let config = read_openai_provider_config("asr")?; let model = CredentialsVault::get(CredentialAccount::AsrModel) @@ -279,6 +291,43 @@ async fn validate_asr_provider() -> Result<(), String> { validate_asr_transcription(&config, model.trim()).await } +/// StepFun 实时 WS 验证:真连 + session.update + 500ms 静音 + 收尾。 +/// 协议无 finish 事件,收尾走静音帧 + 宽限期(纯静音会话以空文本成功返回, +/// 见 stepfun_realtime 模块注释),全程 ~2s。 +async fn validate_stepfun_realtime_asr_provider() -> Result<(), String> { + let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) + .map_err(|e| e.to_string())? + .unwrap_or_default(); + if api_key.trim().is_empty() { + return Err("API Key 为空".to_string()); + } + let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint) + .map_err(|e| e.to_string())? + .unwrap_or_default(); + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .map_err(|e| e.to_string())? + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::stepfun_realtime::DEFAULT_MODEL.to_string()); + let asr = std::sync::Arc::new(crate::asr::StepfunRealtimeASR::new( + crate::asr::StepfunRealtimeCredentials { + api_key, + endpoint, + model, + prompt: None, + }, + )); + asr.open_session().await.map_err(|e| e.to_string())?; + crate::asr::AudioConsumer::consume_pcm_chunk( + &*asr, + &vec![0u8; crate::asr::stepfun_realtime::TARGET_AUDIO_CHUNK_BYTES * 5], + ); + asr.send_last_frame().await.map_err(|e| e.to_string())?; + asr.await_final_result() + .await + .map(|_| ()) + .map_err(|e| e.to_string()) +} + async fn validate_mimo_asr_provider() -> Result<(), String> { let config = read_openai_provider_config("asr")?; let model = CredentialsVault::get(CredentialAccount::AsrModel) @@ -588,6 +637,16 @@ async fn validate_asr_transcription(config: &ProviderConfig, model: &str) -> Res }; let status = response.status(); if !status.is_success() { + // 探针音频是纯静音,有的厂商(StepFun)对无语音内容直接 400 + // "no speech found"。走到这一步说明鉴权(错 key 是 401)和模型名 + // (错模型是 404 model_invalid)都已通过、转写管线是通的——这类 + // 内容拒收判为验证成功,避免对静音敏感的厂商恒报假阴性。 + if status.as_u16() == 400 { + let body = response.text().await.unwrap_or_default(); + if asr_error_is_no_speech_rejection(&body) { + return Ok(()); + } + } return Err(format!("providerHttpStatus:{}", status.as_u16())); } if let Some(len) = response.content_length() { @@ -612,6 +671,13 @@ async fn validate_asr_transcription(config: &ProviderConfig, model: &str) -> Res Ok(()) } +/// 400 应答体是否是「音频里没有语音」类内容拒收(而非参数错误)。 +/// 只匹配语义明确的措辞,宁可漏判(用户看到 400 后实测仍可用)也不误判 +/// 真正的参数错误为成功。 +fn asr_error_is_no_speech_rejection(body: &str) -> bool { + body.to_ascii_lowercase().contains("no speech") +} + pub(crate) fn asr_transcriptions_url(base_url: &str) -> Result { let parsed = reqwest::Url::parse(base_url.trim()).map_err(|_| "endpointInvalid".to_string())?; @@ -836,11 +902,26 @@ mod tests { // LLM 路径的 SSRF 校验。read_openai_provider_config 依赖凭据库无法纯单测,这里直接对它调用 // 的校验器锁定 ASR 形态 endpoint 的拒绝/放行契约。 use super::{ - fetch_provider_models, models_url, provider_llm_error_message, provider_log_context, - provider_request_error_message, sanitized_provider_destination, ProviderConfig, + asr_error_is_no_speech_rejection, fetch_provider_models, models_url, + provider_llm_error_message, provider_log_context, provider_request_error_message, + sanitized_provider_destination, ProviderConfig, }; use crate::endpoint_security::validate_http_endpoint; + #[test] + fn silence_probe_content_rejection_is_not_a_credential_error() { + // StepFun 对静音探针的实测应答(2026-07):鉴权/模型都通过,只是探针 + // 音频没有语音内容——不能报成凭据错误。 + assert!(asr_error_is_no_speech_rejection( + r#"{"error":{"message":"no speech found","type":"request_params_invalid"}}"# + )); + // 真正的参数错误不能被误判成功。 + assert!(!asr_error_is_no_speech_rejection( + r#"{"error":{"message":"Request param: response_format is invalid","type":"input_invalid"}}"# + )); + assert!(!asr_error_is_no_speech_rejection("")); + } + #[test] fn provider_destination_redacts_userinfo_query_and_fragment() { let raw = "https://alice:password@example.com:8443/v1/models?api_key=query-secret#private-fragment"; diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 148198c3..fc5a2736 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -187,6 +187,8 @@ enum ActiveAsr { Bailian(Arc), /// 百炼 Qwen3-ASR-Flash 实时(OpenAI Realtime 风格 WS 协议)。 Qwen3Realtime(Arc), + /// 阶跃星辰 StepAudio 实时(OpenAI Realtime 风格 WS,收尾靠静音帧驱动 VAD)。 + StepfunRealtime(Arc), #[cfg(target_os = "windows")] FoundryLocalWhisper(Arc), /// Windows sherpa-onnx 本地 ASR(offline batch + 实验 online streaming)。 @@ -224,6 +226,7 @@ fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool { pub(crate) enum ActiveAsrProviderKind { Bailian, Qwen3Realtime, + StepfunRealtime, Mimo, DashScopeMultimodal, ElevenLabs, @@ -259,6 +262,7 @@ impl ActiveAsrProviderKind { match self { ActiveAsrProviderKind::Bailian | ActiveAsrProviderKind::Qwen3Realtime + | ActiveAsrProviderKind::StepfunRealtime | ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal | ActiveAsrProviderKind::ElevenLabs @@ -277,7 +281,11 @@ impl ActiveAsrProviderKind { ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal => { AsrConfiguredFields::ApiKeyEndpointModel } - ActiveAsrProviderKind::WhisperCompatible => AsrConfiguredFields::EndpointModelOnly, + // StepfunRealtime 只经 `stepfun` 的模型路由可达(隐藏 effective id), + // 「已配置」判定看真实 active `stepfun` → WhisperCompatible;此处形态 + // 与之对齐,保证直接停在该 id 上也语义一致。 + ActiveAsrProviderKind::WhisperCompatible + | ActiveAsrProviderKind::StepfunRealtime => AsrConfiguredFields::EndpointModelOnly, ActiveAsrProviderKind::Volcengine => AsrConfiguredFields::VolcAppKey, } } @@ -288,6 +296,8 @@ pub(crate) fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind { ActiveAsrProviderKind::Bailian } else if is_qwen3_realtime_provider(id) { ActiveAsrProviderKind::Qwen3Realtime + } else if is_stepfun_realtime_provider(id) { + ActiveAsrProviderKind::StepfunRealtime } else if is_mimo_provider(id) { ActiveAsrProviderKind::Mimo } else if is_dashscope_multimodal_provider(id) { @@ -318,6 +328,12 @@ pub(crate) fn resolve_effective_asr_provider( if is_dashscope_multimodal_provider(active_asr) { validate_dashscope_multimodal_model(model)?; } + // StepFun 同款「一个入口按模型切协议」:`*-stream` 模型走实时 WS + // 客户端,其余走批式 Whisper 兼容路径。凭据 / 「已配置」判定仍看 + // 真实 active `stepfun`。 + if active_asr == "stepfun" && stepfun_model_is_stream(model) { + return Ok(crate::asr::stepfun_realtime::PROVIDER_ID.to_string()); + } return Ok(active_asr.to_string()); } @@ -347,6 +363,12 @@ fn is_classic_bailian_realtime_model(model: &str) -> bool { || model.starts_with("sensevoice-realtime") } +/// StepFun 的流式模型命名恒以 `-stream` 结尾(stepaudio-2.5-asr-stream / +/// step-asr-1.1-stream),其余(含空 = 默认批式模型)走批式。 +pub(crate) fn stepfun_model_is_stream(model: &str) -> bool { + model.trim().ends_with("-stream") +} + pub(crate) fn validate_dashscope_multimodal_model(model: &str) -> Result<(), String> { let model = model.trim(); if model.is_empty() || model == crate::asr::dashscope_multimodal::DEFAULT_MODEL { @@ -1881,6 +1903,13 @@ impl Coordinator { .map_err(|_| "重新转录超时".to_string())? .map_err(|e| e.to_string())? } + ActiveAsr::StepfunRealtime(asr) => { + asr.send_last_frame().await.map_err(|e| e.to_string())?; + tokio::time::timeout(timeout, asr.await_final_result()) + .await + .map_err(|_| "重新转录超时".to_string())? + .map_err(|e| e.to_string())? + } ActiveAsr::Whisper(w) => tokio::time::timeout(timeout, w.transcribe()) .await .map_err(|_| "重新转录超时".to_string())? @@ -2423,6 +2452,33 @@ fn read_qwen3_realtime_credentials() -> Qwen3RealtimeCredentials { } } +/// StepFun 实时凭据与批式共用同一组槽位(一把 key、同一个 https base、 +/// 模型名区分协议);wss URL 由 client 的 `connect_url()` 从 base 派生。 +/// `prompt` 由调用方按用户词典填充(实时协议接受 prompt、批式只认 hotwords)。 +fn read_stepfun_realtime_credentials( + prompt: Option, +) -> crate::asr::StepfunRealtimeCredentials { + let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) + .ok() + .flatten() + .unwrap_or_default(); + let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint) + .ok() + .flatten() + .unwrap_or_default(); + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .ok() + .flatten() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::stepfun_realtime::DEFAULT_MODEL.to_string()); + crate::asr::StepfunRealtimeCredentials { + api_key, + endpoint, + model, + prompt, + } +} + fn read_volc_credentials() -> VolcengineCredentials { let app_id = CredentialsVault::get(CredentialAccount::VolcengineAppKey) .ok() @@ -2859,6 +2915,58 @@ mod tests { assert_eq!(batch_asr_chunk_limit_ms("openrouter"), Some(30_000)); } + #[test] + fn stepfun_is_whisper_compatible_with_hotwords_vocab() { + use crate::asr::whisper::AsrRequestFormat; + // StepFun /audio/transcriptions 是标准 multipart(实测 2026-07), + // response_format 只认 json/text → verbose_json 关闭;100MB 上限 + // (约 54 分钟 16k WAV)→ 无需切分。 + assert!(is_whisper_compatible_provider("stepfun")); + assert_eq!( + active_asr_provider_kind("stepfun"), + ActiveAsrProviderKind::WhisperCompatible + ); + assert_eq!( + whisper_request_format("stepfun"), + AsrRequestFormat::Multipart + ); + assert!(!whisper_supports_verbose_json("stepfun")); + assert_eq!(batch_asr_chunk_limit_ms("stepfun"), None); + + // 一入口双协议:`*-stream` 模型路由到实时 WS 客户端,其余留在批式。 + assert_eq!( + resolve_effective_asr_provider("stepfun", "stepaudio-2.5-asr").unwrap(), + "stepfun" + ); + assert_eq!( + resolve_effective_asr_provider("stepfun", "").unwrap(), + "stepfun" + ); + assert_eq!( + resolve_effective_asr_provider("stepfun", "stepaudio-2.5-asr-stream").unwrap(), + crate::asr::stepfun_realtime::PROVIDER_ID + ); + assert_eq!( + resolve_effective_asr_provider("stepfun", "step-asr-1.1-stream").unwrap(), + crate::asr::stepfun_realtime::PROVIDER_ID + ); + assert_eq!( + active_asr_provider_kind(crate::asr::stepfun_realtime::PROVIDER_ID), + ActiveAsrProviderKind::StepfunRealtime + ); + + // 词典路由:StepFun 批式忽略 prompt,走一等 hotwords;其余厂商维持 prompt。 + assert!(whisper_uses_hotwords("stepfun")); + assert!(!whisper_uses_hotwords("whisper")); + let phrases = vec!["阶跃星辰".to_string()]; + let (prompt, hotwords) = whisper_vocab_for_provider("stepfun", phrases.clone()); + assert_eq!(prompt, None); + assert_eq!(hotwords, phrases); + let (prompt, hotwords) = whisper_vocab_for_provider("groq", phrases); + assert_eq!(prompt.as_deref(), Some("阶跃星辰.")); + assert!(hotwords.is_empty()); + } + #[test] fn qa_asr_provider_kind_tracks_active_provider() { assert_eq!( diff --git a/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs b/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs index b11387db..a32396a0 100644 --- a/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs +++ b/openless-all/app/src-tauri/src/coordinator/asr_wiring.rs @@ -320,8 +320,8 @@ pub(super) fn build_apple_speech( Arc::new(crate::asr::local::AppleSpeechAsr::new(locale)) } -/// `whisper` 是 OpenAI 原生;`siliconflow` / `zhipu` / `groq` 都暴露 -/// OpenAI 兼容的 `/audio/transcriptions`,统一走 `WhisperBatchASR`。 +/// `whisper` 是 OpenAI 原生;`siliconflow` / `zhipu` / `groq` / `stepfun` +/// 都暴露 OpenAI 兼容的 `/audio/transcriptions`,统一走 `WhisperBatchASR`。 /// 新增 OpenAI 兼容 ASR 时只需在这里加一项。 /// /// 注:DashScope 的 Qwen3-ASR-Flash 不在此列——它用 MultiModalConversation @@ -330,10 +330,35 @@ pub(super) fn build_apple_speech( pub(super) fn is_whisper_compatible_provider(id: &str) -> bool { matches!( id, - "whisper" | "siliconflow" | "zhipu" | "groq" | "openrouter" + "whisper" | "siliconflow" | "zhipu" | "groq" | "openrouter" | "stepfun" ) } +/// 用户词典该走 `prompt` 还是一等 `hotwords` 参数。 +/// +/// StepFun 的 `/audio/transcriptions` **静默忽略** `prompt`(实测 2026-07:带 +/// prompt 返回 200 但不参与偏置),词汇偏置走专门的 `hotwords` 字段(可解析的 +/// JSON 数组字符串)。其余兼容厂商维持 Whisper 惯例的 `prompt`。 +pub(super) fn whisper_uses_hotwords(provider_id: &str) -> bool { + provider_id == "stepfun" +} + +/// 词典启用词条 → (prompt, hotwords) 二选一路由,QA 与听写两处构造点共用。 +/// hotwords 厂商不再拼 prompt(免得白占请求体),prompt 厂商 hotwords 恒空。 +pub(super) fn whisper_vocab_for_provider( + provider_id: &str, + phrases: Vec, +) -> (Option, Vec) { + if whisper_uses_hotwords(provider_id) { + (None, phrases) + } else { + ( + crate::asr::whisper::build_prompt_from_phrases(&phrases), + Vec::new(), + ) + } +} + /// 该 provider 的请求体编码方式。OpenRouter 的 `/audio/transcriptions` 是 /// `application/json` + base64 音频(issue #582),其余兼容厂商沿用 multipart。 pub(super) fn whisper_request_format(provider_id: &str) -> crate::asr::whisper::AsrRequestFormat { @@ -364,6 +389,10 @@ pub(super) fn is_qwen3_realtime_provider(id: &str) -> bool { id == crate::asr::qwen_realtime::PROVIDER_ID } +pub(super) fn is_stepfun_realtime_provider(id: &str) -> bool { + id == crate::asr::stepfun_realtime::PROVIDER_ID +} + pub(super) fn is_mimo_provider(id: &str) -> bool { id == crate::asr::mimo::PROVIDER_ID } @@ -410,6 +439,10 @@ pub(super) enum QaAsrStart { asr: Arc, bridge: Arc, }, + StepfunRealtime { + asr: Arc, + bridge: Arc, + }, Ready { active: ActiveAsr, consumer: Arc, @@ -422,6 +455,7 @@ impl QaAsrStart { QaAsrStart::Volcengine { asr, .. } => ActiveAsr::Volcengine(Arc::clone(asr)), QaAsrStart::Bailian { asr, .. } => ActiveAsr::Bailian(Arc::clone(asr)), QaAsrStart::Qwen3Realtime { asr, .. } => ActiveAsr::Qwen3Realtime(Arc::clone(asr)), + QaAsrStart::StepfunRealtime { asr, .. } => ActiveAsr::StepfunRealtime(Arc::clone(asr)), QaAsrStart::Ready { active, .. } => active.clone(), } } @@ -431,6 +465,7 @@ impl QaAsrStart { QaAsrStart::Volcengine { bridge, .. } => Arc::clone(bridge) as _, QaAsrStart::Bailian { bridge, .. } => Arc::clone(bridge) as _, QaAsrStart::Qwen3Realtime { bridge, .. } => Arc::clone(bridge) as _, + QaAsrStart::StepfunRealtime { bridge, .. } => Arc::clone(bridge) as _, QaAsrStart::Ready { consumer, .. } => Arc::clone(consumer), } } @@ -462,6 +497,15 @@ impl QaAsrStart { ); Ok(()) } + QaAsrStart::StepfunRealtime { asr, bridge } => { + asr.open_session().await.map_err(|e| e.to_string())?; + let target: Arc = Arc::clone(asr) as _; + let flushed = bridge.attach(target); + log::info!( + "[coord] QA StepFun realtime ASR connected; flushed {flushed} deferred audio bytes" + ); + Ok(()) + } QaAsrStart::Ready { .. } => Ok(()), } } @@ -585,6 +629,18 @@ pub(super) async fn build_qa_asr_start( label, )) } + ActiveAsrProviderKind::StepfunRealtime => { + let prompt = crate::asr::whisper::build_prompt_from_phrases(&enabled_phrases(inner)); + let creds = read_stepfun_realtime_credentials(prompt); + let label = AsrCallLabel::new(effective_asr.clone(), Some(creds.model.clone())); + Ok(( + QaAsrStart::StepfunRealtime { + asr: Arc::new(crate::asr::StepfunRealtimeASR::new(creds)), + bridge: Arc::new(DeferredAsrBridge::new()), + }, + label, + )) + } ActiveAsrProviderKind::Mimo => { let (api_key, base_url, model) = read_mimo_credentials(); let label = AsrCallLabel::new(effective_asr.clone(), Some(model.clone())); @@ -612,8 +668,8 @@ pub(super) async fn build_qa_asr_start( ActiveAsrProviderKind::WhisperCompatible => { let (api_key, base_url, model) = read_whisper_credentials(); let label = AsrCallLabel::new(effective_asr.clone(), Some(model.clone())); - let whisper_prompt = - crate::asr::whisper::build_prompt_from_phrases(&enabled_phrases(inner)); + let (whisper_prompt, hotwords) = + whisper_vocab_for_provider(active_asr, enabled_phrases(inner)); let whisper = Arc::new( WhisperBatchASR::new( api_key, @@ -623,7 +679,8 @@ pub(super) async fn build_qa_asr_start( batch_asr_chunk_limit_ms(active_asr), whisper_supports_verbose_json(active_asr), ) - .with_request_format(whisper_request_format(active_asr)), + .with_request_format(whisper_request_format(active_asr)) + .with_hotwords(hotwords), ); let active = ActiveAsr::Whisper(Arc::clone(&whisper)); let consumer: Arc = whisper; diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index 94cf53e6..f396df07 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -1672,6 +1672,7 @@ pub(super) async fn begin_session_as( match active_asr_provider_kind(&effective_asr) { ActiveAsrProviderKind::Bailian | ActiveAsrProviderKind::Qwen3Realtime + | ActiveAsrProviderKind::StepfunRealtime | ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal | ActiveAsrProviderKind::ElevenLabs @@ -1834,6 +1835,88 @@ pub(super) async fn begin_session_as( "[coord] Qwen3 realtime ASR connected; flushed {flushed_bytes} deferred audio bytes" ); finish_starting_session(inner, current_session_id).await; + } else if is_stepfun_realtime_provider(&effective_asr) { + // 与 Qwen3 realtime 分支同构:流式 WS 会话 + DeferredAsrBridge 缓冲开链前音频。 + // 实时协议的词汇偏置走 transcription.prompt(批式 stepfun 则相反走 hotwords)。 + let prompt = crate::asr::whisper::build_prompt_from_phrases(&enabled_phrases(inner)); + let creds = read_stepfun_realtime_credentials(prompt); + let asr_call_label = AsrCallLabel::new(effective_asr.clone(), Some(creds.model.clone())); + let asr = Arc::new(crate::asr::StepfunRealtimeASR::new(creds)); + let bridge = Arc::new(DeferredAsrBridge::new()); + let consumer: Arc = bridge.clone(); + store_asr_for_session( + inner, + current_session_id, + ActiveAsr::StepfunRealtime(Arc::clone(&asr)), + asr_call_label, + ); + start_recorder_for_starting(inner, current_session_id, &active_asr, consumer).await?; + + if let Err(e) = asr.open_session().await { + log::error!("[coord] open StepFun realtime ASR session failed: {e}"); + match startup_race_status_for_starting(inner, current_session_id) { + StartupRaceStatus::StaleContinuation => { + log::info!( + "[coord] stale StepFun realtime ASR open_session error from session {current_session_id} — ignoring" + ); + asr.cancel(); + discard_startup_resources_for_session(inner, current_session_id); + restore_prepared_windows_ime_session(inner, current_session_id); + return Ok(()); + } + StartupRaceStatus::CancelRaced => { + asr.cancel(); + discard_startup_resources_for_session(inner, current_session_id); + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + return Ok(()); + } + StartupRaceStatus::ActiveStarting => { + asr.cancel(); + } + } + discard_startup_resources_for_session(inner, current_session_id); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + 0, + Some(format!("ASR 连接失败: {e}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(e.to_string()); + } + match startup_race_status_for_starting(inner, current_session_id) { + StartupRaceStatus::ActiveStarting => {} + StartupRaceStatus::CancelRaced => { + log::info!( + "[coord] cancel raced during StepFun realtime ASR open_session — aborting begin" + ); + asr.cancel(); + discard_startup_resources_for_session(inner, current_session_id); + restore_prepared_windows_ime_session(inner, current_session_id); + set_phase_idle_if_session_matches(inner, current_session_id); + return Ok(()); + } + StartupRaceStatus::StaleContinuation => { + log::info!( + "[coord] stale StepFun realtime ASR open_session continuation from session {current_session_id} — ignoring" + ); + asr.cancel(); + discard_startup_resources_for_session(inner, current_session_id); + restore_prepared_windows_ime_session(inner, current_session_id); + return Ok(()); + } + } + let target: Arc = asr; + let flushed_bytes = bridge.attach(target); + log::info!( + "[coord] StepFun realtime ASR connected; flushed {flushed_bytes} deferred audio bytes" + ); + finish_starting_session(inner, current_session_id).await; } else if is_mimo_provider(&effective_asr) { let (api_key, base_url, model) = read_mimo_credentials(); let asr_call_label = AsrCallLabel::new(effective_asr.clone(), Some(model.clone())); @@ -1882,8 +1965,8 @@ pub(super) async fn begin_session_as( // hotword を受け取っており、UI 説明文も「ASR ホットワードと後処理 // モデルのコンテキスト両方に渡される」と明示しているので、Whisper // 互換プロバイダにも揃えるのが筋。 - let whisper_prompt = - crate::asr::whisper::build_prompt_from_phrases(&enabled_phrases(inner)); + let (whisper_prompt, hotwords) = + whisper_vocab_for_provider(&active_asr, enabled_phrases(inner)); let asr_call_label = AsrCallLabel::new(effective_asr.clone(), Some(model.clone())); let whisper = Arc::new( WhisperBatchASR::new( @@ -1894,7 +1977,8 @@ pub(super) async fn begin_session_as( batch_asr_chunk_limit_ms(&active_asr), whisper_supports_verbose_json(&active_asr), ) - .with_request_format(whisper_request_format(&active_asr)), + .with_request_format(whisper_request_format(&active_asr)) + .with_hotwords(hotwords), ); store_asr_for_session( inner, @@ -2743,6 +2827,34 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { } } } + ActiveAsr::StepfunRealtime(asr) => { + debug_assert!(uses_global_timeout); + if let Err(e) = asr.send_last_frame().await { + log::error!("[coord] StepFun realtime send last frame failed: {e}"); + } + let timeout_duration = + std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, asr.await_final_result()).await { + Ok(Ok(r)) => Ok(r), + Ok(Err(e)) => { + log::error!("[coord] StepFun realtime await final failed: {e}"); + // 关闭 WebSocket 连接,避免流式 ASR 资源泄漏 + asr.cancel(); + Err(TranscribeFail::new(format!("识别失败: {e}"), e.to_string())) + } + Err(_) => { + log::error!( + "[coord] StepFun realtime 全局超时 {} 秒", + COORDINATOR_GLOBAL_TIMEOUT_SECS + ); + asr.cancel(); + Err(TranscribeFail::new( + "识别超时".to_string(), + "stepfun realtime global timeout".to_string(), + )) + } + } + } #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => { debug_assert!(!uses_global_timeout); diff --git a/openless-all/app/src-tauri/src/coordinator/qa_session.rs b/openless-all/app/src-tauri/src/coordinator/qa_session.rs index 97ec40b2..c720bc67 100644 --- a/openless-all/app/src-tauri/src/coordinator/qa_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/qa_session.rs @@ -444,6 +444,21 @@ pub(super) async fn transcribe_overlay_dictation_asr( } } } + ActiveAsr::StepfunRealtime(asr) => { + debug_assert!(uses_global_timeout); + if let Err(error) = asr.send_last_frame().await { + log::error!("[coord] overlay QA: StepFun realtime send last frame failed: {error}"); + } + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, asr.await_final_result()).await { + Ok(Ok(raw)) => Ok(raw), + Ok(Err(error)) => Err(error.to_string()), + Err(_) => { + asr.cancel(); + Err("stepfun realtime global timeout".to_string()) + } + } + } ActiveAsr::Whisper(whisper) => { debug_assert!(uses_global_timeout); let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); @@ -1039,6 +1054,30 @@ pub(super) async fn end_qa_session(inner: &Arc) -> Result<(), String> { } } } + ActiveAsr::StepfunRealtime(asr) => { + debug_assert!(uses_global_timeout); + if let Err(e) = asr.send_last_frame().await { + log::error!("[coord] QA: StepFun realtime send last frame failed: {e}"); + } + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, asr.await_final_result()).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + log::error!("[coord] QA: StepFun realtime await final failed: {e}"); + finish_qa_with_error_if_current(inner, session_id, format!("识别失败: {e}")); + return Err(e.to_string()); + } + Err(_) => { + log::error!( + "[coord] QA: StepFun realtime 全局超时 {} 秒", + COORDINATOR_GLOBAL_TIMEOUT_SECS + ); + asr.cancel(); + finish_qa_with_error_if_current(inner, session_id, "识别超时".to_string()); + return Err("stepfun realtime global timeout".to_string()); + } + } + } ActiveAsr::Qwen3Realtime(asr) => { debug_assert!(uses_global_timeout); if let Err(e) = asr.send_last_frame().await { diff --git a/openless-all/app/src-tauri/src/coordinator/resources.rs b/openless-all/app/src-tauri/src/coordinator/resources.rs index 1d633088..615f2b3e 100644 --- a/openless-all/app/src-tauri/src/coordinator/resources.rs +++ b/openless-all/app/src-tauri/src/coordinator/resources.rs @@ -90,6 +90,7 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) { ActiveAsr::ElevenLabs(e) => e.cancel(), ActiveAsr::Bailian(b) => b.cancel(), ActiveAsr::Qwen3Realtime(q) => q.cancel(), + ActiveAsr::StepfunRealtime(s) => s.cancel(), #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => local.cancel(), #[cfg(target_os = "windows")] diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 6a2a5085..93cfbd36 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -788,6 +788,7 @@ export const en: typeof zhCN = { asrBailianQwen3: 'Bailian Qwen3 Realtime ASR', asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash (recorded file)', asrSiliconflow: 'SiliconFlow SenseVoice', + asrStepfun: 'StepFun StepAudio ASR', asrZhipu: 'Zhipu GLM-ASR', asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper (compatible)', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 0347a7c5..5e70c8dc 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -790,6 +790,7 @@ export const ja: typeof zhCN = { asrBailianQwen3: 'Bailian Qwen3 リアルタイム ASR', asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash(録音ファイル)', asrSiliconflow: 'SiliconFlow SenseVoice', + asrStepfun: 'StepFun StepAudio ASR', asrZhipu: 'Zhipu GLM-ASR', asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper(互換)', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 23aaf5f1..832254d0 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -790,6 +790,7 @@ export const ko: typeof zhCN = { asrBailianQwen3: 'Bailian Qwen3 실시간 ASR', asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash (녹음 파일)', asrSiliconflow: 'SiliconFlow SenseVoice', + asrStepfun: 'StepFun StepAudio ASR', asrZhipu: 'Zhipu GLM-ASR', asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper(호환)', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 55087fe1..9202b600 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -786,6 +786,7 @@ export const zhCN = { asrBailianQwen3: '阿里云百炼 Qwen3 实时 ASR', asrBailianFunAsrFlash: '阿里云百炼 Fun-ASR-Flash(录音文件)', asrSiliconflow: '硅基流动 SenseVoice', + asrStepfun: '阶跃星辰 StepAudio', asrZhipu: '智谱 GLM-ASR', asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper(兼容)', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 882c4844..222844ee 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -788,6 +788,7 @@ export const zhTW: typeof zhCN = { asrBailianQwen3: '阿里雲百煉 Qwen3 即時 ASR', asrBailianFunAsrFlash: '阿里雲百煉 Fun-ASR-Flash(錄音檔)', asrSiliconflow: '硅基流動 SenseVoice', + asrStepfun: '階躍星辰 StepAudio', asrZhipu: '智譜 GLM-ASR', asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper(兼容)', diff --git a/openless-all/app/src/pages/settings/shared.tsx b/openless-all/app/src/pages/settings/shared.tsx index 396e4f59..fd89b92c 100644 --- a/openless-all/app/src/pages/settings/shared.tsx +++ b/openless-all/app/src/pages/settings/shared.tsx @@ -223,6 +223,12 @@ export const ASR_PRESETS = [ // 由 asr/dashscope_multimodal.rs 专用批量 client 处理。API key 与百炼同一把。 { id: 'bailian-fun-asr-flash', nameKey: 'asrBailianFunAsrFlash', baseUrl: 'https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation', model: 'fun-asr-flash-2026-06-15' }, { id: 'siliconflow', nameKey: 'asrSiliconflow', baseUrl: 'https://api.siliconflow.cn/v1', model: 'FunAudioLLM/SenseVoiceSmall' }, + // 阶跃星辰 StepAudio:一个入口双协议(照百炼先例,后端按模型名二次路由)。 + // 默认批式模型走 Whisper 兼容 multipart,词典经一等 hotwords 参数生效; + // 模型改成 stepaudio-2.5-asr-stream 则走实时 WS(asr/stepfun_realtime.rs), + // 词典改经 transcription.prompt 生效。endpoint 一律填 https base, + // wss URL 由后端自动派生。 + { id: 'stepfun', nameKey: 'asrStepfun', baseUrl: 'https://api.stepfun.com/v1', model: 'stepaudio-2.5-asr' }, { id: 'zhipu', nameKey: 'asrZhipu', baseUrl: 'https://open.bigmodel.cn/api/paas/v4', model: 'glm-asr-2512' }, { id: 'groq', nameKey: 'asrGroq', baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3-turbo' }, { id: 'whisper', nameKey: 'asrWhisper', baseUrl: 'https://api.openai.com/v1', model: 'whisper-1' },