feat(asr): 百炼 ASR 支持 fun-asr-flash 并统一三协议为单一入口 (#793)

* feat(asr): 百炼 ASR 支持 fun-asr-flash 并统一三协议为单一入口

* fix(asr): harden DashScope model routing and transcription

* fix(asr): address Bailian protocol review feedback

---------

Co-authored-by: Chris233 <h-chris233@outlook.com>
This commit is contained in:
bigsongandChris233 authored and GitHub committed 2026-07-14 10:51:36 +08:00
1 parent 3f51edafab
commit 6e79a0c289
18 files changed
+1237 -139

No files matched your search

@@ -0,0 +1,412 @@
//! 阿里云百炼(DashScope)多模态生成同步接口的批量 ASR 客户端。
//!
//! `fun-asr-flash` 系列是**非实时录音文件识别**模型,走 DashScope 私有的
//! `multimodal-generation/generation` HTTP 接口,既不是实时 WebSocket 双工
//! (见 `bailian.rs`),也不是 OpenAI 兼容的 `/audio/transcriptions`
//! (见 `whisper.rs`)。因此单独成一路批量客户端:录音结束后把整段 PCM 编成
//! WAV、base64 进 JSON body、POST 一次拿整段文本。
//!
//! 结构与 `mimo.rs`(同为「攒 PCM → POST 一段音频 → 解析私有 JSON」)一致,
//! 复用其 `split_pcm_by_duration` / `join_transcript_chunks` 分片与拼接逻辑,
//! 只有请求信封与响应解析不同。
use anyhow::{Context, Result};
use base64::Engine;
use parking_lot::Mutex;
use serde_json::Value;
use crate::asr::mimo::{join_transcript_chunks, split_pcm_by_duration};
use crate::asr::wav::encode_wav_16k_mono;
use crate::asr::RawTranscript;
// fun-asr-flash 单条音频上限 5 分钟;但真正的硬约束是 base64 进 JSON 的请求体
// 体积。沿用 mimo 验证过的 180s 预算(16k/16-bit/mono WAV base64 后约 7.7MB),
// 稳稳落在时长和常见网关体积上限之内。超长录音按此切分后逐段识别再拼接。
const DASHSCOPE_MAX_CHUNK_DURATION_MS: u64 = 180_000;
pub const PROVIDER_ID: &str = "bailian-fun-asr-flash";
pub const DEFAULT_ENDPOINT: &str =
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation";
pub const DEFAULT_MODEL: &str = "fun-asr-flash-2026-06-15";
pub struct DashScopeMultimodalASR {
api_key: String,
base_url: String,
model: String,
buffer: Mutex<Vec<u8>>,
}
impl DashScopeMultimodalASR {
pub fn new(api_key: String, base_url: String, model: String) -> Self {
Self {
api_key,
base_url,
model,
buffer: Mutex::new(Vec::new()),
}
}
pub fn buffer_duration_ms(&self) -> u64 {
crate::asr::pcm::pcm_duration_ms(&self.buffer.lock())
}
pub async fn transcribe(&self) -> Result<RawTranscript> {
let pcm = self.buffer.lock().clone();
if pcm.is_empty() {
return Ok(RawTranscript {
text: String::new(),
duration_ms: 0,
});
}
let result = self.transcribe_inner(&pcm).await;
if result.is_ok() {
self.buffer.lock().clear();
}
result
}
async fn transcribe_inner(&self, pcm: &[u8]) -> Result<RawTranscript> {
if self.api_key.trim().is_empty() {
anyhow::bail!("DashScope API key missing");
}
let duration_ms = crate::asr::pcm::pcm_duration_ms(pcm);
let chunks = split_pcm_by_duration(pcm, DASHSCOPE_MAX_CHUNK_DURATION_MS);
let mut texts = Vec::with_capacity(chunks.len());
for chunk in chunks {
texts.push(self.transcribe_chunk(chunk).await?);
}
Ok(RawTranscript {
text: join_transcript_chunks(&texts),
duration_ms,
})
}
async fn transcribe_chunk(&self, pcm: &[u8]) -> Result<String> {
let samples: Vec<i16> = pcm
.chunks_exact(2)
.map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
.collect();
let wav = encode_wav_16k_mono(&samples);
let body = dashscope_multimodal_body(&self.model, &wav);
let url = generation_url(&self.base_url)?;
let client = reqwest::Client::new();
let resp = client
.post(&url)
.header("Authorization", format!("Bearer {}", self.api_key.trim()))
.header("Content-Type", "application/json")
// multimodal-generation 默认可 SSE 流式;显式关掉走一次性 JSON 响应。
.header("X-DashScope-SSE", "disable")
.json(&body)
.send()
.await
.context("DashScope ASR HTTP request failed")?;
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
anyhow::bail!("DashScope ASR API error {}: {}", status, body);
}
let json: Value = resp.json().await.context("parse DashScope ASR response")?;
Ok(extract_dashscope_text(&json).trim().to_string())
}
pub fn cancel(&self) {
self.buffer.lock().clear();
}
}
impl crate::recorder::AudioConsumer for DashScopeMultimodalASR {
fn consume_pcm_chunk(&self, pcm: &[u8]) {
self.buffer.lock().extend_from_slice(pcm);
}
}
/// 归一化到 multimodal-generation 的完整 endpoint。
///
/// preset 默认下发的就是完整地址,命中首个分支直接用;用户若只填了业务空间
/// 专属域名根(`https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com`)则补上标准
/// 路径。其余情况保守地把标准后缀拼到用户给的路径后面。
pub fn generation_url(base_url: &str) -> Result<String> {
const CANONICAL_PATH: &str = "/api/v1/services/aigc/multimodal-generation/generation";
let trimmed = base_url.trim();
let parsed = reqwest::Url::parse(trimmed).context("parse DashScope base URL")?;
let path = parsed.path().trim_end_matches('/');
if path.ends_with("/multimodal-generation/generation") {
let mut url = parsed.clone();
url.set_path(path);
return Ok(url.to_string());
}
let mut url = parsed.clone();
if path.is_empty() {
url.set_path(CANONICAL_PATH);
} else {
url.set_path(&format!("{path}{CANONICAL_PATH}"));
}
Ok(url.to_string())
}
pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value {
let audio_data = format!(
"data:audio/wav;base64,{}",
base64::engine::general_purpose::STANDARD.encode(wav)
);
serde_json::json!({
"model": model,
"input": {
"messages": [{
"role": "user",
"content": [{
"type": "input_audio",
"input_audio": { "data": audio_data },
}],
}],
},
"parameters": {
"format": "wav",
"sample_rate": "16000",
},
})
}
/// fun-asr-flash 的响应信封与标准多模态接口不同,且不同模型版本字段路径略有
/// 差异(`output.text` / `output.output.sentence.text` / 标准 `choices`)。
/// 这里按已知路径逐一兜底提取,取到第一个非空文本即返回,避免因单一路径假设
/// 而在某个版本上静默丢字。
pub fn extract_dashscope_text(json: &Value) -> String {
let output = json.get("output");
// 1) output.text —— fun-asr-flash 文档主路径
if let Some(text) = output.and_then(|o| o.get("text")).and_then(Value::as_str) {
if !text.trim().is_empty() {
return text.trim().to_string();
}
}
// 2) output.output.sentence.text —— 文档给出的另一种嵌套形态
if let Some(text) = output
.and_then(|o| o.get("output"))
.and_then(|o| o.get("sentence"))
.and_then(|s| s.get("text"))
.and_then(Value::as_str)
{
if !text.trim().is_empty() {
return text.trim().to_string();
}
}
// 3) output.sentence.text
if let Some(text) = output
.and_then(|o| o.get("sentence"))
.and_then(|s| s.get("text"))
.and_then(Value::as_str)
{
if !text.trim().is_empty() {
return text.trim().to_string();
}
}
// 4) 标准多模态 output.choices[0].message.content(字符串或 [{text}] 数组)
if let Some(content) = output
.and_then(|o| o.get("choices"))
.and_then(|c| c.as_array())
.and_then(|choices| choices.first())
.and_then(|choice| choice.get("message"))
.and_then(|message| message.get("content"))
{
if let Some(text) = content.as_str() {
return text.trim().to_string();
}
if let Some(items) = content.as_array() {
return items
.iter()
.filter_map(|item| item.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("")
.trim()
.to_string();
}
}
String::new()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::recorder::AudioConsumer;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::thread;
use std::time::{Duration, Instant};
#[test]
fn generation_url_from_full_endpoint_is_unchanged() {
assert_eq!(generation_url(DEFAULT_ENDPOINT).unwrap(), DEFAULT_ENDPOINT);
assert_eq!(
generation_url("https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation/").unwrap(),
DEFAULT_ENDPOINT
);
}
#[test]
fn generation_url_from_workspace_host_gets_canonical_path() {
assert_eq!(
generation_url("https://ws-xxx.cn-beijing.maas.aliyuncs.com").unwrap(),
"https://ws-xxx.cn-beijing.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
);
}
#[test]
fn body_uses_multimodal_generation_shape() {
let body = dashscope_multimodal_body(DEFAULT_MODEL, b"wav");
assert_eq!(body["model"], DEFAULT_MODEL);
let audio = &body["input"]["messages"][0]["content"][0];
assert_eq!(audio["type"], "input_audio");
assert!(audio["input_audio"]["data"]
.as_str()
.unwrap()
.starts_with("data:audio/wav;base64,"));
assert_eq!(body["parameters"]["format"], "wav");
assert_eq!(body["parameters"]["sample_rate"], "16000");
assert!(body["parameters"].get("vocabulary_id").is_none());
}
#[test]
fn extract_text_prefers_output_text() {
let json = serde_json::json!({ "output": { "text": " 你好世界 " } });
assert_eq!(extract_dashscope_text(&json), "你好世界");
}
#[test]
fn extract_text_falls_back_to_nested_sentence() {
let json = serde_json::json!({
"output": { "output": { "sentence": { "text": "嵌套句" } } }
});
assert_eq!(extract_dashscope_text(&json), "嵌套句");
}
#[test]
fn extract_text_falls_back_to_choices_content_array() {
let json = serde_json::json!({
"output": {
"choices": [{
"message": { "content": [{ "text": "第一段" }, { "text": "第二段" }] }
}]
}
});
assert_eq!(extract_dashscope_text(&json), "第一段第二段");
}
#[test]
fn extract_text_empty_when_no_known_path() {
let json = serde_json::json!({ "request_id": "abc", "output": {} });
assert_eq!(extract_dashscope_text(&json), "");
}
#[tokio::test]
async fn posts_multimodal_generation_request() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let addr = listener.local_addr().unwrap();
let server = thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(5);
let mut stream = loop {
match listener.accept() {
Ok((stream, _)) => break stream,
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
assert!(
Instant::now() < deadline,
"timed out waiting for DashScope ASR test request"
);
thread::sleep(Duration::from_millis(10));
}
Err(err) => panic!("accept DashScope ASR test request failed: {err}"),
}
};
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let request = read_http_request(&mut stream);
let request_text = String::from_utf8_lossy(&request);
let lower = request_text.to_ascii_lowercase();
assert!(request_text.starts_with(
"POST /api/v1/services/aigc/multimodal-generation/generation HTTP/1.1"
));
assert!(lower.contains("authorization: bearer sk-test"));
assert!(lower.contains("content-type: application/json"));
assert!(request_text.contains(r#""model":"fun-asr-flash-2026-06-15""#));
assert!(request_text.contains(r#""type":"input_audio""#));
assert!(request_text.contains("data:audio/wav;base64,"));
assert!(!request_text.contains("vocabulary_id"));
write_json_response(
&mut stream,
r#"{"output":{"text":"你好百炼"},"request_id":"r1"}"#,
);
});
let asr = DashScopeMultimodalASR::new(
"sk-test".to_string(),
format!(
"http://{}/api/v1/services/aigc/multimodal-generation/generation",
addr
),
DEFAULT_MODEL.to_string(),
);
asr.consume_pcm_chunk(&vec![0u8; 32_000]);
assert_eq!(asr.buffer_duration_ms(), 1_000);
let transcript = asr.transcribe().await.unwrap();
assert_eq!(transcript.text, "你好百炼");
assert_eq!(transcript.duration_ms, 1_000);
server.join().unwrap();
}
fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
let mut request = Vec::new();
let mut buf = [0u8; 4096];
let mut expected_len = None;
loop {
let read = stream.read(&mut buf).unwrap();
if read == 0 {
break;
}
request.extend_from_slice(&buf[..read]);
if expected_len.is_none() {
expected_len = parse_expected_request_len(&request);
}
if expected_len.is_some_and(|len| request.len() >= len) {
break;
}
}
request
}
fn parse_expected_request_len(request: &[u8]) -> Option<usize> {
let header_end = request.windows(4).position(|w| w == b"\r\n\r\n")? + 4;
let headers = String::from_utf8_lossy(&request[..header_end]);
let content_len = headers.lines().find_map(|line| {
let (name, value) = line.split_once(':')?;
if name.eq_ignore_ascii_case("content-length") {
value.trim().parse::<usize>().ok()
} else {
None
}
})?;
Some(header_end + content_len)
}
fn write_json_response(stream: &mut std::net::TcpStream, body: &str) {
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
stream.write_all(response.as_bytes()).unwrap();
stream.flush().unwrap();
}
}
+6 -2
View File
@@ -181,7 +181,9 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
super::pcm::pcm_duration_ms(pcm)
}
fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
/// 按时长把 PCM 切成多段(base64 进 JSON 的批量 ASR 都受单请求体积/时长限制)。
/// `dashscope_multimodal` 复用同一套切分逻辑,故 `pub(crate)`。
pub(crate) fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
if max_chunk_duration_ms == 0 {
return vec![pcm];
}
@@ -195,7 +197,9 @@ fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
pcm.chunks(bytes_per_chunk).collect()
}
fn join_transcript_chunks(chunks: &[String]) -> String {
/// 把分段识别文本按 CJK/标点规则拼回一句(段间按需补空格)。
/// `dashscope_multimodal` 复用同一套拼接逻辑,故 `pub(crate)`。
pub(crate) fn join_transcript_chunks(chunks: &[String]) -> String {
let mut joined = String::new();
for chunk in chunks.iter().map(|chunk| chunk.trim()) {
if chunk.is_empty() {
@@ -6,6 +6,7 @@
//! `volcengine.rs`.
pub mod bailian;
pub mod dashscope_multimodal;
mod frame;
pub mod local;
pub mod mimo;
@@ -16,6 +17,7 @@ pub mod wav;
pub mod whisper;
pub use bailian::{BailianCredentials, BailianRealtimeASR};
pub use dashscope_multimodal::DashScopeMultimodalASR;
pub use mimo::MimoBatchASR;
pub use qwen_realtime::{Qwen3RealtimeASR, Qwen3RealtimeCredentials};
pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR};
@@ -27,9 +27,8 @@ fn volcengine_configured(snap: &CredentialsSnapshot) -> bool {
}
pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool {
if provider == "volcengine" {
return volcengine_configured(snap);
}
// 本地 / 无凭据引擎不属于云端分类枚举(ActiveAsrProviderKind),由平台 cfg 门
// 在此单独判定;移动端上这些引擎不可用直接判未配置。
if cfg!(mobile)
&& (provider == crate::asr::local::PROVIDER_ID
|| provider == crate::asr::local::sherpa::PROVIDER_ID
@@ -46,17 +45,21 @@ pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnap
// 本地 ASR 不依赖云端凭据。
return true;
}
if provider == crate::asr::bailian::PROVIDER_ID
|| provider == crate::asr::qwen_realtime::PROVIDER_ID
{
return configured(&snap.asr_api_key);
// 云端 provider:所需字段由 ActiveAsrProviderKind 统一判定(穷尽 match,新增
// kind 编译器强制补齐)。volcengine 亦经此路(VolcAppKey)。
use crate::coordinator::{active_asr_provider_kind, AsrConfiguredFields};
match active_asr_provider_kind(provider).configured_fields() {
AsrConfiguredFields::ApiKeyOnly => configured(&snap.asr_api_key),
AsrConfiguredFields::ApiKeyEndpointModel => {
configured(&snap.asr_api_key)
&& configured(&snap.asr_endpoint)
&& configured(&snap.asr_model)
}
AsrConfiguredFields::EndpointModelOnly => {
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
}
AsrConfiguredFields::VolcAppKey => volcengine_configured(snap),
}
if provider == crate::asr::mimo::PROVIDER_ID {
return configured(&snap.asr_api_key)
&& configured(&snap.asr_endpoint)
&& configured(&snap.asr_model);
}
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
}
pub(crate) fn llm_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool {
@@ -27,11 +27,32 @@ pub async fn validate_provider_credentials(kind: String) -> Result<ProviderCheck
#[tauri::command]
pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult, String> {
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID {
// Bailian 实时 ASR 没有模型列表 HTTP 接口,列表只能是静态的。但直接返回
// 硬编码列表会让「拉取模型」在 Key / endpoint 全错时也显示成功,给用户
// "已连通"的错觉(其他厂商这颗按钮是真实带 Key GET /models)。先跑一次
// 与「验证」相同的 WebSocket 连通性检查,语义对齐后再返回静态列表。
if kind == "asr"
&& CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID
&& !cfg!(mobile)
{
// 统一「阿里云百炼」入口:三条协议(实时 fun-asr-realtime / 实时 qwen3 /
// 录音文件 fun-asr-flash)收成一个 provider。百炼各网关都没有模型列表 HTTP
// 接口,列表是静态的;但先跑一次与「验证」相同的、按当前所选模型对应协议的
// 连通性检查(validate_asr_provider 已按模型路由),避免 Key/endpoint 全错时
// 也显示成功。随后返回三个可选模型供下拉。
validate_asr_provider().await?;
// 静态清单只是常用快捷项;协议按模型名自动路由,用户也可在模型框直接手填
// 已支持的 DashScope ASR 模型;不支持的模型会在验证/开始录音前明确拒绝。
return Ok(ProviderModelsResult {
models: vec![
crate::asr::bailian::DEFAULT_MODEL.to_string(),
crate::asr::qwen_realtime::DEFAULT_MODEL.to_string(),
"qwen3-asr-flash-realtime-2026-02-10".to_string(),
"qwen3-asr-flash-realtime-2025-10-27".to_string(),
crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string(),
],
});
}
if kind == "asr"
&& CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID
&& cfg!(mobile)
{
validate_bailian_asr_provider().await?;
return Ok(ProviderModelsResult {
models: vec![crate::asr::bailian::DEFAULT_MODEL.to_string()],
@@ -55,6 +76,14 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
models: vec![crate::asr::mimo::DEFAULT_MODEL.to_string()],
});
}
if kind == "asr"
&& CredentialsVault::get_active_asr() == crate::asr::dashscope_multimodal::PROVIDER_ID
{
// multimodal-generation 无模型列表 HTTP 接口;与 mimo 同,返回静态别名。
return Ok(ProviderModelsResult {
models: vec![crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()],
});
}
if kind == "llm" && CredentialsVault::get_active_llm() == CODEX_OAUTH_PROVIDER_ID {
return Ok(ProviderModelsResult {
models: vec![
@@ -203,6 +232,19 @@ async fn validate_asr_provider() -> Result<(), String> {
}
if active_asr == crate::asr::bailian::PROVIDER_ID {
// 统一百炼:按所选模型验证对应协议(endpoint 由前端按模型同步,各 validator
// 读到的都是该协议的正确地址)。
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.unwrap_or_default();
let effective = crate::coordinator::resolve_effective_asr_provider(&active_asr, &model)?;
if effective == crate::asr::qwen_realtime::PROVIDER_ID {
return validate_qwen3_realtime_asr_provider().await;
}
if effective == crate::asr::dashscope_multimodal::PROVIDER_ID {
return validate_dashscope_multimodal_asr_provider().await;
}
return validate_bailian_asr_provider().await;
}
if active_asr == crate::asr::qwen_realtime::PROVIDER_ID {
@@ -211,6 +253,13 @@ async fn validate_asr_provider() -> Result<(), String> {
if active_asr == crate::asr::mimo::PROVIDER_ID {
return validate_mimo_asr_provider().await;
}
if active_asr == crate::asr::dashscope_multimodal::PROVIDER_ID {
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.unwrap_or_default();
crate::coordinator::validate_dashscope_multimodal_model(&model)?;
return validate_dashscope_multimodal_asr_provider().await;
}
let config = read_openai_provider_config("asr")?;
let model = CredentialsVault::get(CredentialAccount::AsrModel)
@@ -237,6 +286,73 @@ async fn validate_mimo_asr_provider() -> Result<(), String> {
.map_err(|e| e.to_string())
}
/// fun-asr-flash 官方公开示例音频,用于连通性校验。该模型对纯静音会返回
/// 400("no speech" 类错误),无法像 Whisper/Mimo 那样发静音探活;改用这段
/// 阿里官方文档在案的示例 wav(由 DashScope 侧拉取),key/endpoint/model 有效
/// 即返回 200。
const DASHSCOPE_ASR_VALIDATE_SAMPLE_URL: &str =
"https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav";
async fn validate_dashscope_multimodal_asr_provider() -> Result<(), String> {
// 统一百炼复用配置中的区域/工作空间主机,并推导 multimodal 的 https 路径。
// 隐藏别名仍按原有完整 endpoint 读取。
let (api_key, base_url) = if crate::coordinator::unified_bailian_is_active() {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.ok_or_else(|| "API Key 为空".to_string())?;
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.unwrap_or_default();
let endpoint = crate::coordinator::derive_bailian_endpoint(
&endpoint,
crate::coordinator::BailianEndpointProtocol::Multimodal,
)?;
(api_key, endpoint)
} else {
let config = read_openai_provider_config("asr")?;
(config.api_key, config.base_url)
};
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string());
crate::coordinator::validate_dashscope_multimodal_model(&model)?;
let url = crate::asr::dashscope_multimodal::generation_url(&base_url)
.map_err(|_| "endpointInvalid".to_string())?;
let body = serde_json::json!({
"model": model,
"input": { "messages": [{ "role": "user", "content": [{
"type": "input_audio",
"input_audio": { "data": DASHSCOPE_ASR_VALIDATE_SAMPLE_URL },
}]}]},
"parameters": { "format": "wav", "sample_rate": "16000" },
});
let client = http_client_builder(&url, 20)
.build()
.map_err(|_| "providerClientInitFailed".to_string())?;
let response = client
.post(&url)
.header("Authorization", format!("Bearer {}", api_key))
.header("Content-Type", "application/json")
.header("X-DashScope-SSE", "disable")
.json(&body)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
"providerRequestTimeout".to_string()
} else {
"providerNetworkError".to_string()
}
})?;
let status = response.status();
if !status.is_success() {
return Err(format!("providerHttpStatus:{}", status.as_u16()));
}
Ok(())
}
async fn validate_bailian_asr_provider() -> Result<(), String> {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
@@ -246,10 +362,18 @@ async fn validate_bailian_asr_provider() -> Result<(), String> {
}
// 已知残留(issue #609 F-01 孪生 gap):Bailian endpoint 走 `wss://`,与 http/https-only 的
// validate_llm_endpoint 不兼容,无法直接复用,需单独的 ws/wss 感知 SSRF 校验器(超本次范围)。
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
let stored_endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::bailian::DEFAULT_ENDPOINT.to_string());
let endpoint = if crate::coordinator::unified_bailian_is_active() {
crate::coordinator::derive_bailian_endpoint(
&stored_endpoint,
crate::coordinator::BailianEndpointProtocol::ClassicRealtime,
)?
} else {
stored_endpoint
};
// 协议头先行校验:填成 https://(百炼兼容模式 / 专属域名地址)时,WebSocket
// 握手报的 "URL scheme not supported" 会被前端兜底成笼统的「操作失败」,
// 用户无从定位。这里拦下并返回专用错误码,前端映射成可操作的提示。
@@ -293,10 +417,21 @@ async fn validate_qwen3_realtime_asr_provider() -> Result<(), String> {
if api_key.trim().is_empty() {
return Err("API Key 为空".to_string());
}
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string());
// 统一百炼保留配置中的区域/工作空间主机,并切换到 Qwen Realtime 路径。
let endpoint = if crate::coordinator::unified_bailian_is_active() {
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.unwrap_or_default();
crate::coordinator::derive_bailian_endpoint(
&endpoint,
crate::coordinator::BailianEndpointProtocol::QwenRealtime,
)?
} else {
CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string())
};
if !crate::asr::qwen_realtime::endpoint_scheme_is_secure_websocket(&endpoint) {
return Err("qwen3EndpointSchemeInvalid".to_string());
}
+357 -19
View File
@@ -25,9 +25,9 @@ use crate::asr::local::{
foundry, sherpa, FoundryLocalRuntime, FoundryLocalWhisperAsr, SherpaOnnxAsr, SherpaOnnxRuntime,
};
use crate::asr::{
BailianCredentials, BailianRealtimeASR, DictionaryHotword, MimoBatchASR, Qwen3RealtimeASR,
Qwen3RealtimeCredentials, RawTranscript, VolcengineCredentials, VolcengineStreamingASR,
WhisperBatchASR,
BailianCredentials, BailianRealtimeASR, DashScopeMultimodalASR, DictionaryHotword,
MimoBatchASR, Qwen3RealtimeASR, Qwen3RealtimeCredentials, RawTranscript, VolcengineCredentials,
VolcengineStreamingASR, WhisperBatchASR,
};
use crate::combo_hotkey::{ComboHotkeyError, ComboHotkeyEvent, ComboHotkeyMonitor};
use crate::coordinator_state::{
@@ -180,6 +180,8 @@ enum ActiveAsr {
Volcengine(Arc<VolcengineStreamingASR>),
Whisper(Arc<WhisperBatchASR>),
Mimo(Arc<MimoBatchASR>),
/// 百炼 Fun-ASR-Flash 录音文件识别(DashScope multimodal-generation 批量 HTTP)。
DashScopeMultimodal(Arc<DashScopeMultimodalASR>),
Bailian(Arc<BailianRealtimeASR>),
/// 百炼 Qwen3-ASR-Flash 实时(OpenAI Realtime 风格 WS 协议)。
Qwen3Realtime(Arc<Qwen3RealtimeASR>),
@@ -208,22 +210,82 @@ fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool {
}
}
/// 单一分类来源:云端 ASR provider id → 协议种类。本地/无凭据引擎(local qwen3 /
/// apple speech / foundry / sherpa)由各调用点在此之前用平台 cfg 门单独处理,不进
/// 这个枚举。
///
/// **加新云端通道的唯一改动点**:在 [`active_asr_provider_kind`] 加一条 id 映射,
/// 然后 [`ActiveAsrProviderKind::preflight_credential`] /
/// [`ActiveAsrProviderKind::configured_fields`] 与各 build/dispatch 的穷尽 `match`
/// 会被编译器逐个报错逼你补齐——不会再出现「装完才发现某处漏了」。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ActiveAsrProviderKind {
pub(crate) enum ActiveAsrProviderKind {
Bailian,
Qwen3Realtime,
Mimo,
DashScopeMultimodal,
WhisperCompatible,
Volcengine,
}
fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
/// 「能否开始一次会话」所需的凭据形态(对应 `ensure_asr_credentials` 预检门)。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AsrPreflightCredential {
/// 需要 ASR API Key(endpoint/model 有默认值兜底)。
AsrApiKey,
/// 需要火山引擎 App Key + Access Key。
VolcAppKey,
}
/// 概览页「已配置 / 未配置」状态所需的字段(对应 `asr_configured_for_provider`)。
/// 语义与预检门**有意不同**:预检问「能否开始」,这里问「preset 要求的字段填齐没」。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum AsrConfiguredFields {
/// 只看 API Key(endpoint/model 走默认):bailian / qwen3 实时。
ApiKeyOnly,
/// API Key + endpoint + model 都要填:mimo / dashscope multimodal。
ApiKeyEndpointModel,
/// 只看 endpoint + model(不含 API Key):Whisper 兼容厂商。
EndpointModelOnly,
/// 火山引擎三件套。
VolcAppKey,
}
impl ActiveAsrProviderKind {
pub(crate) fn preflight_credential(self) -> AsrPreflightCredential {
match self {
ActiveAsrProviderKind::Bailian
| ActiveAsrProviderKind::Qwen3Realtime
| ActiveAsrProviderKind::Mimo
| ActiveAsrProviderKind::DashScopeMultimodal
| ActiveAsrProviderKind::WhisperCompatible => AsrPreflightCredential::AsrApiKey,
ActiveAsrProviderKind::Volcengine => AsrPreflightCredential::VolcAppKey,
}
}
pub(crate) fn configured_fields(self) -> AsrConfiguredFields {
match self {
ActiveAsrProviderKind::Bailian | ActiveAsrProviderKind::Qwen3Realtime => {
AsrConfiguredFields::ApiKeyOnly
}
ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal => {
AsrConfiguredFields::ApiKeyEndpointModel
}
ActiveAsrProviderKind::WhisperCompatible => AsrConfiguredFields::EndpointModelOnly,
ActiveAsrProviderKind::Volcengine => AsrConfiguredFields::VolcAppKey,
}
}
}
pub(crate) fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
if is_bailian_provider(id) {
ActiveAsrProviderKind::Bailian
} else if is_qwen3_realtime_provider(id) {
ActiveAsrProviderKind::Qwen3Realtime
} else if is_mimo_provider(id) {
ActiveAsrProviderKind::Mimo
} else if is_dashscope_multimodal_provider(id) {
ActiveAsrProviderKind::DashScopeMultimodal
} else if is_whisper_compatible_provider(id) {
ActiveAsrProviderKind::WhisperCompatible
} else {
@@ -231,6 +293,105 @@ fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
}
}
/// 统一「阿里云百炼」入口的模型 → 底层协议 id 路由。
///
/// 三条百炼协议(fun-asr-realtime 经典实时 / qwen3-asr-flash-realtime Realtime /
/// fun-asr-flash 录音文件)在 UI 上收成一个 provider `bailian`(一把 key),**构建时**
/// 按所选模型二次路由到具体协议客户端。凭据 / 「已配置」判定仍看真实 active
/// `bailian`(→ ApiKeyOnly,一把 key),只有这里的 build 分发用得上 effective id。
///
/// 老用户若停在别名 id(`bailian-qwen3-realtime` / `bailian-fun-asr-flash`)上,
/// 非 `bailian` 直接原样返回,各走各的旧路径——即「隐藏别名」向后兼容。
pub(crate) fn resolve_effective_asr_provider(
active_asr: &str,
model: &str,
) -> Result<String, String> {
if !is_bailian_provider(active_asr) {
if is_dashscope_multimodal_provider(active_asr) {
validate_dashscope_multimodal_model(model)?;
}
return Ok(active_asr.to_string());
}
// Android/iOS 继续使用原来的 Bailian WebSocket 配置。统一入口的模型路由
// 只在桌面端启用,避免共享设置页意外改变移动端行为。
if cfg!(mobile) {
return Ok(crate::asr::bailian::PROVIDER_ID.to_string());
}
let model = model.trim();
if model.is_empty() || is_classic_bailian_realtime_model(model) {
Ok(crate::asr::bailian::PROVIDER_ID.to_string())
} else if model.starts_with("qwen3-asr-flash-realtime") {
Ok(crate::asr::qwen_realtime::PROVIDER_ID.to_string())
} else if model == crate::asr::dashscope_multimodal::DEFAULT_MODEL {
Ok(crate::asr::dashscope_multimodal::PROVIDER_ID.to_string())
} else {
Err(format!(
"不支持的百炼 ASR 模型:{model}。支持 fun-asr-realtime、paraformer-realtime、sensevoice-realtime、qwen3-asr-flash-realtime 和 fun-asr-flash-2026-06-15"
))
}
}
fn is_classic_bailian_realtime_model(model: &str) -> bool {
model.starts_with("fun-asr-realtime")
|| model.starts_with("paraformer-realtime")
|| model.starts_with("sensevoice-realtime")
}
pub(crate) fn validate_dashscope_multimodal_model(model: &str) -> Result<(), String> {
let model = model.trim();
if model.is_empty() || model == crate::asr::dashscope_multimodal::DEFAULT_MODEL {
return Ok(());
}
Err(format!(
"不支持的 DashScope 录音文件 ASR 模型:{model}。该协议仅支持 fun-asr-flash-2026-06-15;fun-asr-flash-8k-realtime 系列属于 8 kHz 实时 WebSocket 模型,当前尚未支持"
))
}
#[derive(Clone, Copy)]
pub(crate) enum BailianEndpointProtocol {
ClassicRealtime,
QwenRealtime,
Multimodal,
}
/// 统一百炼配置只需要表达区域/工作空间主机;具体协议的 scheme 与 path 由模型路由决定。
/// 这样既能复用同一个 endpoint 字段,也不会把中国区默认网关强加给新加坡或专属工作空间。
pub(crate) fn derive_bailian_endpoint(
endpoint: &str,
protocol: BailianEndpointProtocol,
) -> Result<String, String> {
let default_endpoint = match protocol {
BailianEndpointProtocol::ClassicRealtime => crate::asr::bailian::DEFAULT_ENDPOINT,
BailianEndpointProtocol::QwenRealtime => crate::asr::qwen_realtime::DEFAULT_ENDPOINT,
BailianEndpointProtocol::Multimodal => crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT,
};
let source = if endpoint.trim().is_empty() {
default_endpoint
} else {
endpoint.trim()
};
let mut url = url::Url::parse(source).map_err(|_| "endpointInvalid".to_string())?;
if url.host_str().is_none() {
return Err("endpointInvalid".to_string());
}
let (scheme, path) = match protocol {
BailianEndpointProtocol::ClassicRealtime => ("wss", "/api-ws/v1/inference/"),
BailianEndpointProtocol::QwenRealtime => ("wss", "/api-ws/v1/realtime"),
BailianEndpointProtocol::Multimodal => (
"https",
"/api/v1/services/aigc/multimodal-generation/generation",
),
};
url.set_scheme(scheme)
.map_err(|_| "endpointInvalid".to_string())?;
url.set_path(path);
url.set_query(None);
url.set_fragment(None);
Ok(url.to_string())
}
fn batch_asr_chunk_limit_ms(provider_id: &str) -> Option<u64> {
match provider_id {
// OpenRouter 把音频 base64 进 JSON body,体积比二进制大 ~33%,长录音易撞
@@ -1623,6 +1784,9 @@ impl Coordinator {
let consumer = start.recorder_consumer();
consumer.consume_pcm_chunk(&pcm);
let timeout = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS);
let dashscope_timeout = whisper_transcribe_timeout(
crate::asr::pcm::pcm_duration_ms(&pcm) as f64 / 1000.0,
);
let raw = match start.active_asr() {
ActiveAsr::Volcengine(asr) => {
asr.send_last_frame().await.map_err(|e| e.to_string())?;
@@ -1653,6 +1817,12 @@ impl Coordinator {
.await
.map_err(|_| "重新转录超时".to_string())?
.map_err(|e| e.to_string())?,
ActiveAsr::DashScopeMultimodal(m) => {
tokio::time::timeout(dashscope_timeout, m.transcribe())
.await
.map_err(|_| "重新转录超时".to_string())?
.map_err(|e| e.to_string())?
}
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
@@ -2050,25 +2220,54 @@ fn read_mimo_credentials() -> (String, String, String) {
(api_key, base_url, model)
}
fn read_dashscope_multimodal_credentials() -> (String, String, String) {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
let base_url = if unified_bailian_is_active() {
let endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT);
derive_bailian_endpoint(&endpoint, BailianEndpointProtocol::Multimodal).unwrap_or(endpoint)
} else {
CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT.to_string())
};
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string());
(api_key, base_url, model)
}
fn read_asr_vocabulary_id() -> Option<String> {
CredentialsVault::get(CredentialAccount::AsrVocabularyId)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
}
fn read_bailian_credentials() -> BailianCredentials {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::bailian::DEFAULT_ENDPOINT.to_string());
let stored_endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT);
let endpoint = if unified_bailian_is_active() {
derive_bailian_endpoint(&stored_endpoint, BailianEndpointProtocol::ClassicRealtime)
.unwrap_or(stored_endpoint)
} else {
stored_endpoint
};
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::bailian::DEFAULT_MODEL.to_string());
let vocabulary_id = CredentialsVault::get(CredentialAccount::AsrVocabularyId)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty());
let vocabulary_id = read_asr_vocabulary_id();
BailianCredentials {
api_key,
endpoint,
@@ -2077,16 +2276,38 @@ fn read_bailian_credentials() -> BailianCredentials {
}
}
fn read_asr_endpoint(default_endpoint: &str) -> String {
CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| default_endpoint.to_string())
}
/// 统一「阿里云百炼」入口的三条协议共用同一个 `bailian` 凭据条目。存储 endpoint
/// 提供区域/工作空间主机,运行时再按所选模型推导各协议的 scheme 与 path。
/// 老用户停在别名 id(`bailian-qwen3-realtime` / `bailian-fun-asr-flash`)上时不触发,
/// 仍读自己条目里存的 endpoint。
pub(crate) fn unified_bailian_is_active() -> bool {
CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID
}
fn read_qwen3_realtime_credentials() -> Qwen3RealtimeCredentials {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string());
let endpoint = if unified_bailian_is_active() {
let endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT);
derive_bailian_endpoint(&endpoint, BailianEndpointProtocol::QwenRealtime)
.unwrap_or(endpoint)
} else {
CredentialsVault::get(CredentialAccount::AsrEndpoint)
.ok()
.flatten()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string())
};
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
@@ -2461,10 +2682,127 @@ mod tests {
active_asr_provider_kind(crate::asr::mimo::PROVIDER_ID),
ActiveAsrProviderKind::Mimo
);
assert_eq!(
active_asr_provider_kind(crate::asr::dashscope_multimodal::PROVIDER_ID),
ActiveAsrProviderKind::DashScopeMultimodal
);
assert_eq!(
active_asr_provider_kind("volcengine"),
ActiveAsrProviderKind::Volcengine
);
// 未知 id 落到 Volcengine(与构建/凭据分发的兜底一致)。
assert_eq!(
active_asr_provider_kind("some-unknown-provider"),
ActiveAsrProviderKind::Volcengine
);
}
// 锁定分类枚举派生的凭据语义:重构把 ensure_asr_credentials /
// asr_configured_for_provider 从「字符串白名单 + 静默 else」改成对这两个方法的
// 穷尽 match,这里逐 kind 钉死映射,防止未来悄悄改动某个 provider 的凭据形态。
#[test]
fn preflight_credential_maps_every_kind() {
use AsrPreflightCredential::*;
use ActiveAsrProviderKind::*;
assert_eq!(Bailian.preflight_credential(), AsrApiKey);
assert_eq!(Qwen3Realtime.preflight_credential(), AsrApiKey);
assert_eq!(Mimo.preflight_credential(), AsrApiKey);
assert_eq!(DashScopeMultimodal.preflight_credential(), AsrApiKey);
assert_eq!(WhisperCompatible.preflight_credential(), AsrApiKey);
assert_eq!(Volcengine.preflight_credential(), VolcAppKey);
}
#[test]
fn resolve_effective_asr_provider_routes_bailian_by_model() {
let bailian = crate::asr::bailian::PROVIDER_ID;
// 统一百炼:按模型名路由到底层协议 id。
assert_eq!(
resolve_effective_asr_provider(bailian, "fun-asr-realtime").unwrap(),
crate::asr::bailian::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider(bailian, "qwen3-asr-flash-realtime").unwrap(),
crate::asr::qwen_realtime::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider(bailian, "qwen3-asr-flash-realtime-2026-02-10")
.unwrap(),
crate::asr::qwen_realtime::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider(bailian, "fun-asr-flash-2026-06-15").unwrap(),
crate::asr::dashscope_multimodal::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider(bailian, "paraformer-realtime-v2").unwrap(),
crate::asr::bailian::PROVIDER_ID
);
// 空模型 → 经典实时(百炼默认);未知模型应被拒绝。
assert_eq!(
resolve_effective_asr_provider(bailian, "").unwrap(),
crate::asr::bailian::PROVIDER_ID
);
// 非百炼 provider 原样返回(隐藏别名与其它厂商各走各的旧路径)。
assert_eq!(
resolve_effective_asr_provider(crate::asr::qwen_realtime::PROVIDER_ID, "anything")
.unwrap(),
crate::asr::qwen_realtime::PROVIDER_ID
);
assert_eq!(
resolve_effective_asr_provider("whisper", "whisper-1").unwrap(),
"whisper"
);
}
#[test]
fn resolve_effective_asr_provider_rejects_unsupported_bailian_model() {
let error = resolve_effective_asr_provider(crate::asr::bailian::PROVIDER_ID, "paraformer-v2")
.unwrap_err();
assert!(error.contains("不支持的百炼 ASR 模型"));
let error = resolve_effective_asr_provider(
crate::asr::bailian::PROVIDER_ID,
"fun-asr-flash-8k-realtime",
)
.unwrap_err();
assert!(error.contains("不支持的百炼 ASR 模型"));
}
#[test]
fn derive_bailian_endpoint_preserves_region_host_and_selects_protocol_path() {
let endpoint = "https://workspace.ap-southeast-1.maas.aliyuncs.com/custom?x=1";
assert_eq!(
derive_bailian_endpoint(endpoint, BailianEndpointProtocol::ClassicRealtime).unwrap(),
"wss://workspace.ap-southeast-1.maas.aliyuncs.com/api-ws/v1/inference/"
);
assert_eq!(
derive_bailian_endpoint(endpoint, BailianEndpointProtocol::QwenRealtime).unwrap(),
"wss://workspace.ap-southeast-1.maas.aliyuncs.com/api-ws/v1/realtime"
);
assert_eq!(
derive_bailian_endpoint(endpoint, BailianEndpointProtocol::Multimodal).unwrap(),
"https://workspace.ap-southeast-1.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
);
}
#[test]
fn derive_bailian_endpoint_uses_protocol_default_for_empty_value() {
assert_eq!(
derive_bailian_endpoint("", BailianEndpointProtocol::QwenRealtime).unwrap(),
crate::asr::qwen_realtime::DEFAULT_ENDPOINT
);
}
#[test]
fn configured_fields_maps_every_kind() {
use AsrConfiguredFields::*;
use ActiveAsrProviderKind::*;
assert_eq!(Bailian.configured_fields(), ApiKeyOnly);
assert_eq!(Qwen3Realtime.configured_fields(), ApiKeyOnly);
assert_eq!(Mimo.configured_fields(), ApiKeyEndpointModel);
assert_eq!(DashScopeMultimodal.configured_fields(), ApiKeyEndpointModel);
assert_eq!(WhisperCompatible.configured_fields(), EndpointModelOnly);
assert_eq!(Volcengine.configured_fields(), VolcAppKey);
}
#[cfg(target_os = "windows")]
@@ -104,26 +104,28 @@ pub(super) fn ensure_asr_credentials() -> Result<(), String> {
}
}
if is_whisper_compatible_provider(&active_asr)
|| is_bailian_provider(&active_asr)
|| is_qwen3_realtime_provider(&active_asr)
|| is_mimo_provider(&active_asr)
{
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
if api_key.trim().is_empty() {
return Err("请先在设置中填写 ASR 服务商 API Key".to_string());
// 云端 provider 的预检凭据由 ActiveAsrProviderKind 统一判定(穷尽 match,
// 编译器保证新增 kind 不会被漏掉 —— 取代旧的「provider 白名单 + 火山兜底」,
// 那个静默 else 曾让新通道误落到火山分支)。
match active_asr_provider_kind(&active_asr).preflight_credential() {
AsrPreflightCredential::AsrApiKey => {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.ok()
.flatten()
.unwrap_or_default();
if api_key.trim().is_empty() {
return Err("请先在设置中填写 ASR 服务商 API Key".to_string());
}
Ok(())
}
AsrPreflightCredential::VolcAppKey => {
let creds = read_volc_credentials();
if creds.app_id.trim().is_empty() || creds.access_token.trim().is_empty() {
Err("请先在设置中填写火山引擎 ASR App Key 和 Access Key".to_string())
} else {
Ok(())
}
}
return Ok(());
}
let creds = read_volc_credentials();
if creds.app_id.trim().is_empty() || creds.access_token.trim().is_empty() {
Err("请先在设置中填写火山引擎 ASR App Key 和 Access Key".to_string())
} else {
Ok(())
}
}
@@ -360,6 +362,10 @@ pub(super) fn is_mimo_provider(id: &str) -> bool {
id == crate::asr::mimo::PROVIDER_ID
}
pub(super) fn is_dashscope_multimodal_provider(id: &str) -> bool {
id == crate::asr::dashscope_multimodal::PROVIDER_ID
}
pub(super) fn apply_chinese_script_preference(text: &str, pref: ChineseScriptPreference) -> String {
if text.is_empty() {
return String::new();
@@ -530,7 +536,14 @@ pub(super) async fn build_qa_asr_start(inner: &Arc<Inner>, active_asr: &str) ->
return Ok(QaAsrStart::Ready { active, consumer });
}
match active_asr_provider_kind(active_asr) {
// 统一百炼:按所选模型把 build 分发重定向到具体协议(凭据仍读真实 active
// `bailian` 的那把 key;endpoint 由前端按模型同步好)。别名 id 原样返回。
let asr_model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.unwrap_or_default();
let effective_asr = resolve_effective_asr_provider(active_asr, &asr_model)?;
match active_asr_provider_kind(&effective_asr) {
ActiveAsrProviderKind::Bailian => Ok(QaAsrStart::Bailian {
asr: Arc::new(BailianRealtimeASR::new(read_bailian_credentials())),
bridge: Arc::new(DeferredAsrBridge::new()),
@@ -546,6 +559,13 @@ pub(super) async fn build_qa_asr_start(inner: &Arc<Inner>, active_asr: &str) ->
let consumer: Arc<dyn crate::recorder::AudioConsumer> = mimo;
Ok(QaAsrStart::Ready { active, consumer })
}
ActiveAsrProviderKind::DashScopeMultimodal => {
let (api_key, base_url, model) = read_dashscope_multimodal_credentials();
let asr = Arc::new(DashScopeMultimodalASR::new(api_key, base_url, model));
let active = ActiveAsr::DashScopeMultimodal(Arc::clone(&asr));
let consumer: Arc<dyn crate::recorder::AudioConsumer> = asr;
Ok(QaAsrStart::Ready { active, consumer })
}
ActiveAsrProviderKind::WhisperCompatible => {
let (api_key, base_url, model) = read_whisper_credentials();
let whisper_prompt =
@@ -1428,6 +1428,28 @@ pub(super) async fn begin_session_as(
}
let active_asr = CredentialsVault::get_active_asr();
let asr_model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.unwrap_or_default();
let effective_asr = match resolve_effective_asr_provider(&active_asr, &asr_model) {
Ok(provider) => provider,
Err(message) => {
log::warn!("[coord] ASR model routing rejected: {message}");
emit_capsule(
inner,
CapsuleState::Error,
0.0,
0,
Some(message.clone()),
None,
);
restore_prepared_windows_ime_session(inner, current_session_id);
inner.state.lock().phase = SessionPhase::Idle;
schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS);
return Err(message);
}
};
if let Err(message) = ensure_microphone_permission(inner) {
log::warn!("[coord] microphone permission gate failed: {message}");
@@ -1597,7 +1619,23 @@ pub(super) async fn begin_session_as(
return Ok(());
}
if is_bailian_provider(&active_asr) {
// 统一百炼:按所选模型把 build 分发重定向到具体协议 id(凭据仍读真实 active
// `bailian` 的那把 key;endpoint 由前端按模型同步)。别名 id 原样返回,走旧路径。
// 编译期护栏(exhaustiveness tripwire):下面这条云端构建 if-else 链最后是
// `else` 静默落到火山。这个穷尽的空 match 本身不做事,但新增
// ActiveAsrProviderKind 时会在此编译失败,逼作者回来给新 kind 补一条构建分支
// ——把「装完才发现漏了」的运行期坑变成编译期错误。QA 侧的 build_qa_asr_start
// 已是穷尽 match,两条构建路径都受编译器保护。
match active_asr_provider_kind(&effective_asr) {
ActiveAsrProviderKind::Bailian
| ActiveAsrProviderKind::Qwen3Realtime
| ActiveAsrProviderKind::Mimo
| ActiveAsrProviderKind::DashScopeMultimodal
| ActiveAsrProviderKind::WhisperCompatible
| ActiveAsrProviderKind::Volcengine => {}
}
if is_bailian_provider(&effective_asr) {
let asr = Arc::new(BailianRealtimeASR::new(read_bailian_credentials()));
let bridge = Arc::new(DeferredAsrBridge::new());
let consumer: Arc<dyn crate::recorder::AudioConsumer> = bridge.clone();
@@ -1669,7 +1707,7 @@ pub(super) async fn begin_session_as(
let flushed_bytes = bridge.attach(target);
log::info!("[coord] Bailian ASR connected; flushed {flushed_bytes} deferred audio bytes");
finish_starting_session(inner, current_session_id).await;
} else if is_qwen3_realtime_provider(&active_asr) {
} else if is_qwen3_realtime_provider(&effective_asr) {
// 与 Bailian 分支同构:流式 WS 会话 + DeferredAsrBridge 缓冲开链前音频。
let asr = Arc::new(Qwen3RealtimeASR::new(read_qwen3_realtime_credentials()));
let bridge = Arc::new(DeferredAsrBridge::new());
@@ -1746,7 +1784,7 @@ pub(super) async fn begin_session_as(
"[coord] Qwen3 realtime ASR connected; flushed {flushed_bytes} deferred audio bytes"
);
finish_starting_session(inner, current_session_id).await;
} else if is_mimo_provider(&active_asr) {
} else if is_mimo_provider(&effective_asr) {
let (api_key, base_url, model) = read_mimo_credentials();
let mimo = Arc::new(MimoBatchASR::new(api_key, base_url, model));
store_asr_for_session(
@@ -1757,7 +1795,18 @@ pub(super) async fn begin_session_as(
let consumer: Arc<dyn crate::recorder::AudioConsumer> = mimo;
start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer)
.await?;
} else if is_whisper_compatible_provider(&active_asr) {
} else if is_dashscope_multimodal_provider(&effective_asr) {
let (api_key, base_url, model) = read_dashscope_multimodal_credentials();
let asr = Arc::new(DashScopeMultimodalASR::new(api_key, base_url, model));
store_asr_for_session(
inner,
current_session_id,
ActiveAsr::DashScopeMultimodal(Arc::clone(&asr)),
);
let consumer: Arc<dyn crate::recorder::AudioConsumer> = asr;
start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer)
.await?;
} else if is_whisper_compatible_provider(&effective_asr) {
let (api_key, base_url, model) = read_whisper_credentials();
// 用户辞書の有効フレーズを Whisper の `prompt` に流し込む。固有名詞や
// 専門用語の同音・近形誤認識を ASR 段階で抑える。Polish LLM 側には
@@ -2453,6 +2502,34 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
}
}
}
ActiveAsr::DashScopeMultimodal(m) => {
debug_assert!(uses_global_timeout);
let audio_secs = m.buffer_duration_ms() as f64 / 1000.0;
let timeout_duration = whisper_transcribe_timeout(audio_secs);
log::info!(
"[coord] DashScope Fun-ASR-Flash dynamic timeout: {}s (audio {:.2}s)",
timeout_duration.as_secs(),
audio_secs
);
match tokio::time::timeout(timeout_duration, m.transcribe()).await {
Ok(Ok(r)) => Ok(r),
Ok(Err(e)) => {
log::error!("[coord] DashScope Fun-ASR-Flash transcribe failed: {e}");
Err(TranscribeFail::new(format!("识别失败: {e}"), e.to_string()))
}
Err(_) => {
log::error!(
"[coord] DashScope Fun-ASR-Flash dynamic timeout {}s (audio {:.2}s)",
timeout_duration.as_secs(),
audio_secs
);
Err(TranscribeFail::new(
"识别超时".to_string(),
"dashscope multimodal global timeout".to_string(),
))
}
}
}
ActiveAsr::Bailian(asr) => {
debug_assert!(uses_global_timeout);
if let Err(e) = asr.send_last_frame().await {
@@ -275,6 +275,16 @@ pub(super) async fn transcribe_overlay_dictation_asr(
Err(_) => Err("mimo global timeout".to_string()),
}
}
ActiveAsr::DashScopeMultimodal(asr) => {
debug_assert!(uses_global_timeout);
let audio_secs = asr.buffer_duration_ms() as f64 / 1000.0;
let timeout_duration = whisper_transcribe_timeout(audio_secs);
match tokio::time::timeout(timeout_duration, asr.transcribe()).await {
Ok(Ok(raw)) => Ok(raw),
Ok(Err(error)) => Err(error.to_string()),
Err(_) => Err("dashscope multimodal global timeout".to_string()),
}
}
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
@@ -860,6 +870,28 @@ pub(super) async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
}
}
}
ActiveAsr::DashScopeMultimodal(m) => {
debug_assert!(uses_global_timeout);
let audio_secs = m.buffer_duration_ms() as f64 / 1000.0;
let timeout_duration = whisper_transcribe_timeout(audio_secs);
match tokio::time::timeout(timeout_duration, m.transcribe()).await {
Ok(Ok(r)) => r,
Ok(Err(e)) => {
log::error!("[coord] QA: DashScope Fun-ASR-Flash transcribe failed: {e}");
finish_qa_with_error(inner, format!("识别失败: {e}"));
return Err(e.to_string());
}
Err(_) => {
log::error!(
"[coord] QA: DashScope Fun-ASR-Flash dynamic timeout {}s (audio {:.2}s)",
timeout_duration.as_secs(),
audio_secs
);
finish_qa_with_error(inner, "识别超时".to_string());
return Err("dashscope multimodal global timeout".to_string());
}
}
}
#[cfg(target_os = "windows")]
ActiveAsr::FoundryLocalWhisper(local) => {
debug_assert!(!uses_global_timeout);
@@ -68,6 +68,7 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) {
ActiveAsr::Volcengine(v) => v.cancel(),
ActiveAsr::Whisper(w) => w.cancel(),
ActiveAsr::Mimo(m) => m.cancel(),
ActiveAsr::DashScopeMultimodal(m) => m.cancel(),
ActiveAsr::Bailian(b) => b.cancel(),
ActiveAsr::Qwen3Realtime(q) => q.cancel(),
#[cfg(target_os = "windows")]
+3
View File
@@ -780,6 +780,7 @@ export const en: typeof zhCN = {
asrVolcengine: 'Volcengine bigasr',
asrBailian: 'Alibaba Bailian realtime ASR',
asrBailianQwen3: 'Bailian Qwen3 Realtime ASR',
asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash (recorded file)',
asrSiliconflow: 'SiliconFlow SenseVoice',
asrZhipu: 'Zhipu GLM-ASR',
asrGroq: 'Groq Whisper-large-v3',
@@ -820,6 +821,8 @@ export const en: typeof zhCN = {
thinkingModeHint: 'Off disables or minimizes thinking with provider-level official parameters. On enables thinking by channel defaults. No prompt injection or per-model adapters.',
bailianVocabularyIdLabel: 'Hotword Vocabulary ID (optional)',
bailianVocabularyIdNote: 'If you have created a DashScope hotword vocabulary, enter its vocab-... ID. Leave blank to skip hotwords.',
bailianModelRealtimeHint: 'Realtime model · transcribes as you speak.',
bailianModelRecordedFileHint: 'Recorded-file model · transcribes after you finish (single clip ≤ 5 min).',
appIdLabel: 'App ID',
accessKeyLabel: 'Access Key',
resourceIdLabel: 'Resource ID',
+3
View File
@@ -782,6 +782,7 @@ export const ja: typeof zhCN = {
asrVolcengine: 'Volcengine bigasr',
asrBailian: 'Alibaba Bailian リアルタイム ASR',
asrBailianQwen3: 'Bailian Qwen3 リアルタイム ASR',
asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash(録音ファイル)',
asrSiliconflow: 'SiliconFlow SenseVoice',
asrZhipu: 'Zhipu GLM-ASR',
asrGroq: 'Groq Whisper-large-v3',
@@ -822,6 +823,8 @@ export const ja: typeof zhCN = {
thinkingModeHint: 'オフではチャネル単位の公式パラメーターで思考を無効化または最小化します。オンではチャネル既定で思考を有効化します。prompt 注入やモデル別適配は行いません。',
bailianVocabularyIdLabel: 'ホットワード Vocabulary ID(任意)',
bailianVocabularyIdNote: 'DashScope でホットワード辞書を作成済みの場合は vocab-... ID を入力します。空欄なら送信しません。',
bailianModelRealtimeHint: 'リアルタイムモデル · 話しながら文字起こし。',
bailianModelRecordedFileHint: '録音ファイルモデル · 話し終えてから一括で文字起こし(1 本 ≤ 5 分)。',
appIdLabel: 'App ID(アプリケーション ID)',
accessKeyLabel: 'Access Key',
resourceIdLabel: 'Resource ID',
+3
View File
@@ -782,6 +782,7 @@ export const ko: typeof zhCN = {
asrVolcengine: 'Volcengine bigasr',
asrBailian: 'Alibaba Bailian 실시간 ASR',
asrBailianQwen3: 'Bailian Qwen3 실시간 ASR',
asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash (녹음 파일)',
asrSiliconflow: 'SiliconFlow SenseVoice',
asrZhipu: 'Zhipu GLM-ASR',
asrGroq: 'Groq Whisper-large-v3',
@@ -822,6 +823,8 @@ export const ko: typeof zhCN = {
thinkingModeHint: '꺼짐은 채널 단위 공식 파라미터로 사고를 끄거나 최소화합니다. 켜짐은 채널 기본값으로 사고를 켭니다. prompt 주입이나 모델별 어댑터는 사용하지 않습니다.',
bailianVocabularyIdLabel: '핫워드 Vocabulary ID(선택)',
bailianVocabularyIdNote: 'DashScope에서 핫워드 사전을 만들었다면 vocab-... ID를 입력하세요. 비워 두면 핫워드를 전송하지 않습니다.',
bailianModelRealtimeHint: '실시간 모델 · 말하는 동안 바로 전사.',
bailianModelRecordedFileHint: '녹음 파일 모델 · 말을 마친 뒤 전체 전사(한 클립 ≤ 5분).',
appIdLabel: 'App ID(애플리케이션 ID)',
accessKeyLabel: 'Access Key',
resourceIdLabel: 'Resource ID',
+3
View File
@@ -778,6 +778,7 @@ export const zhCN = {
asrVolcengine: '火山引擎 bigasr',
asrBailian: '阿里云百炼实时 ASR',
asrBailianQwen3: '阿里云百炼 Qwen3 实时 ASR',
asrBailianFunAsrFlash: '阿里云百炼 Fun-ASR-Flash(录音文件)',
asrSiliconflow: '硅基流动 SenseVoice',
asrZhipu: '智谱 GLM-ASR',
asrGroq: 'Groq Whisper-large-v3',
@@ -818,6 +819,8 @@ export const zhCN = {
thinkingModeHint: '关闭时按渠道级官方参数关闭或压低思考;开启时按渠道默认启用思考。不注入 prompt,也不做单模型适配。',
bailianVocabularyIdLabel: '热词 Vocabulary ID(可选)',
bailianVocabularyIdNote: '如已在百炼创建热词表,可填写 vocab-...;留空则不下发热词。',
bailianModelRealtimeHint: '实时模型 · 边说边出字。',
bailianModelRecordedFileHint: '录音文件模型 · 说完后整段转写(单条 ≤ 5 分钟)。',
appIdLabel: 'App ID(应用 ID)',
accessKeyLabel: 'Access Key',
resourceIdLabel: '资源 ID',
+3
View File
@@ -780,6 +780,7 @@ export const zhTW: typeof zhCN = {
asrVolcengine: '火山引擎 bigasr',
asrBailian: '阿里雲百煉即時 ASR',
asrBailianQwen3: '阿里雲百煉 Qwen3 即時 ASR',
asrBailianFunAsrFlash: '阿里雲百煉 Fun-ASR-Flash(錄音檔)',
asrSiliconflow: '硅基流動 SenseVoice',
asrZhipu: '智譜 GLM-ASR',
asrGroq: 'Groq Whisper-large-v3',
@@ -820,6 +821,8 @@ export const zhTW: typeof zhCN = {
thinkingModeHint: '關閉時按渠道級官方參數關閉或降低思考;開啟時按渠道預設啟用思考。不注入 prompt,也不做單模型適配。',
bailianVocabularyIdLabel: '熱詞 Vocabulary ID(可選)',
bailianVocabularyIdNote: '如已在百煉建立熱詞表,可填寫 vocab-...;留空則不下發熱詞。',
bailianModelRealtimeHint: '即時模型 · 邊說邊出字。',
bailianModelRecordedFileHint: '錄音檔模型 · 說完後整段轉寫(單條 ≤ 5 分鐘)。',
appIdLabel: 'App ID(應用 ID)',
accessKeyLabel: 'Access Key',
resourceIdLabel: '資源 ID',
+6 -13
View File
@@ -10,6 +10,7 @@ import { useMobileLayout } from '../lib/useMobileLayout';
import type { ActivityDay, CredentialsStatus, DictationSession, PolishMode } from '../lib/types';
import { useHotkeySettings } from '../state/HotkeySettingsContext';
import { Btn, Card, PageHeader, Pill } from './_atoms';
import { ASR_PRESETS } from './settings/shared';
function useModeLabels(): Record<PolishMode, string> {
const { t } = useTranslation();
@@ -25,19 +26,11 @@ interface OverviewProps {
onOpenHistory?: () => void;
}
const ASR_NAME_KEY_BY_ID: Record<string, string> = {
volcengine: 'asrVolcengine',
bailian: 'asrBailian',
siliconflow: 'asrSiliconflow',
zhipu: 'asrZhipu',
groq: 'asrGroq',
whisper: 'asrWhisper',
openrouter: 'asrOpenrouter',
'xiaomi-mimo-asr': 'asrXiaomiMimo',
'foundry-local-whisper': 'asrFoundryLocalWhisper',
'sherpa-onnx-local': 'asrSherpaOnnxLocal',
'local-qwen3': 'asrLocalQwen3',
};
// id → i18n nameKey,从 ASR_PRESETS 单一来源派生,避免这里再手维护一份 id 列表
// (之前漏了 bailian-qwen3-realtime / apple-speech,会退化成显示裸 id)。
const ASR_NAME_KEY_BY_ID: Record<string, string> = Object.fromEntries(
ASR_PRESETS.map(p => [p.id, p.nameKey]),
);
const LLM_NAME_KEY_BY_ID: Record<string, string> = {
ark: 'ark',
@@ -18,7 +18,7 @@ import { useMobileLayout } from '../../lib/useMobileLayout';
import { useHotkeySettings } from '../../state/HotkeySettingsContext';
import { SelectLite } from '../../components/ui/SelectLite';
import { Card } from '../_atoms';
import { SettingRow, SectionTitle, Toggle, inputStyle, type AsrPresetId } from './shared';
import { SettingRow, SectionTitle, Toggle, inputStyle, ASR_PRESETS, type AsrPresetId } from './shared';
function LlmThinkingToggle({ enabled, onToggle }: { enabled: boolean; onToggle: (next: boolean) => void }) {
const { t } = useTranslation();
@@ -144,39 +144,8 @@ type LlmPresetId = typeof LLM_PRESETS[number]['id'];
const ASR_DEFAULT_RESOURCE_ID = 'volc.seedasr.sauc.duration';
// `volcengine` / `bailian` 走自建流式客户端;其余走 OpenAI 兼容
// `/audio/transcriptions`(`coordinator.rs::is_whisper_compatible_provider`)。
// 新增兼容厂商:
// 1. 在这里加一项 `{ id, nameKey, baseUrl, model }`;
// 2. 若走 Whisper 协议,`coordinator.rs::is_whisper_compatible_provider` 加同名 id;
// 若是专有协议,新增独立 ASR client 与 provider kind;
// 3. 在 i18n 的 `settings.providers.presets.<nameKey>` 加文案。
// `AsrPresetId` 定义在 settings/shared.tsx,LocalModelSection / ProvidersSection 共用同一份。
const ASR_PRESETS: ReadonlyArray<{ id: AsrPresetId; nameKey: string; baseUrl: string; model: string }> = [
{ id: 'volcengine', nameKey: 'asrVolcengine', baseUrl: '', model: '' },
{ id: 'bailian', nameKey: 'asrBailian', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/inference/', model: 'fun-asr-realtime' },
// Qwen3-ASR-Flash 实时:OpenAI Realtime 风格 WS(/api-ws/v1/realtime),
// 与上面经典 inference 协议不同,由 asr/qwen_realtime.rs 专用 client 处理。
// 业务空间专属域名(wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime)同样可用。
{ id: 'bailian-qwen3-realtime', nameKey: 'asrBailianQwen3', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime', model: 'qwen3-asr-flash-realtime' },
{ id: 'siliconflow', nameKey: 'asrSiliconflow', baseUrl: 'https://api.siliconflow.cn/v1', model: 'FunAudioLLM/SenseVoiceSmall' },
{ id: 'zhipu', nameKey: 'asrZhipu', baseUrl: 'https://open.bigmodel.cn/api/paas/v4', model: 'glm-asr-2512' },
{ id: 'groq', nameKey: 'asrGroq', baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3-turbo' },
{ id: 'whisper', nameKey: 'asrWhisper', baseUrl: 'https://api.openai.com/v1', model: 'whisper-1' },
// OpenRouter 的 /audio/transcriptions 走 application/json + base64(issue #582),
// 后端 coordinator.rs::whisper_request_format 对该 id 切换到 OpenRouterJson 编码。
{ id: 'openrouter', nameKey: 'asrOpenrouter', baseUrl: 'https://openrouter.ai/api/v1', model: 'openai/whisper-large-v3-turbo' },
// 小米 MiMo ASR 按官方文档走 /chat/completions + input_audio,不是
// Whisper /audio/transcriptions;后端由 asr/mimo.rs 专用 client 处理。
{ id: 'xiaomi-mimo-asr', nameKey: 'asrXiaomiMimo', baseUrl: 'https://api.xiaomimimo.com/v1', model: 'mimo-v2.5-asr' },
{ id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' },
// 本地引擎(Foundry / sherpa-onnx / Qwen3):无 baseUrl/model 配置,
// 模型在「高级 → 本地模型」里下载与切换。
{ id: 'sherpa-onnx-local', nameKey: 'asrSherpaOnnxLocal', baseUrl: '', model: '' },
{ id: 'local-qwen3', nameKey: 'asrLocalQwen3', baseUrl: '', model: '' },
// Apple 系统语音识别(macOS):无 baseUrl/model、无下载、无凭据。
{ id: 'apple-speech', nameKey: 'asrAppleSpeech', baseUrl: '', model: '' },
];
// ASR_PRESETS 已上移到 settings/shared.tsx 作为单一来源(AsrPresetId 由其派生,
// Overview 的显示名映射也从那里取)。新增厂商的步骤见 shared.tsx 的注释。
type ProvidersSectionKind = 'all' | 'llm' | 'asr';
@@ -204,6 +173,12 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
const [llmModelRevision, setLlmModelRevision] = useState(0);
const [asrModelRevision, setAsrModelRevision] = useState(0);
const os = detectOS();
const unifiedBailian = committedAsrProvider === 'bailian' && os !== 'android';
const [bailianModel, setBailianModel] = useState('');
useEffect(() => {
if (committedAsrProvider !== 'bailian') setBailianModel('');
}, [committedAsrProvider]);
// 本地重引擎(qwen3 / sherpa / foundry)仍只在「高级 → 本地模型」里启用,
// 防止新手在主下拉误开 CPU 推理。Apple 语音是系统自带、零凭据、轻量,
// 在 macOS 上直接作为常规选项放进主下拉,方便随时选用 / 切走。
@@ -211,7 +186,12 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
p => p.id !== 'foundry-local-whisper'
&& p.id !== 'local-qwen3'
&& p.id !== 'sherpa-onnx-local'
&& (p.id !== 'apple-speech' || os === 'mac'),
&& (p.id !== 'apple-speech' || os === 'mac')
// 百炼三协议收成一个「阿里云百炼」入口(id=bailian)+ 模型下拉。qwen3 / fun-asr-flash
// 两个旧 id 作隐藏别名:新用户下拉里看不到,只有已经停在该 id 上的老用户仍显示,
// 保证其配置不被打断(见 coordinator::resolve_effective_asr_provider 的向后兼容)。
&& (p.id !== 'bailian-qwen3-realtime' || asrProvider === 'bailian-qwen3-realtime')
&& (p.id !== 'bailian-fun-asr-flash' || asrProvider === 'bailian-fun-asr-flash'),
);
useEffect(() => {
@@ -494,12 +474,17 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
) : (
<>
<CredentialField key={`${committedAsrProvider}:api_key`} label={t('settings.providers.apiKeyLabel')} account="asr.api_key" mono mask />
{/* 统一百炼保留 endpoint 供用户选择区域或工作空间域名;后端按模型转换协议与路径。 */}
<CredentialField key={`${committedAsrProvider}:endpoint`} label={t('settings.providers.baseUrlLabel')} account="asr.endpoint"
placeholder={asrPreset?.baseUrl || 'https://api.openai.com/v1'}
defaultValue={asrPreset?.baseUrl || undefined} />
<CredentialField key={`${committedAsrProvider}:model:${asrModelRevision}`} label={t('settings.providers.modelLabel')} account="asr.model"
placeholder={asrPreset?.model || 'whisper-1'} />
{committedAsrProvider === 'bailian' && (
placeholder={unifiedBailian ? 'fun-asr-realtime' : (asrPreset?.model || 'whisper-1')}
onValueChange={unifiedBailian ? setBailianModel : undefined} />
{unifiedBailian && (
<BailianProtocolHint key={`${committedAsrProvider}:proto:${asrModelRevision}`} currentModel={bailianModel} />
)}
{unifiedBailian && bailianModelSupportsVocabulary(bailianModel) && (
<>
<CredentialField
key={`${committedAsrProvider}:vocabulary_id`}
@@ -513,6 +498,7 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
</div>
</>
)}
{/* 统一百炼「拉取模型」只写 model,不覆盖用户选择的区域或工作空间 endpoint。 */}
<ProviderTools kind="asr" modelAccount="asr.model" onModelSelected={() => setAsrModelRevision(v => v + 1)} />
</>
)}
@@ -522,9 +508,55 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
);
}
// 统一「阿里云百炼」下,按模型名判断走哪种协议(与后端
// coordinator::resolve_effective_asr_provider 保持一致):qwen3-asr-flash-realtime* 与
// fun-asr-realtime* 与 fun-asr-flash-8k-realtime* 都是实时模型;当前支持的
// fun-asr-flash-2026-06-15 是「录音文件·说完转写」。
function bailianModelIsRecordedFile(model: string): boolean {
const m = model.trim();
return m === 'fun-asr-flash-2026-06-15';
}
function bailianModelSupportsVocabulary(model: string): boolean {
const m = model.trim();
return !m
|| m.startsWith('fun-asr-realtime')
|| m.startsWith('paraformer-realtime')
|| m.startsWith('sensevoice-realtime');
}
// 模型框下的一行协议提示,解决「三种模型看不出区别」——告诉用户当前模型是实时还是
// 录音文件、行为差异如何。随 asrModelRevision(拉取/选择模型时)与挂载时重读 asr.model。
function BailianProtocolHint({ currentModel }: { currentModel: string }) {
const { t } = useTranslation();
const [model, setModel] = useState('');
useEffect(() => {
let cancelled = false;
readCredential('asr.model')
.then(v => { if (!cancelled) setModel(v || 'fun-asr-realtime'); })
.catch(() => { /* 读失败按默认实时提示 */ });
return () => { cancelled = true; };
}, []);
useEffect(() => {
setModel(currentModel || 'fun-asr-realtime');
}, [currentModel]);
const hint = bailianModelIsRecordedFile(model)
? t('settings.providers.bailianModelRecordedFileHint')
: t('settings.providers.bailianModelRealtimeHint');
return (
<div style={{ marginTop: 2, fontSize: 11.5, color: 'var(--ol-ink-4)', lineHeight: 1.6 }}>
{hint}
</div>
);
}
type ProviderToolStatus = 'idle' | 'loading' | 'success' | 'empty' | 'error';
function ProviderTools({ kind, modelAccount, onModelSelected }: { kind: 'llm' | 'asr'; modelAccount: string; onModelSelected: () => void }) {
function ProviderTools({ kind, modelAccount, onModelSelected, showFetchModels = true }: { kind: 'llm' | 'asr'; modelAccount: string; onModelSelected: () => void; showFetchModels?: boolean }) {
const { t } = useTranslation();
const mobile = useMobileLayout();
const [models, setModels] = useState<string[]>([]);
@@ -595,8 +627,10 @@ function ProviderTools({ kind, modelAccount, onModelSelected }: { kind: 'llm' |
<div style={{ display: 'flex', flexDirection: 'column', gap: 8, width: '100%', maxWidth: mobile ? '100%' : 420 }}>
<div style={{ display: 'flex', gap: 6, alignItems: 'center', flexWrap: 'wrap', width: '100%' }}>
<button onClick={validate} style={miniBtnStyle} disabled={status === 'loading'}>{t('settings.providers.validate')}</button>
<button onClick={loadModels} style={miniBtnStyle} disabled={status === 'loading'}>{t('settings.providers.fetchModels')}</button>
{models.length > 0 && (
{showFetchModels && (
<button onClick={loadModels} style={miniBtnStyle} disabled={status === 'loading'}>{t('settings.providers.fetchModels')}</button>
)}
{showFetchModels && models.length > 0 && (
<SelectLite
value={selectedModel}
onChange={applyModel}
@@ -649,9 +683,10 @@ interface CredentialFieldProps {
mask?: boolean;
defaultValue?: string;
trailing?: ReactNode;
onValueChange?: (value: string) => void;
}
function CredentialField({ label, account, placeholder, mono, mask, defaultValue, trailing }: CredentialFieldProps) {
function CredentialField({ label, account, placeholder, mono, mask, defaultValue, trailing, onValueChange }: CredentialFieldProps) {
const { t } = useTranslation();
const mobile = useMobileLayout();
const [value, setValue] = useState('');
@@ -669,6 +704,7 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
setDirty(false);
setStatus('idle');
setValue('');
onValueChange?.('');
if (debounceRef.current) {
clearTimeout(debounceRef.current);
debounceRef.current = null;
@@ -677,18 +713,20 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
.then(v => {
if (cancelled) return;
setValue(v ?? '');
onValueChange?.(v ?? '');
setLoaded(true);
})
.catch(error => {
if (cancelled) return;
console.error('[settings] failed to read credential', account, error);
onValueChange?.('');
setLoaded(true);
setStatus('readError');
});
return () => {
cancelled = true;
};
}, [account]);
}, [account, onValueChange]);
useEffect(() => {
mountedRef.current = true;
@@ -739,6 +777,7 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
const handleChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const v = e.target.value;
setValue(v);
onValueChange?.(v);
if (!loaded) return;
setDirty(true);
if (debounceRef.current) clearTimeout(debounceRef.current);
@@ -757,6 +796,7 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
const fillDefault = async () => {
if (!loaded || !defaultValue) return;
setValue(defaultValue);
onValueChange?.(defaultValue);
setDirty(true);
await save(defaultValue, true);
};
+42 -16
View File
@@ -199,19 +199,45 @@ export const inputStyle: CSSProperties = {
"background 0.16s var(--ol-motion-quick), border-color 0.16s var(--ol-motion-quick)",
}
// ASR provider id 集合,跟 ProvidersSection.tsx::ASR_PRESETS 一一对应。
// 拆成独立类型让 LocalModelSection / ProvidersSection 都能用同一份不互相依赖。
export type AsrPresetId =
| "volcengine"
| "bailian"
| "bailian-qwen3-realtime"
| "siliconflow"
| "zhipu"
| "groq"
| "whisper"
| "openrouter"
| "xiaomi-mimo-asr"
| "foundry-local-whisper"
| "sherpa-onnx-local"
| "local-qwen3"
| "apple-speech"
// ASR provider preset 清单 —— 单一来源,放这里让 ProvidersSection /
// LocalModelSection / Overview 都从同一处取,不再各维护一份 id 列表导致漂移。
// `AsrPresetId` 由此派生(与 ProvidersSection 的 LLM_PRESETS→LlmPresetId 同构)。
//
// 新增兼容厂商:
// 1. 这里加一项 `{ id, nameKey, baseUrl, model }`;
// 2. 后端 `coordinator.rs::active_asr_provider_kind` 加 id→kind 映射 —— 之后
// `preflight_credential` / `configured_fields` 等穷尽 match 会被编译器逐个
// 报错逼你补齐;走 Whisper 协议再加进 `is_whisper_compatible_provider`,
// 专有协议另配独立 ASR client 与 provider kind;
// 3. i18n 的 `settings.providers.presets.<nameKey>` 补各语言文案。
export const ASR_PRESETS = [
{ id: 'volcengine', nameKey: 'asrVolcengine', baseUrl: '', model: '' },
{ id: 'bailian', nameKey: 'asrBailian', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/inference/', model: 'fun-asr-realtime' },
// Qwen3-ASR-Flash 实时:OpenAI Realtime 风格 WS(/api-ws/v1/realtime),
// 与上面经典 inference 协议不同,由 asr/qwen_realtime.rs 专用 client 处理。
// 业务空间专属域名(wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime)同样可用。
{ id: 'bailian-qwen3-realtime', nameKey: 'asrBailianQwen3', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime', model: 'qwen3-asr-flash-realtime' },
// Fun-ASR-Flash 录音文件识别:非实时,走 DashScope 私有的
// multimodal-generation HTTP 接口(既非实时 WS,也非 OpenAI /audio/transcriptions),
// 由 asr/dashscope_multimodal.rs 专用批量 client 处理。API key 与百炼同一把。
{ id: 'bailian-fun-asr-flash', nameKey: 'asrBailianFunAsrFlash', baseUrl: 'https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation', model: 'fun-asr-flash-2026-06-15' },
{ id: 'siliconflow', nameKey: 'asrSiliconflow', baseUrl: 'https://api.siliconflow.cn/v1', model: 'FunAudioLLM/SenseVoiceSmall' },
{ id: 'zhipu', nameKey: 'asrZhipu', baseUrl: 'https://open.bigmodel.cn/api/paas/v4', model: 'glm-asr-2512' },
{ id: 'groq', nameKey: 'asrGroq', baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3-turbo' },
{ id: 'whisper', nameKey: 'asrWhisper', baseUrl: 'https://api.openai.com/v1', model: 'whisper-1' },
// OpenRouter 的 /audio/transcriptions 走 application/json + base64(issue #582),
// 后端 coordinator.rs::whisper_request_format 对该 id 切换到 OpenRouterJson 编码。
{ id: 'openrouter', nameKey: 'asrOpenrouter', baseUrl: 'https://openrouter.ai/api/v1', model: 'openai/whisper-large-v3-turbo' },
// 小米 MiMo ASR 按官方文档走 /chat/completions + input_audio,不是
// Whisper /audio/transcriptions;后端由 asr/mimo.rs 专用 client 处理。
{ id: 'xiaomi-mimo-asr', nameKey: 'asrXiaomiMimo', baseUrl: 'https://api.xiaomimimo.com/v1', model: 'mimo-v2.5-asr' },
{ id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' },
// 本地引擎(Foundry / sherpa-onnx / Qwen3):无 baseUrl/model 配置,
// 模型在「高级 → 本地模型」里下载与切换。
{ id: 'sherpa-onnx-local', nameKey: 'asrSherpaOnnxLocal', baseUrl: '', model: '' },
{ id: 'local-qwen3', nameKey: 'asrLocalQwen3', baseUrl: '', model: '' },
// Apple 系统语音识别(macOS):无 baseUrl/model、无下载、无凭据。
{ id: 'apple-speech', nameKey: 'asrAppleSpeech', baseUrl: '', model: '' },
] as const;
export type AsrPresetId = typeof ASR_PRESETS[number]['id'];