feat(asr): 百炼 ASR 支持 fun-asr-flash 并统一三协议为单一入口 (#793)
* feat(asr): 百炼 ASR 支持 fun-asr-flash 并统一三协议为单一入口 * fix(asr): harden DashScope model routing and transcription * fix(asr): address Bailian protocol review feedback --------- Co-authored-by: Chris233 <h-chris233@outlook.com>
This commit is contained in:
1 parent
3f51edafab
commit
6e79a0c289
18 files changed
+1237
-139
No files matched your search
@@ -0,0 +1,412 @@
|
||||
//! 阿里云百炼(DashScope)多模态生成同步接口的批量 ASR 客户端。
|
||||
//!
|
||||
//! `fun-asr-flash` 系列是**非实时录音文件识别**模型,走 DashScope 私有的
|
||||
//! `multimodal-generation/generation` HTTP 接口,既不是实时 WebSocket 双工
|
||||
//! (见 `bailian.rs`),也不是 OpenAI 兼容的 `/audio/transcriptions`
|
||||
//! (见 `whisper.rs`)。因此单独成一路批量客户端:录音结束后把整段 PCM 编成
|
||||
//! WAV、base64 进 JSON body、POST 一次拿整段文本。
|
||||
//!
|
||||
//! 结构与 `mimo.rs`(同为「攒 PCM → POST 一段音频 → 解析私有 JSON」)一致,
|
||||
//! 复用其 `split_pcm_by_duration` / `join_transcript_chunks` 分片与拼接逻辑,
|
||||
//! 只有请求信封与响应解析不同。
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use base64::Engine;
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::asr::mimo::{join_transcript_chunks, split_pcm_by_duration};
|
||||
use crate::asr::wav::encode_wav_16k_mono;
|
||||
use crate::asr::RawTranscript;
|
||||
|
||||
// fun-asr-flash 单条音频上限 5 分钟;但真正的硬约束是 base64 进 JSON 的请求体
|
||||
// 体积。沿用 mimo 验证过的 180s 预算(16k/16-bit/mono WAV base64 后约 7.7MB),
|
||||
// 稳稳落在时长和常见网关体积上限之内。超长录音按此切分后逐段识别再拼接。
|
||||
const DASHSCOPE_MAX_CHUNK_DURATION_MS: u64 = 180_000;
|
||||
|
||||
pub const PROVIDER_ID: &str = "bailian-fun-asr-flash";
|
||||
pub const DEFAULT_ENDPOINT: &str =
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation";
|
||||
pub const DEFAULT_MODEL: &str = "fun-asr-flash-2026-06-15";
|
||||
|
||||
pub struct DashScopeMultimodalASR {
|
||||
api_key: String,
|
||||
base_url: String,
|
||||
model: String,
|
||||
buffer: Mutex<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl DashScopeMultimodalASR {
|
||||
pub fn new(api_key: String, base_url: String, model: String) -> Self {
|
||||
Self {
|
||||
api_key,
|
||||
base_url,
|
||||
model,
|
||||
buffer: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn buffer_duration_ms(&self) -> u64 {
|
||||
crate::asr::pcm::pcm_duration_ms(&self.buffer.lock())
|
||||
}
|
||||
|
||||
pub async fn transcribe(&self) -> Result<RawTranscript> {
|
||||
let pcm = self.buffer.lock().clone();
|
||||
if pcm.is_empty() {
|
||||
return Ok(RawTranscript {
|
||||
text: String::new(),
|
||||
duration_ms: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let result = self.transcribe_inner(&pcm).await;
|
||||
if result.is_ok() {
|
||||
self.buffer.lock().clear();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn transcribe_inner(&self, pcm: &[u8]) -> Result<RawTranscript> {
|
||||
if self.api_key.trim().is_empty() {
|
||||
anyhow::bail!("DashScope API key missing");
|
||||
}
|
||||
|
||||
let duration_ms = crate::asr::pcm::pcm_duration_ms(pcm);
|
||||
let chunks = split_pcm_by_duration(pcm, DASHSCOPE_MAX_CHUNK_DURATION_MS);
|
||||
let mut texts = Vec::with_capacity(chunks.len());
|
||||
for chunk in chunks {
|
||||
texts.push(self.transcribe_chunk(chunk).await?);
|
||||
}
|
||||
|
||||
Ok(RawTranscript {
|
||||
text: join_transcript_chunks(&texts),
|
||||
duration_ms,
|
||||
})
|
||||
}
|
||||
|
||||
async fn transcribe_chunk(&self, pcm: &[u8]) -> Result<String> {
|
||||
let samples: Vec<i16> = pcm
|
||||
.chunks_exact(2)
|
||||
.map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
|
||||
.collect();
|
||||
let wav = encode_wav_16k_mono(&samples);
|
||||
let body = dashscope_multimodal_body(&self.model, &wav);
|
||||
let url = generation_url(&self.base_url)?;
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.api_key.trim()))
|
||||
.header("Content-Type", "application/json")
|
||||
// multimodal-generation 默认可 SSE 流式;显式关掉走一次性 JSON 响应。
|
||||
.header("X-DashScope-SSE", "disable")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.context("DashScope ASR HTTP request failed")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("DashScope ASR API error {}: {}", status, body);
|
||||
}
|
||||
|
||||
let json: Value = resp.json().await.context("parse DashScope ASR response")?;
|
||||
Ok(extract_dashscope_text(&json).trim().to_string())
|
||||
}
|
||||
|
||||
pub fn cancel(&self) {
|
||||
self.buffer.lock().clear();
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::recorder::AudioConsumer for DashScopeMultimodalASR {
|
||||
fn consume_pcm_chunk(&self, pcm: &[u8]) {
|
||||
self.buffer.lock().extend_from_slice(pcm);
|
||||
}
|
||||
}
|
||||
|
||||
/// 归一化到 multimodal-generation 的完整 endpoint。
|
||||
///
|
||||
/// preset 默认下发的就是完整地址,命中首个分支直接用;用户若只填了业务空间
|
||||
/// 专属域名根(`https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com`)则补上标准
|
||||
/// 路径。其余情况保守地把标准后缀拼到用户给的路径后面。
|
||||
pub fn generation_url(base_url: &str) -> Result<String> {
|
||||
const CANONICAL_PATH: &str = "/api/v1/services/aigc/multimodal-generation/generation";
|
||||
let trimmed = base_url.trim();
|
||||
let parsed = reqwest::Url::parse(trimmed).context("parse DashScope base URL")?;
|
||||
let path = parsed.path().trim_end_matches('/');
|
||||
if path.ends_with("/multimodal-generation/generation") {
|
||||
let mut url = parsed.clone();
|
||||
url.set_path(path);
|
||||
return Ok(url.to_string());
|
||||
}
|
||||
let mut url = parsed.clone();
|
||||
if path.is_empty() {
|
||||
url.set_path(CANONICAL_PATH);
|
||||
} else {
|
||||
url.set_path(&format!("{path}{CANONICAL_PATH}"));
|
||||
}
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
pub fn dashscope_multimodal_body(model: &str, wav: &[u8]) -> Value {
|
||||
let audio_data = format!(
|
||||
"data:audio/wav;base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(wav)
|
||||
);
|
||||
serde_json::json!({
|
||||
"model": model,
|
||||
"input": {
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_audio",
|
||||
"input_audio": { "data": audio_data },
|
||||
}],
|
||||
}],
|
||||
},
|
||||
"parameters": {
|
||||
"format": "wav",
|
||||
"sample_rate": "16000",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// fun-asr-flash 的响应信封与标准多模态接口不同,且不同模型版本字段路径略有
|
||||
/// 差异(`output.text` / `output.output.sentence.text` / 标准 `choices`)。
|
||||
/// 这里按已知路径逐一兜底提取,取到第一个非空文本即返回,避免因单一路径假设
|
||||
/// 而在某个版本上静默丢字。
|
||||
pub fn extract_dashscope_text(json: &Value) -> String {
|
||||
let output = json.get("output");
|
||||
|
||||
// 1) output.text —— fun-asr-flash 文档主路径
|
||||
if let Some(text) = output.and_then(|o| o.get("text")).and_then(Value::as_str) {
|
||||
if !text.trim().is_empty() {
|
||||
return text.trim().to_string();
|
||||
}
|
||||
}
|
||||
|
||||
// 2) output.output.sentence.text —— 文档给出的另一种嵌套形态
|
||||
if let Some(text) = output
|
||||
.and_then(|o| o.get("output"))
|
||||
.and_then(|o| o.get("sentence"))
|
||||
.and_then(|s| s.get("text"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
if !text.trim().is_empty() {
|
||||
return text.trim().to_string();
|
||||
}
|
||||
}
|
||||
|
||||
// 3) output.sentence.text
|
||||
if let Some(text) = output
|
||||
.and_then(|o| o.get("sentence"))
|
||||
.and_then(|s| s.get("text"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
if !text.trim().is_empty() {
|
||||
return text.trim().to_string();
|
||||
}
|
||||
}
|
||||
|
||||
// 4) 标准多模态 output.choices[0].message.content(字符串或 [{text}] 数组)
|
||||
if let Some(content) = output
|
||||
.and_then(|o| o.get("choices"))
|
||||
.and_then(|c| c.as_array())
|
||||
.and_then(|choices| choices.first())
|
||||
.and_then(|choice| choice.get("message"))
|
||||
.and_then(|message| message.get("content"))
|
||||
{
|
||||
if let Some(text) = content.as_str() {
|
||||
return text.trim().to_string();
|
||||
}
|
||||
if let Some(items) = content.as_array() {
|
||||
return items
|
||||
.iter()
|
||||
.filter_map(|item| item.get("text").and_then(Value::as_str))
|
||||
.collect::<Vec<_>>()
|
||||
.join("")
|
||||
.trim()
|
||||
.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
String::new()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::recorder::AudioConsumer;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[test]
|
||||
fn generation_url_from_full_endpoint_is_unchanged() {
|
||||
assert_eq!(generation_url(DEFAULT_ENDPOINT).unwrap(), DEFAULT_ENDPOINT);
|
||||
assert_eq!(
|
||||
generation_url("https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation/").unwrap(),
|
||||
DEFAULT_ENDPOINT
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_url_from_workspace_host_gets_canonical_path() {
|
||||
assert_eq!(
|
||||
generation_url("https://ws-xxx.cn-beijing.maas.aliyuncs.com").unwrap(),
|
||||
"https://ws-xxx.cn-beijing.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_uses_multimodal_generation_shape() {
|
||||
let body = dashscope_multimodal_body(DEFAULT_MODEL, b"wav");
|
||||
assert_eq!(body["model"], DEFAULT_MODEL);
|
||||
let audio = &body["input"]["messages"][0]["content"][0];
|
||||
assert_eq!(audio["type"], "input_audio");
|
||||
assert!(audio["input_audio"]["data"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("data:audio/wav;base64,"));
|
||||
assert_eq!(body["parameters"]["format"], "wav");
|
||||
assert_eq!(body["parameters"]["sample_rate"], "16000");
|
||||
assert!(body["parameters"].get("vocabulary_id").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_text_prefers_output_text() {
|
||||
let json = serde_json::json!({ "output": { "text": " 你好世界 " } });
|
||||
assert_eq!(extract_dashscope_text(&json), "你好世界");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_text_falls_back_to_nested_sentence() {
|
||||
let json = serde_json::json!({
|
||||
"output": { "output": { "sentence": { "text": "嵌套句" } } }
|
||||
});
|
||||
assert_eq!(extract_dashscope_text(&json), "嵌套句");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_text_falls_back_to_choices_content_array() {
|
||||
let json = serde_json::json!({
|
||||
"output": {
|
||||
"choices": [{
|
||||
"message": { "content": [{ "text": "第一段" }, { "text": "第二段" }] }
|
||||
}]
|
||||
}
|
||||
});
|
||||
assert_eq!(extract_dashscope_text(&json), "第一段第二段");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_text_empty_when_no_known_path() {
|
||||
let json = serde_json::json!({ "request_id": "abc", "output": {} });
|
||||
assert_eq!(extract_dashscope_text(&json), "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn posts_multimodal_generation_request() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let mut stream = loop {
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => break stream,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"timed out waiting for DashScope ASR test request"
|
||||
);
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Err(err) => panic!("accept DashScope ASR test request failed: {err}"),
|
||||
}
|
||||
};
|
||||
stream.set_nonblocking(false).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
let request = read_http_request(&mut stream);
|
||||
let request_text = String::from_utf8_lossy(&request);
|
||||
let lower = request_text.to_ascii_lowercase();
|
||||
assert!(request_text.starts_with(
|
||||
"POST /api/v1/services/aigc/multimodal-generation/generation HTTP/1.1"
|
||||
));
|
||||
assert!(lower.contains("authorization: bearer sk-test"));
|
||||
assert!(lower.contains("content-type: application/json"));
|
||||
assert!(request_text.contains(r#""model":"fun-asr-flash-2026-06-15""#));
|
||||
assert!(request_text.contains(r#""type":"input_audio""#));
|
||||
assert!(request_text.contains("data:audio/wav;base64,"));
|
||||
assert!(!request_text.contains("vocabulary_id"));
|
||||
write_json_response(
|
||||
&mut stream,
|
||||
r#"{"output":{"text":"你好百炼"},"request_id":"r1"}"#,
|
||||
);
|
||||
});
|
||||
|
||||
let asr = DashScopeMultimodalASR::new(
|
||||
"sk-test".to_string(),
|
||||
format!(
|
||||
"http://{}/api/v1/services/aigc/multimodal-generation/generation",
|
||||
addr
|
||||
),
|
||||
DEFAULT_MODEL.to_string(),
|
||||
);
|
||||
asr.consume_pcm_chunk(&vec![0u8; 32_000]);
|
||||
assert_eq!(asr.buffer_duration_ms(), 1_000);
|
||||
let transcript = asr.transcribe().await.unwrap();
|
||||
|
||||
assert_eq!(transcript.text, "你好百炼");
|
||||
assert_eq!(transcript.duration_ms, 1_000);
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
|
||||
let mut request = Vec::new();
|
||||
let mut buf = [0u8; 4096];
|
||||
let mut expected_len = None;
|
||||
loop {
|
||||
let read = stream.read(&mut buf).unwrap();
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
request.extend_from_slice(&buf[..read]);
|
||||
if expected_len.is_none() {
|
||||
expected_len = parse_expected_request_len(&request);
|
||||
}
|
||||
if expected_len.is_some_and(|len| request.len() >= len) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
fn parse_expected_request_len(request: &[u8]) -> Option<usize> {
|
||||
let header_end = request.windows(4).position(|w| w == b"\r\n\r\n")? + 4;
|
||||
let headers = String::from_utf8_lossy(&request[..header_end]);
|
||||
let content_len = headers.lines().find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
if name.eq_ignore_ascii_case("content-length") {
|
||||
value.trim().parse::<usize>().ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})?;
|
||||
Some(header_end + content_len)
|
||||
}
|
||||
|
||||
fn write_json_response(stream: &mut std::net::TcpStream, body: &str) {
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
stream.write_all(response.as_bytes()).unwrap();
|
||||
stream.flush().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -181,7 +181,9 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
|
||||
super::pcm::pcm_duration_ms(pcm)
|
||||
}
|
||||
|
||||
fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
|
||||
/// 按时长把 PCM 切成多段(base64 进 JSON 的批量 ASR 都受单请求体积/时长限制)。
|
||||
/// `dashscope_multimodal` 复用同一套切分逻辑,故 `pub(crate)`。
|
||||
pub(crate) fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
|
||||
if max_chunk_duration_ms == 0 {
|
||||
return vec![pcm];
|
||||
}
|
||||
@@ -195,7 +197,9 @@ fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
|
||||
pcm.chunks(bytes_per_chunk).collect()
|
||||
}
|
||||
|
||||
fn join_transcript_chunks(chunks: &[String]) -> String {
|
||||
/// 把分段识别文本按 CJK/标点规则拼回一句(段间按需补空格)。
|
||||
/// `dashscope_multimodal` 复用同一套拼接逻辑,故 `pub(crate)`。
|
||||
pub(crate) fn join_transcript_chunks(chunks: &[String]) -> String {
|
||||
let mut joined = String::new();
|
||||
for chunk in chunks.iter().map(|chunk| chunk.trim()) {
|
||||
if chunk.is_empty() {
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! `volcengine.rs`.
|
||||
|
||||
pub mod bailian;
|
||||
pub mod dashscope_multimodal;
|
||||
mod frame;
|
||||
pub mod local;
|
||||
pub mod mimo;
|
||||
@@ -16,6 +17,7 @@ pub mod wav;
|
||||
pub mod whisper;
|
||||
|
||||
pub use bailian::{BailianCredentials, BailianRealtimeASR};
|
||||
pub use dashscope_multimodal::DashScopeMultimodalASR;
|
||||
pub use mimo::MimoBatchASR;
|
||||
pub use qwen_realtime::{Qwen3RealtimeASR, Qwen3RealtimeCredentials};
|
||||
pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR};
|
||||
|
||||
@@ -27,9 +27,8 @@ fn volcengine_configured(snap: &CredentialsSnapshot) -> bool {
|
||||
}
|
||||
|
||||
pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool {
|
||||
if provider == "volcengine" {
|
||||
return volcengine_configured(snap);
|
||||
}
|
||||
// 本地 / 无凭据引擎不属于云端分类枚举(ActiveAsrProviderKind),由平台 cfg 门
|
||||
// 在此单独判定;移动端上这些引擎不可用直接判未配置。
|
||||
if cfg!(mobile)
|
||||
&& (provider == crate::asr::local::PROVIDER_ID
|
||||
|| provider == crate::asr::local::sherpa::PROVIDER_ID
|
||||
@@ -46,17 +45,21 @@ pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnap
|
||||
// 本地 ASR 不依赖云端凭据。
|
||||
return true;
|
||||
}
|
||||
if provider == crate::asr::bailian::PROVIDER_ID
|
||||
|| provider == crate::asr::qwen_realtime::PROVIDER_ID
|
||||
{
|
||||
return configured(&snap.asr_api_key);
|
||||
// 云端 provider:所需字段由 ActiveAsrProviderKind 统一判定(穷尽 match,新增
|
||||
// kind 编译器强制补齐)。volcengine 亦经此路(VolcAppKey)。
|
||||
use crate::coordinator::{active_asr_provider_kind, AsrConfiguredFields};
|
||||
match active_asr_provider_kind(provider).configured_fields() {
|
||||
AsrConfiguredFields::ApiKeyOnly => configured(&snap.asr_api_key),
|
||||
AsrConfiguredFields::ApiKeyEndpointModel => {
|
||||
configured(&snap.asr_api_key)
|
||||
&& configured(&snap.asr_endpoint)
|
||||
&& configured(&snap.asr_model)
|
||||
}
|
||||
AsrConfiguredFields::EndpointModelOnly => {
|
||||
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
|
||||
}
|
||||
AsrConfiguredFields::VolcAppKey => volcengine_configured(snap),
|
||||
}
|
||||
if provider == crate::asr::mimo::PROVIDER_ID {
|
||||
return configured(&snap.asr_api_key)
|
||||
&& configured(&snap.asr_endpoint)
|
||||
&& configured(&snap.asr_model);
|
||||
}
|
||||
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
|
||||
}
|
||||
|
||||
pub(crate) fn llm_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool {
|
||||
|
||||
@@ -27,11 +27,32 @@ pub async fn validate_provider_credentials(kind: String) -> Result<ProviderCheck
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult, String> {
|
||||
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID {
|
||||
// Bailian 实时 ASR 没有模型列表 HTTP 接口,列表只能是静态的。但直接返回
|
||||
// 硬编码列表会让「拉取模型」在 Key / endpoint 全错时也显示成功,给用户
|
||||
// "已连通"的错觉(其他厂商这颗按钮是真实带 Key GET /models)。先跑一次
|
||||
// 与「验证」相同的 WebSocket 连通性检查,语义对齐后再返回静态列表。
|
||||
if kind == "asr"
|
||||
&& CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID
|
||||
&& !cfg!(mobile)
|
||||
{
|
||||
// 统一「阿里云百炼」入口:三条协议(实时 fun-asr-realtime / 实时 qwen3 /
|
||||
// 录音文件 fun-asr-flash)收成一个 provider。百炼各网关都没有模型列表 HTTP
|
||||
// 接口,列表是静态的;但先跑一次与「验证」相同的、按当前所选模型对应协议的
|
||||
// 连通性检查(validate_asr_provider 已按模型路由),避免 Key/endpoint 全错时
|
||||
// 也显示成功。随后返回三个可选模型供下拉。
|
||||
validate_asr_provider().await?;
|
||||
// 静态清单只是常用快捷项;协议按模型名自动路由,用户也可在模型框直接手填
|
||||
// 已支持的 DashScope ASR 模型;不支持的模型会在验证/开始录音前明确拒绝。
|
||||
return Ok(ProviderModelsResult {
|
||||
models: vec![
|
||||
crate::asr::bailian::DEFAULT_MODEL.to_string(),
|
||||
crate::asr::qwen_realtime::DEFAULT_MODEL.to_string(),
|
||||
"qwen3-asr-flash-realtime-2026-02-10".to_string(),
|
||||
"qwen3-asr-flash-realtime-2025-10-27".to_string(),
|
||||
crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string(),
|
||||
],
|
||||
});
|
||||
}
|
||||
if kind == "asr"
|
||||
&& CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID
|
||||
&& cfg!(mobile)
|
||||
{
|
||||
validate_bailian_asr_provider().await?;
|
||||
return Ok(ProviderModelsResult {
|
||||
models: vec![crate::asr::bailian::DEFAULT_MODEL.to_string()],
|
||||
@@ -55,6 +76,14 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
|
||||
models: vec![crate::asr::mimo::DEFAULT_MODEL.to_string()],
|
||||
});
|
||||
}
|
||||
if kind == "asr"
|
||||
&& CredentialsVault::get_active_asr() == crate::asr::dashscope_multimodal::PROVIDER_ID
|
||||
{
|
||||
// multimodal-generation 无模型列表 HTTP 接口;与 mimo 同,返回静态别名。
|
||||
return Ok(ProviderModelsResult {
|
||||
models: vec![crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()],
|
||||
});
|
||||
}
|
||||
if kind == "llm" && CredentialsVault::get_active_llm() == CODEX_OAUTH_PROVIDER_ID {
|
||||
return Ok(ProviderModelsResult {
|
||||
models: vec![
|
||||
@@ -203,6 +232,19 @@ async fn validate_asr_provider() -> Result<(), String> {
|
||||
}
|
||||
|
||||
if active_asr == crate::asr::bailian::PROVIDER_ID {
|
||||
// 统一百炼:按所选模型验证对应协议(endpoint 由前端按模型同步,各 validator
|
||||
// 读到的都是该协议的正确地址)。
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let effective = crate::coordinator::resolve_effective_asr_provider(&active_asr, &model)?;
|
||||
if effective == crate::asr::qwen_realtime::PROVIDER_ID {
|
||||
return validate_qwen3_realtime_asr_provider().await;
|
||||
}
|
||||
if effective == crate::asr::dashscope_multimodal::PROVIDER_ID {
|
||||
return validate_dashscope_multimodal_asr_provider().await;
|
||||
}
|
||||
return validate_bailian_asr_provider().await;
|
||||
}
|
||||
if active_asr == crate::asr::qwen_realtime::PROVIDER_ID {
|
||||
@@ -211,6 +253,13 @@ async fn validate_asr_provider() -> Result<(), String> {
|
||||
if active_asr == crate::asr::mimo::PROVIDER_ID {
|
||||
return validate_mimo_asr_provider().await;
|
||||
}
|
||||
if active_asr == crate::asr::dashscope_multimodal::PROVIDER_ID {
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.map_err(|e| e.to_string())?
|
||||
.unwrap_or_default();
|
||||
crate::coordinator::validate_dashscope_multimodal_model(&model)?;
|
||||
return validate_dashscope_multimodal_asr_provider().await;
|
||||
}
|
||||
|
||||
let config = read_openai_provider_config("asr")?;
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
@@ -237,6 +286,73 @@ async fn validate_mimo_asr_provider() -> Result<(), String> {
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// fun-asr-flash 官方公开示例音频,用于连通性校验。该模型对纯静音会返回
|
||||
/// 400("no speech" 类错误),无法像 Whisper/Mimo 那样发静音探活;改用这段
|
||||
/// 阿里官方文档在案的示例 wav(由 DashScope 侧拉取),key/endpoint/model 有效
|
||||
/// 即返回 200。
|
||||
const DASHSCOPE_ASR_VALIDATE_SAMPLE_URL: &str =
|
||||
"https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav";
|
||||
|
||||
async fn validate_dashscope_multimodal_asr_provider() -> Result<(), String> {
|
||||
// 统一百炼复用配置中的区域/工作空间主机,并推导 multimodal 的 https 路径。
|
||||
// 隐藏别名仍按原有完整 endpoint 读取。
|
||||
let (api_key, base_url) = if crate::coordinator::unified_bailian_is_active() {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.map_err(|e| e.to_string())?
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.ok_or_else(|| "API Key 为空".to_string())?;
|
||||
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.map_err(|e| e.to_string())?
|
||||
.unwrap_or_default();
|
||||
let endpoint = crate::coordinator::derive_bailian_endpoint(
|
||||
&endpoint,
|
||||
crate::coordinator::BailianEndpointProtocol::Multimodal,
|
||||
)?;
|
||||
(api_key, endpoint)
|
||||
} else {
|
||||
let config = read_openai_provider_config("asr")?;
|
||||
(config.api_key, config.base_url)
|
||||
};
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.map_err(|e| e.to_string())?
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string());
|
||||
crate::coordinator::validate_dashscope_multimodal_model(&model)?;
|
||||
let url = crate::asr::dashscope_multimodal::generation_url(&base_url)
|
||||
.map_err(|_| "endpointInvalid".to_string())?;
|
||||
let body = serde_json::json!({
|
||||
"model": model,
|
||||
"input": { "messages": [{ "role": "user", "content": [{
|
||||
"type": "input_audio",
|
||||
"input_audio": { "data": DASHSCOPE_ASR_VALIDATE_SAMPLE_URL },
|
||||
}]}]},
|
||||
"parameters": { "format": "wav", "sample_rate": "16000" },
|
||||
});
|
||||
let client = http_client_builder(&url, 20)
|
||||
.build()
|
||||
.map_err(|_| "providerClientInitFailed".to_string())?;
|
||||
let response = client
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", api_key))
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-DashScope-SSE", "disable")
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.is_timeout() {
|
||||
"providerRequestTimeout".to_string()
|
||||
} else {
|
||||
"providerNetworkError".to_string()
|
||||
}
|
||||
})?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(format!("providerHttpStatus:{}", status.as_u16()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_bailian_asr_provider() -> Result<(), String> {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.map_err(|e| e.to_string())?
|
||||
@@ -246,10 +362,18 @@ async fn validate_bailian_asr_provider() -> Result<(), String> {
|
||||
}
|
||||
// 已知残留(issue #609 F-01 孪生 gap):Bailian endpoint 走 `wss://`,与 http/https-only 的
|
||||
// validate_llm_endpoint 不兼容,无法直接复用,需单独的 ws/wss 感知 SSRF 校验器(超本次范围)。
|
||||
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
let stored_endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.map_err(|e| e.to_string())?
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::bailian::DEFAULT_ENDPOINT.to_string());
|
||||
let endpoint = if crate::coordinator::unified_bailian_is_active() {
|
||||
crate::coordinator::derive_bailian_endpoint(
|
||||
&stored_endpoint,
|
||||
crate::coordinator::BailianEndpointProtocol::ClassicRealtime,
|
||||
)?
|
||||
} else {
|
||||
stored_endpoint
|
||||
};
|
||||
// 协议头先行校验:填成 https://(百炼兼容模式 / 专属域名地址)时,WebSocket
|
||||
// 握手报的 "URL scheme not supported" 会被前端兜底成笼统的「操作失败」,
|
||||
// 用户无从定位。这里拦下并返回专用错误码,前端映射成可操作的提示。
|
||||
@@ -293,10 +417,21 @@ async fn validate_qwen3_realtime_asr_provider() -> Result<(), String> {
|
||||
if api_key.trim().is_empty() {
|
||||
return Err("API Key 为空".to_string());
|
||||
}
|
||||
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.map_err(|e| e.to_string())?
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string());
|
||||
// 统一百炼保留配置中的区域/工作空间主机,并切换到 Qwen Realtime 路径。
|
||||
let endpoint = if crate::coordinator::unified_bailian_is_active() {
|
||||
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.map_err(|e| e.to_string())?
|
||||
.unwrap_or_default();
|
||||
crate::coordinator::derive_bailian_endpoint(
|
||||
&endpoint,
|
||||
crate::coordinator::BailianEndpointProtocol::QwenRealtime,
|
||||
)?
|
||||
} else {
|
||||
CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.map_err(|e| e.to_string())?
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string())
|
||||
};
|
||||
if !crate::asr::qwen_realtime::endpoint_scheme_is_secure_websocket(&endpoint) {
|
||||
return Err("qwen3EndpointSchemeInvalid".to_string());
|
||||
}
|
||||
|
||||
@@ -25,9 +25,9 @@ use crate::asr::local::{
|
||||
foundry, sherpa, FoundryLocalRuntime, FoundryLocalWhisperAsr, SherpaOnnxAsr, SherpaOnnxRuntime,
|
||||
};
|
||||
use crate::asr::{
|
||||
BailianCredentials, BailianRealtimeASR, DictionaryHotword, MimoBatchASR, Qwen3RealtimeASR,
|
||||
Qwen3RealtimeCredentials, RawTranscript, VolcengineCredentials, VolcengineStreamingASR,
|
||||
WhisperBatchASR,
|
||||
BailianCredentials, BailianRealtimeASR, DashScopeMultimodalASR, DictionaryHotword,
|
||||
MimoBatchASR, Qwen3RealtimeASR, Qwen3RealtimeCredentials, RawTranscript, VolcengineCredentials,
|
||||
VolcengineStreamingASR, WhisperBatchASR,
|
||||
};
|
||||
use crate::combo_hotkey::{ComboHotkeyError, ComboHotkeyEvent, ComboHotkeyMonitor};
|
||||
use crate::coordinator_state::{
|
||||
@@ -180,6 +180,8 @@ enum ActiveAsr {
|
||||
Volcengine(Arc<VolcengineStreamingASR>),
|
||||
Whisper(Arc<WhisperBatchASR>),
|
||||
Mimo(Arc<MimoBatchASR>),
|
||||
/// 百炼 Fun-ASR-Flash 录音文件识别(DashScope multimodal-generation 批量 HTTP)。
|
||||
DashScopeMultimodal(Arc<DashScopeMultimodalASR>),
|
||||
Bailian(Arc<BailianRealtimeASR>),
|
||||
/// 百炼 Qwen3-ASR-Flash 实时(OpenAI Realtime 风格 WS 协议)。
|
||||
Qwen3Realtime(Arc<Qwen3RealtimeASR>),
|
||||
@@ -208,22 +210,82 @@ fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// 单一分类来源:云端 ASR provider id → 协议种类。本地/无凭据引擎(local qwen3 /
|
||||
/// apple speech / foundry / sherpa)由各调用点在此之前用平台 cfg 门单独处理,不进
|
||||
/// 这个枚举。
|
||||
///
|
||||
/// **加新云端通道的唯一改动点**:在 [`active_asr_provider_kind`] 加一条 id 映射,
|
||||
/// 然后 [`ActiveAsrProviderKind::preflight_credential`] /
|
||||
/// [`ActiveAsrProviderKind::configured_fields`] 与各 build/dispatch 的穷尽 `match`
|
||||
/// 会被编译器逐个报错逼你补齐——不会再出现「装完才发现某处漏了」。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ActiveAsrProviderKind {
|
||||
pub(crate) enum ActiveAsrProviderKind {
|
||||
Bailian,
|
||||
Qwen3Realtime,
|
||||
Mimo,
|
||||
DashScopeMultimodal,
|
||||
WhisperCompatible,
|
||||
Volcengine,
|
||||
}
|
||||
|
||||
fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
|
||||
/// 「能否开始一次会话」所需的凭据形态(对应 `ensure_asr_credentials` 预检门)。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum AsrPreflightCredential {
|
||||
/// 需要 ASR API Key(endpoint/model 有默认值兜底)。
|
||||
AsrApiKey,
|
||||
/// 需要火山引擎 App Key + Access Key。
|
||||
VolcAppKey,
|
||||
}
|
||||
|
||||
/// 概览页「已配置 / 未配置」状态所需的字段(对应 `asr_configured_for_provider`)。
|
||||
/// 语义与预检门**有意不同**:预检问「能否开始」,这里问「preset 要求的字段填齐没」。
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum AsrConfiguredFields {
|
||||
/// 只看 API Key(endpoint/model 走默认):bailian / qwen3 实时。
|
||||
ApiKeyOnly,
|
||||
/// API Key + endpoint + model 都要填:mimo / dashscope multimodal。
|
||||
ApiKeyEndpointModel,
|
||||
/// 只看 endpoint + model(不含 API Key):Whisper 兼容厂商。
|
||||
EndpointModelOnly,
|
||||
/// 火山引擎三件套。
|
||||
VolcAppKey,
|
||||
}
|
||||
|
||||
impl ActiveAsrProviderKind {
|
||||
pub(crate) fn preflight_credential(self) -> AsrPreflightCredential {
|
||||
match self {
|
||||
ActiveAsrProviderKind::Bailian
|
||||
| ActiveAsrProviderKind::Qwen3Realtime
|
||||
| ActiveAsrProviderKind::Mimo
|
||||
| ActiveAsrProviderKind::DashScopeMultimodal
|
||||
| ActiveAsrProviderKind::WhisperCompatible => AsrPreflightCredential::AsrApiKey,
|
||||
ActiveAsrProviderKind::Volcengine => AsrPreflightCredential::VolcAppKey,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn configured_fields(self) -> AsrConfiguredFields {
|
||||
match self {
|
||||
ActiveAsrProviderKind::Bailian | ActiveAsrProviderKind::Qwen3Realtime => {
|
||||
AsrConfiguredFields::ApiKeyOnly
|
||||
}
|
||||
ActiveAsrProviderKind::Mimo | ActiveAsrProviderKind::DashScopeMultimodal => {
|
||||
AsrConfiguredFields::ApiKeyEndpointModel
|
||||
}
|
||||
ActiveAsrProviderKind::WhisperCompatible => AsrConfiguredFields::EndpointModelOnly,
|
||||
ActiveAsrProviderKind::Volcengine => AsrConfiguredFields::VolcAppKey,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
|
||||
if is_bailian_provider(id) {
|
||||
ActiveAsrProviderKind::Bailian
|
||||
} else if is_qwen3_realtime_provider(id) {
|
||||
ActiveAsrProviderKind::Qwen3Realtime
|
||||
} else if is_mimo_provider(id) {
|
||||
ActiveAsrProviderKind::Mimo
|
||||
} else if is_dashscope_multimodal_provider(id) {
|
||||
ActiveAsrProviderKind::DashScopeMultimodal
|
||||
} else if is_whisper_compatible_provider(id) {
|
||||
ActiveAsrProviderKind::WhisperCompatible
|
||||
} else {
|
||||
@@ -231,6 +293,105 @@ fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
|
||||
}
|
||||
}
|
||||
|
||||
/// 统一「阿里云百炼」入口的模型 → 底层协议 id 路由。
|
||||
///
|
||||
/// 三条百炼协议(fun-asr-realtime 经典实时 / qwen3-asr-flash-realtime Realtime /
|
||||
/// fun-asr-flash 录音文件)在 UI 上收成一个 provider `bailian`(一把 key),**构建时**
|
||||
/// 按所选模型二次路由到具体协议客户端。凭据 / 「已配置」判定仍看真实 active
|
||||
/// `bailian`(→ ApiKeyOnly,一把 key),只有这里的 build 分发用得上 effective id。
|
||||
///
|
||||
/// 老用户若停在别名 id(`bailian-qwen3-realtime` / `bailian-fun-asr-flash`)上,
|
||||
/// 非 `bailian` 直接原样返回,各走各的旧路径——即「隐藏别名」向后兼容。
|
||||
pub(crate) fn resolve_effective_asr_provider(
|
||||
active_asr: &str,
|
||||
model: &str,
|
||||
) -> Result<String, String> {
|
||||
if !is_bailian_provider(active_asr) {
|
||||
if is_dashscope_multimodal_provider(active_asr) {
|
||||
validate_dashscope_multimodal_model(model)?;
|
||||
}
|
||||
return Ok(active_asr.to_string());
|
||||
}
|
||||
|
||||
// Android/iOS 继续使用原来的 Bailian WebSocket 配置。统一入口的模型路由
|
||||
// 只在桌面端启用,避免共享设置页意外改变移动端行为。
|
||||
if cfg!(mobile) {
|
||||
return Ok(crate::asr::bailian::PROVIDER_ID.to_string());
|
||||
}
|
||||
|
||||
let model = model.trim();
|
||||
if model.is_empty() || is_classic_bailian_realtime_model(model) {
|
||||
Ok(crate::asr::bailian::PROVIDER_ID.to_string())
|
||||
} else if model.starts_with("qwen3-asr-flash-realtime") {
|
||||
Ok(crate::asr::qwen_realtime::PROVIDER_ID.to_string())
|
||||
} else if model == crate::asr::dashscope_multimodal::DEFAULT_MODEL {
|
||||
Ok(crate::asr::dashscope_multimodal::PROVIDER_ID.to_string())
|
||||
} else {
|
||||
Err(format!(
|
||||
"不支持的百炼 ASR 模型:{model}。支持 fun-asr-realtime、paraformer-realtime、sensevoice-realtime、qwen3-asr-flash-realtime 和 fun-asr-flash-2026-06-15"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn is_classic_bailian_realtime_model(model: &str) -> bool {
|
||||
model.starts_with("fun-asr-realtime")
|
||||
|| model.starts_with("paraformer-realtime")
|
||||
|| model.starts_with("sensevoice-realtime")
|
||||
}
|
||||
|
||||
pub(crate) fn validate_dashscope_multimodal_model(model: &str) -> Result<(), String> {
|
||||
let model = model.trim();
|
||||
if model.is_empty() || model == crate::asr::dashscope_multimodal::DEFAULT_MODEL {
|
||||
return Ok(());
|
||||
}
|
||||
Err(format!(
|
||||
"不支持的 DashScope 录音文件 ASR 模型:{model}。该协议仅支持 fun-asr-flash-2026-06-15;fun-asr-flash-8k-realtime 系列属于 8 kHz 实时 WebSocket 模型,当前尚未支持"
|
||||
))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum BailianEndpointProtocol {
|
||||
ClassicRealtime,
|
||||
QwenRealtime,
|
||||
Multimodal,
|
||||
}
|
||||
|
||||
/// 统一百炼配置只需要表达区域/工作空间主机;具体协议的 scheme 与 path 由模型路由决定。
|
||||
/// 这样既能复用同一个 endpoint 字段,也不会把中国区默认网关强加给新加坡或专属工作空间。
|
||||
pub(crate) fn derive_bailian_endpoint(
|
||||
endpoint: &str,
|
||||
protocol: BailianEndpointProtocol,
|
||||
) -> Result<String, String> {
|
||||
let default_endpoint = match protocol {
|
||||
BailianEndpointProtocol::ClassicRealtime => crate::asr::bailian::DEFAULT_ENDPOINT,
|
||||
BailianEndpointProtocol::QwenRealtime => crate::asr::qwen_realtime::DEFAULT_ENDPOINT,
|
||||
BailianEndpointProtocol::Multimodal => crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT,
|
||||
};
|
||||
let source = if endpoint.trim().is_empty() {
|
||||
default_endpoint
|
||||
} else {
|
||||
endpoint.trim()
|
||||
};
|
||||
let mut url = url::Url::parse(source).map_err(|_| "endpointInvalid".to_string())?;
|
||||
if url.host_str().is_none() {
|
||||
return Err("endpointInvalid".to_string());
|
||||
}
|
||||
let (scheme, path) = match protocol {
|
||||
BailianEndpointProtocol::ClassicRealtime => ("wss", "/api-ws/v1/inference/"),
|
||||
BailianEndpointProtocol::QwenRealtime => ("wss", "/api-ws/v1/realtime"),
|
||||
BailianEndpointProtocol::Multimodal => (
|
||||
"https",
|
||||
"/api/v1/services/aigc/multimodal-generation/generation",
|
||||
),
|
||||
};
|
||||
url.set_scheme(scheme)
|
||||
.map_err(|_| "endpointInvalid".to_string())?;
|
||||
url.set_path(path);
|
||||
url.set_query(None);
|
||||
url.set_fragment(None);
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
fn batch_asr_chunk_limit_ms(provider_id: &str) -> Option<u64> {
|
||||
match provider_id {
|
||||
// OpenRouter 把音频 base64 进 JSON body,体积比二进制大 ~33%,长录音易撞
|
||||
@@ -1623,6 +1784,9 @@ impl Coordinator {
|
||||
let consumer = start.recorder_consumer();
|
||||
consumer.consume_pcm_chunk(&pcm);
|
||||
let timeout = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS);
|
||||
let dashscope_timeout = whisper_transcribe_timeout(
|
||||
crate::asr::pcm::pcm_duration_ms(&pcm) as f64 / 1000.0,
|
||||
);
|
||||
let raw = match start.active_asr() {
|
||||
ActiveAsr::Volcengine(asr) => {
|
||||
asr.send_last_frame().await.map_err(|e| e.to_string())?;
|
||||
@@ -1653,6 +1817,12 @@ impl Coordinator {
|
||||
.await
|
||||
.map_err(|_| "重新转录超时".to_string())?
|
||||
.map_err(|e| e.to_string())?,
|
||||
ActiveAsr::DashScopeMultimodal(m) => {
|
||||
tokio::time::timeout(dashscope_timeout, m.transcribe())
|
||||
.await
|
||||
.map_err(|_| "重新转录超时".to_string())?
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
ActiveAsr::FoundryLocalWhisper(local) => {
|
||||
let audio_secs = (local.buffer_duration_ms() as f64) / 1000.0;
|
||||
@@ -2050,25 +2220,54 @@ fn read_mimo_credentials() -> (String, String, String) {
|
||||
(api_key, base_url, model)
|
||||
}
|
||||
|
||||
fn read_dashscope_multimodal_credentials() -> (String, String, String) {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let base_url = if unified_bailian_is_active() {
|
||||
let endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT);
|
||||
derive_bailian_endpoint(&endpoint, BailianEndpointProtocol::Multimodal).unwrap_or(endpoint)
|
||||
} else {
|
||||
CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT.to_string())
|
||||
};
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string());
|
||||
(api_key, base_url, model)
|
||||
}
|
||||
|
||||
fn read_asr_vocabulary_id() -> Option<String> {
|
||||
CredentialsVault::get(CredentialAccount::AsrVocabularyId)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
}
|
||||
|
||||
fn read_bailian_credentials() -> BailianCredentials {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::bailian::DEFAULT_ENDPOINT.to_string());
|
||||
let stored_endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT);
|
||||
let endpoint = if unified_bailian_is_active() {
|
||||
derive_bailian_endpoint(&stored_endpoint, BailianEndpointProtocol::ClassicRealtime)
|
||||
.unwrap_or(stored_endpoint)
|
||||
} else {
|
||||
stored_endpoint
|
||||
};
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::bailian::DEFAULT_MODEL.to_string());
|
||||
let vocabulary_id = CredentialsVault::get(CredentialAccount::AsrVocabularyId)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty());
|
||||
let vocabulary_id = read_asr_vocabulary_id();
|
||||
BailianCredentials {
|
||||
api_key,
|
||||
endpoint,
|
||||
@@ -2077,16 +2276,38 @@ fn read_bailian_credentials() -> BailianCredentials {
|
||||
}
|
||||
}
|
||||
|
||||
fn read_asr_endpoint(default_endpoint: &str) -> String {
|
||||
CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| default_endpoint.to_string())
|
||||
}
|
||||
|
||||
/// 统一「阿里云百炼」入口的三条协议共用同一个 `bailian` 凭据条目。存储 endpoint
|
||||
/// 提供区域/工作空间主机,运行时再按所选模型推导各协议的 scheme 与 path。
|
||||
/// 老用户停在别名 id(`bailian-qwen3-realtime` / `bailian-fun-asr-flash`)上时不触发,
|
||||
/// 仍读自己条目里存的 endpoint。
|
||||
pub(crate) fn unified_bailian_is_active() -> bool {
|
||||
CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID
|
||||
}
|
||||
|
||||
fn read_qwen3_realtime_credentials() -> Qwen3RealtimeCredentials {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string());
|
||||
let endpoint = if unified_bailian_is_active() {
|
||||
let endpoint = read_asr_endpoint(crate::asr::bailian::DEFAULT_ENDPOINT);
|
||||
derive_bailian_endpoint(&endpoint, BailianEndpointProtocol::QwenRealtime)
|
||||
.unwrap_or(endpoint)
|
||||
} else {
|
||||
CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string())
|
||||
};
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
@@ -2461,10 +2682,127 @@ mod tests {
|
||||
active_asr_provider_kind(crate::asr::mimo::PROVIDER_ID),
|
||||
ActiveAsrProviderKind::Mimo
|
||||
);
|
||||
assert_eq!(
|
||||
active_asr_provider_kind(crate::asr::dashscope_multimodal::PROVIDER_ID),
|
||||
ActiveAsrProviderKind::DashScopeMultimodal
|
||||
);
|
||||
assert_eq!(
|
||||
active_asr_provider_kind("volcengine"),
|
||||
ActiveAsrProviderKind::Volcengine
|
||||
);
|
||||
// 未知 id 落到 Volcengine(与构建/凭据分发的兜底一致)。
|
||||
assert_eq!(
|
||||
active_asr_provider_kind("some-unknown-provider"),
|
||||
ActiveAsrProviderKind::Volcengine
|
||||
);
|
||||
}
|
||||
|
||||
// 锁定分类枚举派生的凭据语义:重构把 ensure_asr_credentials /
|
||||
// asr_configured_for_provider 从「字符串白名单 + 静默 else」改成对这两个方法的
|
||||
// 穷尽 match,这里逐 kind 钉死映射,防止未来悄悄改动某个 provider 的凭据形态。
|
||||
#[test]
|
||||
fn preflight_credential_maps_every_kind() {
|
||||
use AsrPreflightCredential::*;
|
||||
use ActiveAsrProviderKind::*;
|
||||
assert_eq!(Bailian.preflight_credential(), AsrApiKey);
|
||||
assert_eq!(Qwen3Realtime.preflight_credential(), AsrApiKey);
|
||||
assert_eq!(Mimo.preflight_credential(), AsrApiKey);
|
||||
assert_eq!(DashScopeMultimodal.preflight_credential(), AsrApiKey);
|
||||
assert_eq!(WhisperCompatible.preflight_credential(), AsrApiKey);
|
||||
assert_eq!(Volcengine.preflight_credential(), VolcAppKey);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_effective_asr_provider_routes_bailian_by_model() {
|
||||
let bailian = crate::asr::bailian::PROVIDER_ID;
|
||||
// 统一百炼:按模型名路由到底层协议 id。
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(bailian, "fun-asr-realtime").unwrap(),
|
||||
crate::asr::bailian::PROVIDER_ID
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(bailian, "qwen3-asr-flash-realtime").unwrap(),
|
||||
crate::asr::qwen_realtime::PROVIDER_ID
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(bailian, "qwen3-asr-flash-realtime-2026-02-10")
|
||||
.unwrap(),
|
||||
crate::asr::qwen_realtime::PROVIDER_ID
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(bailian, "fun-asr-flash-2026-06-15").unwrap(),
|
||||
crate::asr::dashscope_multimodal::PROVIDER_ID
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(bailian, "paraformer-realtime-v2").unwrap(),
|
||||
crate::asr::bailian::PROVIDER_ID
|
||||
);
|
||||
// 空模型 → 经典实时(百炼默认);未知模型应被拒绝。
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(bailian, "").unwrap(),
|
||||
crate::asr::bailian::PROVIDER_ID
|
||||
);
|
||||
// 非百炼 provider 原样返回(隐藏别名与其它厂商各走各的旧路径)。
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider(crate::asr::qwen_realtime::PROVIDER_ID, "anything")
|
||||
.unwrap(),
|
||||
crate::asr::qwen_realtime::PROVIDER_ID
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_effective_asr_provider("whisper", "whisper-1").unwrap(),
|
||||
"whisper"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_effective_asr_provider_rejects_unsupported_bailian_model() {
|
||||
let error = resolve_effective_asr_provider(crate::asr::bailian::PROVIDER_ID, "paraformer-v2")
|
||||
.unwrap_err();
|
||||
assert!(error.contains("不支持的百炼 ASR 模型"));
|
||||
|
||||
let error = resolve_effective_asr_provider(
|
||||
crate::asr::bailian::PROVIDER_ID,
|
||||
"fun-asr-flash-8k-realtime",
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(error.contains("不支持的百炼 ASR 模型"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_bailian_endpoint_preserves_region_host_and_selects_protocol_path() {
|
||||
let endpoint = "https://workspace.ap-southeast-1.maas.aliyuncs.com/custom?x=1";
|
||||
assert_eq!(
|
||||
derive_bailian_endpoint(endpoint, BailianEndpointProtocol::ClassicRealtime).unwrap(),
|
||||
"wss://workspace.ap-southeast-1.maas.aliyuncs.com/api-ws/v1/inference/"
|
||||
);
|
||||
assert_eq!(
|
||||
derive_bailian_endpoint(endpoint, BailianEndpointProtocol::QwenRealtime).unwrap(),
|
||||
"wss://workspace.ap-southeast-1.maas.aliyuncs.com/api-ws/v1/realtime"
|
||||
);
|
||||
assert_eq!(
|
||||
derive_bailian_endpoint(endpoint, BailianEndpointProtocol::Multimodal).unwrap(),
|
||||
"https://workspace.ap-southeast-1.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_bailian_endpoint_uses_protocol_default_for_empty_value() {
|
||||
assert_eq!(
|
||||
derive_bailian_endpoint("", BailianEndpointProtocol::QwenRealtime).unwrap(),
|
||||
crate::asr::qwen_realtime::DEFAULT_ENDPOINT
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_fields_maps_every_kind() {
|
||||
use AsrConfiguredFields::*;
|
||||
use ActiveAsrProviderKind::*;
|
||||
assert_eq!(Bailian.configured_fields(), ApiKeyOnly);
|
||||
assert_eq!(Qwen3Realtime.configured_fields(), ApiKeyOnly);
|
||||
assert_eq!(Mimo.configured_fields(), ApiKeyEndpointModel);
|
||||
assert_eq!(DashScopeMultimodal.configured_fields(), ApiKeyEndpointModel);
|
||||
assert_eq!(WhisperCompatible.configured_fields(), EndpointModelOnly);
|
||||
assert_eq!(Volcengine.configured_fields(), VolcAppKey);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
|
||||
@@ -104,26 +104,28 @@ pub(super) fn ensure_asr_credentials() -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
if is_whisper_compatible_provider(&active_asr)
|
||||
|| is_bailian_provider(&active_asr)
|
||||
|| is_qwen3_realtime_provider(&active_asr)
|
||||
|| is_mimo_provider(&active_asr)
|
||||
{
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
if api_key.trim().is_empty() {
|
||||
return Err("请先在设置中填写 ASR 服务商 API Key".to_string());
|
||||
// 云端 provider 的预检凭据由 ActiveAsrProviderKind 统一判定(穷尽 match,
|
||||
// 编译器保证新增 kind 不会被漏掉 —— 取代旧的「provider 白名单 + 火山兜底」,
|
||||
// 那个静默 else 曾让新通道误落到火山分支)。
|
||||
match active_asr_provider_kind(&active_asr).preflight_credential() {
|
||||
AsrPreflightCredential::AsrApiKey => {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
if api_key.trim().is_empty() {
|
||||
return Err("请先在设置中填写 ASR 服务商 API Key".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
AsrPreflightCredential::VolcAppKey => {
|
||||
let creds = read_volc_credentials();
|
||||
if creds.app_id.trim().is_empty() || creds.access_token.trim().is_empty() {
|
||||
Err("请先在设置中填写火山引擎 ASR App Key 和 Access Key".to_string())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let creds = read_volc_credentials();
|
||||
if creds.app_id.trim().is_empty() || creds.access_token.trim().is_empty() {
|
||||
Err("请先在设置中填写火山引擎 ASR App Key 和 Access Key".to_string())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,6 +362,10 @@ pub(super) fn is_mimo_provider(id: &str) -> bool {
|
||||
id == crate::asr::mimo::PROVIDER_ID
|
||||
}
|
||||
|
||||
pub(super) fn is_dashscope_multimodal_provider(id: &str) -> bool {
|
||||
id == crate::asr::dashscope_multimodal::PROVIDER_ID
|
||||
}
|
||||
|
||||
pub(super) fn apply_chinese_script_preference(text: &str, pref: ChineseScriptPreference) -> String {
|
||||
if text.is_empty() {
|
||||
return String::new();
|
||||
@@ -530,7 +536,14 @@ pub(super) async fn build_qa_asr_start(inner: &Arc<Inner>, active_asr: &str) ->
|
||||
return Ok(QaAsrStart::Ready { active, consumer });
|
||||
}
|
||||
|
||||
match active_asr_provider_kind(active_asr) {
|
||||
// 统一百炼:按所选模型把 build 分发重定向到具体协议(凭据仍读真实 active
|
||||
// `bailian` 的那把 key;endpoint 由前端按模型同步好)。别名 id 原样返回。
|
||||
let asr_model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let effective_asr = resolve_effective_asr_provider(active_asr, &asr_model)?;
|
||||
match active_asr_provider_kind(&effective_asr) {
|
||||
ActiveAsrProviderKind::Bailian => Ok(QaAsrStart::Bailian {
|
||||
asr: Arc::new(BailianRealtimeASR::new(read_bailian_credentials())),
|
||||
bridge: Arc::new(DeferredAsrBridge::new()),
|
||||
@@ -546,6 +559,13 @@ pub(super) async fn build_qa_asr_start(inner: &Arc<Inner>, active_asr: &str) ->
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = mimo;
|
||||
Ok(QaAsrStart::Ready { active, consumer })
|
||||
}
|
||||
ActiveAsrProviderKind::DashScopeMultimodal => {
|
||||
let (api_key, base_url, model) = read_dashscope_multimodal_credentials();
|
||||
let asr = Arc::new(DashScopeMultimodalASR::new(api_key, base_url, model));
|
||||
let active = ActiveAsr::DashScopeMultimodal(Arc::clone(&asr));
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = asr;
|
||||
Ok(QaAsrStart::Ready { active, consumer })
|
||||
}
|
||||
ActiveAsrProviderKind::WhisperCompatible => {
|
||||
let (api_key, base_url, model) = read_whisper_credentials();
|
||||
let whisper_prompt =
|
||||
|
||||
@@ -1428,6 +1428,28 @@ pub(super) async fn begin_session_as(
|
||||
}
|
||||
|
||||
let active_asr = CredentialsVault::get_active_asr();
|
||||
let asr_model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let effective_asr = match resolve_effective_asr_provider(&active_asr, &asr_model) {
|
||||
Ok(provider) => provider,
|
||||
Err(message) => {
|
||||
log::warn!("[coord] ASR model routing rejected: {message}");
|
||||
emit_capsule(
|
||||
inner,
|
||||
CapsuleState::Error,
|
||||
0.0,
|
||||
0,
|
||||
Some(message.clone()),
|
||||
None,
|
||||
);
|
||||
restore_prepared_windows_ime_session(inner, current_session_id);
|
||||
inner.state.lock().phase = SessionPhase::Idle;
|
||||
schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS);
|
||||
return Err(message);
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(message) = ensure_microphone_permission(inner) {
|
||||
log::warn!("[coord] microphone permission gate failed: {message}");
|
||||
@@ -1597,7 +1619,23 @@ pub(super) async fn begin_session_as(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if is_bailian_provider(&active_asr) {
|
||||
// 统一百炼:按所选模型把 build 分发重定向到具体协议 id(凭据仍读真实 active
|
||||
// `bailian` 的那把 key;endpoint 由前端按模型同步)。别名 id 原样返回,走旧路径。
|
||||
// 编译期护栏(exhaustiveness tripwire):下面这条云端构建 if-else 链最后是
|
||||
// `else` 静默落到火山。这个穷尽的空 match 本身不做事,但新增
|
||||
// ActiveAsrProviderKind 时会在此编译失败,逼作者回来给新 kind 补一条构建分支
|
||||
// ——把「装完才发现漏了」的运行期坑变成编译期错误。QA 侧的 build_qa_asr_start
|
||||
// 已是穷尽 match,两条构建路径都受编译器保护。
|
||||
match active_asr_provider_kind(&effective_asr) {
|
||||
ActiveAsrProviderKind::Bailian
|
||||
| ActiveAsrProviderKind::Qwen3Realtime
|
||||
| ActiveAsrProviderKind::Mimo
|
||||
| ActiveAsrProviderKind::DashScopeMultimodal
|
||||
| ActiveAsrProviderKind::WhisperCompatible
|
||||
| ActiveAsrProviderKind::Volcengine => {}
|
||||
}
|
||||
|
||||
if is_bailian_provider(&effective_asr) {
|
||||
let asr = Arc::new(BailianRealtimeASR::new(read_bailian_credentials()));
|
||||
let bridge = Arc::new(DeferredAsrBridge::new());
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = bridge.clone();
|
||||
@@ -1669,7 +1707,7 @@ pub(super) async fn begin_session_as(
|
||||
let flushed_bytes = bridge.attach(target);
|
||||
log::info!("[coord] Bailian ASR connected; flushed {flushed_bytes} deferred audio bytes");
|
||||
finish_starting_session(inner, current_session_id).await;
|
||||
} else if is_qwen3_realtime_provider(&active_asr) {
|
||||
} else if is_qwen3_realtime_provider(&effective_asr) {
|
||||
// 与 Bailian 分支同构:流式 WS 会话 + DeferredAsrBridge 缓冲开链前音频。
|
||||
let asr = Arc::new(Qwen3RealtimeASR::new(read_qwen3_realtime_credentials()));
|
||||
let bridge = Arc::new(DeferredAsrBridge::new());
|
||||
@@ -1746,7 +1784,7 @@ pub(super) async fn begin_session_as(
|
||||
"[coord] Qwen3 realtime ASR connected; flushed {flushed_bytes} deferred audio bytes"
|
||||
);
|
||||
finish_starting_session(inner, current_session_id).await;
|
||||
} else if is_mimo_provider(&active_asr) {
|
||||
} else if is_mimo_provider(&effective_asr) {
|
||||
let (api_key, base_url, model) = read_mimo_credentials();
|
||||
let mimo = Arc::new(MimoBatchASR::new(api_key, base_url, model));
|
||||
store_asr_for_session(
|
||||
@@ -1757,7 +1795,18 @@ pub(super) async fn begin_session_as(
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = mimo;
|
||||
start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer)
|
||||
.await?;
|
||||
} else if is_whisper_compatible_provider(&active_asr) {
|
||||
} else if is_dashscope_multimodal_provider(&effective_asr) {
|
||||
let (api_key, base_url, model) = read_dashscope_multimodal_credentials();
|
||||
let asr = Arc::new(DashScopeMultimodalASR::new(api_key, base_url, model));
|
||||
store_asr_for_session(
|
||||
inner,
|
||||
current_session_id,
|
||||
ActiveAsr::DashScopeMultimodal(Arc::clone(&asr)),
|
||||
);
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = asr;
|
||||
start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer)
|
||||
.await?;
|
||||
} else if is_whisper_compatible_provider(&effective_asr) {
|
||||
let (api_key, base_url, model) = read_whisper_credentials();
|
||||
// 用户辞書の有効フレーズを Whisper の `prompt` に流し込む。固有名詞や
|
||||
// 専門用語の同音・近形誤認識を ASR 段階で抑える。Polish LLM 側には
|
||||
@@ -2453,6 +2502,34 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
}
|
||||
ActiveAsr::DashScopeMultimodal(m) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
let audio_secs = m.buffer_duration_ms() as f64 / 1000.0;
|
||||
let timeout_duration = whisper_transcribe_timeout(audio_secs);
|
||||
log::info!(
|
||||
"[coord] DashScope Fun-ASR-Flash dynamic timeout: {}s (audio {:.2}s)",
|
||||
timeout_duration.as_secs(),
|
||||
audio_secs
|
||||
);
|
||||
match tokio::time::timeout(timeout_duration, m.transcribe()).await {
|
||||
Ok(Ok(r)) => Ok(r),
|
||||
Ok(Err(e)) => {
|
||||
log::error!("[coord] DashScope Fun-ASR-Flash transcribe failed: {e}");
|
||||
Err(TranscribeFail::new(format!("识别失败: {e}"), e.to_string()))
|
||||
}
|
||||
Err(_) => {
|
||||
log::error!(
|
||||
"[coord] DashScope Fun-ASR-Flash dynamic timeout {}s (audio {:.2}s)",
|
||||
timeout_duration.as_secs(),
|
||||
audio_secs
|
||||
);
|
||||
Err(TranscribeFail::new(
|
||||
"识别超时".to_string(),
|
||||
"dashscope multimodal global timeout".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
ActiveAsr::Bailian(asr) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
if let Err(e) = asr.send_last_frame().await {
|
||||
|
||||
@@ -275,6 +275,16 @@ pub(super) async fn transcribe_overlay_dictation_asr(
|
||||
Err(_) => Err("mimo global timeout".to_string()),
|
||||
}
|
||||
}
|
||||
ActiveAsr::DashScopeMultimodal(asr) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
let audio_secs = asr.buffer_duration_ms() as f64 / 1000.0;
|
||||
let timeout_duration = whisper_transcribe_timeout(audio_secs);
|
||||
match tokio::time::timeout(timeout_duration, asr.transcribe()).await {
|
||||
Ok(Ok(raw)) => Ok(raw),
|
||||
Ok(Err(error)) => Err(error.to_string()),
|
||||
Err(_) => Err("dashscope multimodal global timeout".to_string()),
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
ActiveAsr::FoundryLocalWhisper(local) => {
|
||||
debug_assert!(!uses_global_timeout);
|
||||
@@ -860,6 +870,28 @@ pub(super) async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
}
|
||||
ActiveAsr::DashScopeMultimodal(m) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
let audio_secs = m.buffer_duration_ms() as f64 / 1000.0;
|
||||
let timeout_duration = whisper_transcribe_timeout(audio_secs);
|
||||
match tokio::time::timeout(timeout_duration, m.transcribe()).await {
|
||||
Ok(Ok(r)) => r,
|
||||
Ok(Err(e)) => {
|
||||
log::error!("[coord] QA: DashScope Fun-ASR-Flash transcribe failed: {e}");
|
||||
finish_qa_with_error(inner, format!("识别失败: {e}"));
|
||||
return Err(e.to_string());
|
||||
}
|
||||
Err(_) => {
|
||||
log::error!(
|
||||
"[coord] QA: DashScope Fun-ASR-Flash dynamic timeout {}s (audio {:.2}s)",
|
||||
timeout_duration.as_secs(),
|
||||
audio_secs
|
||||
);
|
||||
finish_qa_with_error(inner, "识别超时".to_string());
|
||||
return Err("dashscope multimodal global timeout".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
ActiveAsr::FoundryLocalWhisper(local) => {
|
||||
debug_assert!(!uses_global_timeout);
|
||||
|
||||
@@ -68,6 +68,7 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) {
|
||||
ActiveAsr::Volcengine(v) => v.cancel(),
|
||||
ActiveAsr::Whisper(w) => w.cancel(),
|
||||
ActiveAsr::Mimo(m) => m.cancel(),
|
||||
ActiveAsr::DashScopeMultimodal(m) => m.cancel(),
|
||||
ActiveAsr::Bailian(b) => b.cancel(),
|
||||
ActiveAsr::Qwen3Realtime(q) => q.cancel(),
|
||||
#[cfg(target_os = "windows")]
|
||||
|
||||
@@ -780,6 +780,7 @@ export const en: typeof zhCN = {
|
||||
asrVolcengine: 'Volcengine bigasr',
|
||||
asrBailian: 'Alibaba Bailian realtime ASR',
|
||||
asrBailianQwen3: 'Bailian Qwen3 Realtime ASR',
|
||||
asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash (recorded file)',
|
||||
asrSiliconflow: 'SiliconFlow SenseVoice',
|
||||
asrZhipu: 'Zhipu GLM-ASR',
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
@@ -820,6 +821,8 @@ export const en: typeof zhCN = {
|
||||
thinkingModeHint: 'Off disables or minimizes thinking with provider-level official parameters. On enables thinking by channel defaults. No prompt injection or per-model adapters.',
|
||||
bailianVocabularyIdLabel: 'Hotword Vocabulary ID (optional)',
|
||||
bailianVocabularyIdNote: 'If you have created a DashScope hotword vocabulary, enter its vocab-... ID. Leave blank to skip hotwords.',
|
||||
bailianModelRealtimeHint: 'Realtime model · transcribes as you speak.',
|
||||
bailianModelRecordedFileHint: 'Recorded-file model · transcribes after you finish (single clip ≤ 5 min).',
|
||||
appIdLabel: 'App ID',
|
||||
accessKeyLabel: 'Access Key',
|
||||
resourceIdLabel: 'Resource ID',
|
||||
|
||||
@@ -782,6 +782,7 @@ export const ja: typeof zhCN = {
|
||||
asrVolcengine: 'Volcengine bigasr',
|
||||
asrBailian: 'Alibaba Bailian リアルタイム ASR',
|
||||
asrBailianQwen3: 'Bailian Qwen3 リアルタイム ASR',
|
||||
asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash(録音ファイル)',
|
||||
asrSiliconflow: 'SiliconFlow SenseVoice',
|
||||
asrZhipu: 'Zhipu GLM-ASR',
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
@@ -822,6 +823,8 @@ export const ja: typeof zhCN = {
|
||||
thinkingModeHint: 'オフではチャネル単位の公式パラメーターで思考を無効化または最小化します。オンではチャネル既定で思考を有効化します。prompt 注入やモデル別適配は行いません。',
|
||||
bailianVocabularyIdLabel: 'ホットワード Vocabulary ID(任意)',
|
||||
bailianVocabularyIdNote: 'DashScope でホットワード辞書を作成済みの場合は vocab-... ID を入力します。空欄なら送信しません。',
|
||||
bailianModelRealtimeHint: 'リアルタイムモデル · 話しながら文字起こし。',
|
||||
bailianModelRecordedFileHint: '録音ファイルモデル · 話し終えてから一括で文字起こし(1 本 ≤ 5 分)。',
|
||||
appIdLabel: 'App ID(アプリケーション ID)',
|
||||
accessKeyLabel: 'Access Key',
|
||||
resourceIdLabel: 'Resource ID',
|
||||
|
||||
@@ -782,6 +782,7 @@ export const ko: typeof zhCN = {
|
||||
asrVolcengine: 'Volcengine bigasr',
|
||||
asrBailian: 'Alibaba Bailian 실시간 ASR',
|
||||
asrBailianQwen3: 'Bailian Qwen3 실시간 ASR',
|
||||
asrBailianFunAsrFlash: 'Bailian Fun-ASR-Flash (녹음 파일)',
|
||||
asrSiliconflow: 'SiliconFlow SenseVoice',
|
||||
asrZhipu: 'Zhipu GLM-ASR',
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
@@ -822,6 +823,8 @@ export const ko: typeof zhCN = {
|
||||
thinkingModeHint: '꺼짐은 채널 단위 공식 파라미터로 사고를 끄거나 최소화합니다. 켜짐은 채널 기본값으로 사고를 켭니다. prompt 주입이나 모델별 어댑터는 사용하지 않습니다.',
|
||||
bailianVocabularyIdLabel: '핫워드 Vocabulary ID(선택)',
|
||||
bailianVocabularyIdNote: 'DashScope에서 핫워드 사전을 만들었다면 vocab-... ID를 입력하세요. 비워 두면 핫워드를 전송하지 않습니다.',
|
||||
bailianModelRealtimeHint: '실시간 모델 · 말하는 동안 바로 전사.',
|
||||
bailianModelRecordedFileHint: '녹음 파일 모델 · 말을 마친 뒤 전체 전사(한 클립 ≤ 5분).',
|
||||
appIdLabel: 'App ID(애플리케이션 ID)',
|
||||
accessKeyLabel: 'Access Key',
|
||||
resourceIdLabel: 'Resource ID',
|
||||
|
||||
@@ -778,6 +778,7 @@ export const zhCN = {
|
||||
asrVolcengine: '火山引擎 bigasr',
|
||||
asrBailian: '阿里云百炼实时 ASR',
|
||||
asrBailianQwen3: '阿里云百炼 Qwen3 实时 ASR',
|
||||
asrBailianFunAsrFlash: '阿里云百炼 Fun-ASR-Flash(录音文件)',
|
||||
asrSiliconflow: '硅基流动 SenseVoice',
|
||||
asrZhipu: '智谱 GLM-ASR',
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
@@ -818,6 +819,8 @@ export const zhCN = {
|
||||
thinkingModeHint: '关闭时按渠道级官方参数关闭或压低思考;开启时按渠道默认启用思考。不注入 prompt,也不做单模型适配。',
|
||||
bailianVocabularyIdLabel: '热词 Vocabulary ID(可选)',
|
||||
bailianVocabularyIdNote: '如已在百炼创建热词表,可填写 vocab-...;留空则不下发热词。',
|
||||
bailianModelRealtimeHint: '实时模型 · 边说边出字。',
|
||||
bailianModelRecordedFileHint: '录音文件模型 · 说完后整段转写(单条 ≤ 5 分钟)。',
|
||||
appIdLabel: 'App ID(应用 ID)',
|
||||
accessKeyLabel: 'Access Key',
|
||||
resourceIdLabel: '资源 ID',
|
||||
|
||||
@@ -780,6 +780,7 @@ export const zhTW: typeof zhCN = {
|
||||
asrVolcengine: '火山引擎 bigasr',
|
||||
asrBailian: '阿里雲百煉即時 ASR',
|
||||
asrBailianQwen3: '阿里雲百煉 Qwen3 即時 ASR',
|
||||
asrBailianFunAsrFlash: '阿里雲百煉 Fun-ASR-Flash(錄音檔)',
|
||||
asrSiliconflow: '硅基流動 SenseVoice',
|
||||
asrZhipu: '智譜 GLM-ASR',
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
@@ -820,6 +821,8 @@ export const zhTW: typeof zhCN = {
|
||||
thinkingModeHint: '關閉時按渠道級官方參數關閉或降低思考;開啟時按渠道預設啟用思考。不注入 prompt,也不做單模型適配。',
|
||||
bailianVocabularyIdLabel: '熱詞 Vocabulary ID(可選)',
|
||||
bailianVocabularyIdNote: '如已在百煉建立熱詞表,可填寫 vocab-...;留空則不下發熱詞。',
|
||||
bailianModelRealtimeHint: '即時模型 · 邊說邊出字。',
|
||||
bailianModelRecordedFileHint: '錄音檔模型 · 說完後整段轉寫(單條 ≤ 5 分鐘)。',
|
||||
appIdLabel: 'App ID(應用 ID)',
|
||||
accessKeyLabel: 'Access Key',
|
||||
resourceIdLabel: '資源 ID',
|
||||
|
||||
@@ -10,6 +10,7 @@ import { useMobileLayout } from '../lib/useMobileLayout';
|
||||
import type { ActivityDay, CredentialsStatus, DictationSession, PolishMode } from '../lib/types';
|
||||
import { useHotkeySettings } from '../state/HotkeySettingsContext';
|
||||
import { Btn, Card, PageHeader, Pill } from './_atoms';
|
||||
import { ASR_PRESETS } from './settings/shared';
|
||||
|
||||
function useModeLabels(): Record<PolishMode, string> {
|
||||
const { t } = useTranslation();
|
||||
@@ -25,19 +26,11 @@ interface OverviewProps {
|
||||
onOpenHistory?: () => void;
|
||||
}
|
||||
|
||||
const ASR_NAME_KEY_BY_ID: Record<string, string> = {
|
||||
volcengine: 'asrVolcengine',
|
||||
bailian: 'asrBailian',
|
||||
siliconflow: 'asrSiliconflow',
|
||||
zhipu: 'asrZhipu',
|
||||
groq: 'asrGroq',
|
||||
whisper: 'asrWhisper',
|
||||
openrouter: 'asrOpenrouter',
|
||||
'xiaomi-mimo-asr': 'asrXiaomiMimo',
|
||||
'foundry-local-whisper': 'asrFoundryLocalWhisper',
|
||||
'sherpa-onnx-local': 'asrSherpaOnnxLocal',
|
||||
'local-qwen3': 'asrLocalQwen3',
|
||||
};
|
||||
// id → i18n nameKey,从 ASR_PRESETS 单一来源派生,避免这里再手维护一份 id 列表
|
||||
// (之前漏了 bailian-qwen3-realtime / apple-speech,会退化成显示裸 id)。
|
||||
const ASR_NAME_KEY_BY_ID: Record<string, string> = Object.fromEntries(
|
||||
ASR_PRESETS.map(p => [p.id, p.nameKey]),
|
||||
);
|
||||
|
||||
const LLM_NAME_KEY_BY_ID: Record<string, string> = {
|
||||
ark: 'ark',
|
||||
|
||||
@@ -18,7 +18,7 @@ import { useMobileLayout } from '../../lib/useMobileLayout';
|
||||
import { useHotkeySettings } from '../../state/HotkeySettingsContext';
|
||||
import { SelectLite } from '../../components/ui/SelectLite';
|
||||
import { Card } from '../_atoms';
|
||||
import { SettingRow, SectionTitle, Toggle, inputStyle, type AsrPresetId } from './shared';
|
||||
import { SettingRow, SectionTitle, Toggle, inputStyle, ASR_PRESETS, type AsrPresetId } from './shared';
|
||||
|
||||
function LlmThinkingToggle({ enabled, onToggle }: { enabled: boolean; onToggle: (next: boolean) => void }) {
|
||||
const { t } = useTranslation();
|
||||
@@ -144,39 +144,8 @@ type LlmPresetId = typeof LLM_PRESETS[number]['id'];
|
||||
|
||||
const ASR_DEFAULT_RESOURCE_ID = 'volc.seedasr.sauc.duration';
|
||||
|
||||
// `volcengine` / `bailian` 走自建流式客户端;其余走 OpenAI 兼容
|
||||
// `/audio/transcriptions`(`coordinator.rs::is_whisper_compatible_provider`)。
|
||||
// 新增兼容厂商:
|
||||
// 1. 在这里加一项 `{ id, nameKey, baseUrl, model }`;
|
||||
// 2. 若走 Whisper 协议,`coordinator.rs::is_whisper_compatible_provider` 加同名 id;
|
||||
// 若是专有协议,新增独立 ASR client 与 provider kind;
|
||||
// 3. 在 i18n 的 `settings.providers.presets.<nameKey>` 加文案。
|
||||
// `AsrPresetId` 定义在 settings/shared.tsx,LocalModelSection / ProvidersSection 共用同一份。
|
||||
const ASR_PRESETS: ReadonlyArray<{ id: AsrPresetId; nameKey: string; baseUrl: string; model: string }> = [
|
||||
{ id: 'volcengine', nameKey: 'asrVolcengine', baseUrl: '', model: '' },
|
||||
{ id: 'bailian', nameKey: 'asrBailian', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/inference/', model: 'fun-asr-realtime' },
|
||||
// Qwen3-ASR-Flash 实时:OpenAI Realtime 风格 WS(/api-ws/v1/realtime),
|
||||
// 与上面经典 inference 协议不同,由 asr/qwen_realtime.rs 专用 client 处理。
|
||||
// 业务空间专属域名(wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime)同样可用。
|
||||
{ id: 'bailian-qwen3-realtime', nameKey: 'asrBailianQwen3', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime', model: 'qwen3-asr-flash-realtime' },
|
||||
{ id: 'siliconflow', nameKey: 'asrSiliconflow', baseUrl: 'https://api.siliconflow.cn/v1', model: 'FunAudioLLM/SenseVoiceSmall' },
|
||||
{ id: 'zhipu', nameKey: 'asrZhipu', baseUrl: 'https://open.bigmodel.cn/api/paas/v4', model: 'glm-asr-2512' },
|
||||
{ id: 'groq', nameKey: 'asrGroq', baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3-turbo' },
|
||||
{ id: 'whisper', nameKey: 'asrWhisper', baseUrl: 'https://api.openai.com/v1', model: 'whisper-1' },
|
||||
// OpenRouter 的 /audio/transcriptions 走 application/json + base64(issue #582),
|
||||
// 后端 coordinator.rs::whisper_request_format 对该 id 切换到 OpenRouterJson 编码。
|
||||
{ id: 'openrouter', nameKey: 'asrOpenrouter', baseUrl: 'https://openrouter.ai/api/v1', model: 'openai/whisper-large-v3-turbo' },
|
||||
// 小米 MiMo ASR 按官方文档走 /chat/completions + input_audio,不是
|
||||
// Whisper /audio/transcriptions;后端由 asr/mimo.rs 专用 client 处理。
|
||||
{ id: 'xiaomi-mimo-asr', nameKey: 'asrXiaomiMimo', baseUrl: 'https://api.xiaomimimo.com/v1', model: 'mimo-v2.5-asr' },
|
||||
{ id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' },
|
||||
// 本地引擎(Foundry / sherpa-onnx / Qwen3):无 baseUrl/model 配置,
|
||||
// 模型在「高级 → 本地模型」里下载与切换。
|
||||
{ id: 'sherpa-onnx-local', nameKey: 'asrSherpaOnnxLocal', baseUrl: '', model: '' },
|
||||
{ id: 'local-qwen3', nameKey: 'asrLocalQwen3', baseUrl: '', model: '' },
|
||||
// Apple 系统语音识别(macOS):无 baseUrl/model、无下载、无凭据。
|
||||
{ id: 'apple-speech', nameKey: 'asrAppleSpeech', baseUrl: '', model: '' },
|
||||
];
|
||||
// ASR_PRESETS 已上移到 settings/shared.tsx 作为单一来源(AsrPresetId 由其派生,
|
||||
// Overview 的显示名映射也从那里取)。新增厂商的步骤见 shared.tsx 的注释。
|
||||
|
||||
type ProvidersSectionKind = 'all' | 'llm' | 'asr';
|
||||
|
||||
@@ -204,6 +173,12 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
|
||||
const [llmModelRevision, setLlmModelRevision] = useState(0);
|
||||
const [asrModelRevision, setAsrModelRevision] = useState(0);
|
||||
const os = detectOS();
|
||||
const unifiedBailian = committedAsrProvider === 'bailian' && os !== 'android';
|
||||
const [bailianModel, setBailianModel] = useState('');
|
||||
|
||||
useEffect(() => {
|
||||
if (committedAsrProvider !== 'bailian') setBailianModel('');
|
||||
}, [committedAsrProvider]);
|
||||
// 本地重引擎(qwen3 / sherpa / foundry)仍只在「高级 → 本地模型」里启用,
|
||||
// 防止新手在主下拉误开 CPU 推理。Apple 语音是系统自带、零凭据、轻量,
|
||||
// 在 macOS 上直接作为常规选项放进主下拉,方便随时选用 / 切走。
|
||||
@@ -211,7 +186,12 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
|
||||
p => p.id !== 'foundry-local-whisper'
|
||||
&& p.id !== 'local-qwen3'
|
||||
&& p.id !== 'sherpa-onnx-local'
|
||||
&& (p.id !== 'apple-speech' || os === 'mac'),
|
||||
&& (p.id !== 'apple-speech' || os === 'mac')
|
||||
// 百炼三协议收成一个「阿里云百炼」入口(id=bailian)+ 模型下拉。qwen3 / fun-asr-flash
|
||||
// 两个旧 id 作隐藏别名:新用户下拉里看不到,只有已经停在该 id 上的老用户仍显示,
|
||||
// 保证其配置不被打断(见 coordinator::resolve_effective_asr_provider 的向后兼容)。
|
||||
&& (p.id !== 'bailian-qwen3-realtime' || asrProvider === 'bailian-qwen3-realtime')
|
||||
&& (p.id !== 'bailian-fun-asr-flash' || asrProvider === 'bailian-fun-asr-flash'),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -494,12 +474,17 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
|
||||
) : (
|
||||
<>
|
||||
<CredentialField key={`${committedAsrProvider}:api_key`} label={t('settings.providers.apiKeyLabel')} account="asr.api_key" mono mask />
|
||||
{/* 统一百炼保留 endpoint 供用户选择区域或工作空间域名;后端按模型转换协议与路径。 */}
|
||||
<CredentialField key={`${committedAsrProvider}:endpoint`} label={t('settings.providers.baseUrlLabel')} account="asr.endpoint"
|
||||
placeholder={asrPreset?.baseUrl || 'https://api.openai.com/v1'}
|
||||
defaultValue={asrPreset?.baseUrl || undefined} />
|
||||
<CredentialField key={`${committedAsrProvider}:model:${asrModelRevision}`} label={t('settings.providers.modelLabel')} account="asr.model"
|
||||
placeholder={asrPreset?.model || 'whisper-1'} />
|
||||
{committedAsrProvider === 'bailian' && (
|
||||
placeholder={unifiedBailian ? 'fun-asr-realtime' : (asrPreset?.model || 'whisper-1')}
|
||||
onValueChange={unifiedBailian ? setBailianModel : undefined} />
|
||||
{unifiedBailian && (
|
||||
<BailianProtocolHint key={`${committedAsrProvider}:proto:${asrModelRevision}`} currentModel={bailianModel} />
|
||||
)}
|
||||
{unifiedBailian && bailianModelSupportsVocabulary(bailianModel) && (
|
||||
<>
|
||||
<CredentialField
|
||||
key={`${committedAsrProvider}:vocabulary_id`}
|
||||
@@ -513,6 +498,7 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{/* 统一百炼「拉取模型」只写 model,不覆盖用户选择的区域或工作空间 endpoint。 */}
|
||||
<ProviderTools kind="asr" modelAccount="asr.model" onModelSelected={() => setAsrModelRevision(v => v + 1)} />
|
||||
</>
|
||||
)}
|
||||
@@ -522,9 +508,55 @@ export function ProvidersSection({ kind = 'all' }: ProvidersSectionProps = {}) {
|
||||
);
|
||||
}
|
||||
|
||||
// 统一「阿里云百炼」下,按模型名判断走哪种协议(与后端
|
||||
// coordinator::resolve_effective_asr_provider 保持一致):qwen3-asr-flash-realtime* 与
|
||||
// fun-asr-realtime* 与 fun-asr-flash-8k-realtime* 都是实时模型;当前支持的
|
||||
// fun-asr-flash-2026-06-15 是「录音文件·说完转写」。
|
||||
function bailianModelIsRecordedFile(model: string): boolean {
|
||||
const m = model.trim();
|
||||
return m === 'fun-asr-flash-2026-06-15';
|
||||
}
|
||||
|
||||
function bailianModelSupportsVocabulary(model: string): boolean {
|
||||
const m = model.trim();
|
||||
return !m
|
||||
|| m.startsWith('fun-asr-realtime')
|
||||
|| m.startsWith('paraformer-realtime')
|
||||
|| m.startsWith('sensevoice-realtime');
|
||||
}
|
||||
|
||||
// 模型框下的一行协议提示,解决「三种模型看不出区别」——告诉用户当前模型是实时还是
|
||||
// 录音文件、行为差异如何。随 asrModelRevision(拉取/选择模型时)与挂载时重读 asr.model。
|
||||
function BailianProtocolHint({ currentModel }: { currentModel: string }) {
|
||||
const { t } = useTranslation();
|
||||
const [model, setModel] = useState('');
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
readCredential('asr.model')
|
||||
.then(v => { if (!cancelled) setModel(v || 'fun-asr-realtime'); })
|
||||
.catch(() => { /* 读失败按默认实时提示 */ });
|
||||
return () => { cancelled = true; };
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setModel(currentModel || 'fun-asr-realtime');
|
||||
}, [currentModel]);
|
||||
|
||||
const hint = bailianModelIsRecordedFile(model)
|
||||
? t('settings.providers.bailianModelRecordedFileHint')
|
||||
: t('settings.providers.bailianModelRealtimeHint');
|
||||
|
||||
return (
|
||||
<div style={{ marginTop: 2, fontSize: 11.5, color: 'var(--ol-ink-4)', lineHeight: 1.6 }}>
|
||||
{hint}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type ProviderToolStatus = 'idle' | 'loading' | 'success' | 'empty' | 'error';
|
||||
|
||||
function ProviderTools({ kind, modelAccount, onModelSelected }: { kind: 'llm' | 'asr'; modelAccount: string; onModelSelected: () => void }) {
|
||||
function ProviderTools({ kind, modelAccount, onModelSelected, showFetchModels = true }: { kind: 'llm' | 'asr'; modelAccount: string; onModelSelected: () => void; showFetchModels?: boolean }) {
|
||||
const { t } = useTranslation();
|
||||
const mobile = useMobileLayout();
|
||||
const [models, setModels] = useState<string[]>([]);
|
||||
@@ -595,8 +627,10 @@ function ProviderTools({ kind, modelAccount, onModelSelected }: { kind: 'llm' |
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 8, width: '100%', maxWidth: mobile ? '100%' : 420 }}>
|
||||
<div style={{ display: 'flex', gap: 6, alignItems: 'center', flexWrap: 'wrap', width: '100%' }}>
|
||||
<button onClick={validate} style={miniBtnStyle} disabled={status === 'loading'}>{t('settings.providers.validate')}</button>
|
||||
<button onClick={loadModels} style={miniBtnStyle} disabled={status === 'loading'}>{t('settings.providers.fetchModels')}</button>
|
||||
{models.length > 0 && (
|
||||
{showFetchModels && (
|
||||
<button onClick={loadModels} style={miniBtnStyle} disabled={status === 'loading'}>{t('settings.providers.fetchModels')}</button>
|
||||
)}
|
||||
{showFetchModels && models.length > 0 && (
|
||||
<SelectLite
|
||||
value={selectedModel}
|
||||
onChange={applyModel}
|
||||
@@ -649,9 +683,10 @@ interface CredentialFieldProps {
|
||||
mask?: boolean;
|
||||
defaultValue?: string;
|
||||
trailing?: ReactNode;
|
||||
onValueChange?: (value: string) => void;
|
||||
}
|
||||
|
||||
function CredentialField({ label, account, placeholder, mono, mask, defaultValue, trailing }: CredentialFieldProps) {
|
||||
function CredentialField({ label, account, placeholder, mono, mask, defaultValue, trailing, onValueChange }: CredentialFieldProps) {
|
||||
const { t } = useTranslation();
|
||||
const mobile = useMobileLayout();
|
||||
const [value, setValue] = useState('');
|
||||
@@ -669,6 +704,7 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
|
||||
setDirty(false);
|
||||
setStatus('idle');
|
||||
setValue('');
|
||||
onValueChange?.('');
|
||||
if (debounceRef.current) {
|
||||
clearTimeout(debounceRef.current);
|
||||
debounceRef.current = null;
|
||||
@@ -677,18 +713,20 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
|
||||
.then(v => {
|
||||
if (cancelled) return;
|
||||
setValue(v ?? '');
|
||||
onValueChange?.(v ?? '');
|
||||
setLoaded(true);
|
||||
})
|
||||
.catch(error => {
|
||||
if (cancelled) return;
|
||||
console.error('[settings] failed to read credential', account, error);
|
||||
onValueChange?.('');
|
||||
setLoaded(true);
|
||||
setStatus('readError');
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [account]);
|
||||
}, [account, onValueChange]);
|
||||
|
||||
useEffect(() => {
|
||||
mountedRef.current = true;
|
||||
@@ -739,6 +777,7 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
|
||||
const handleChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const v = e.target.value;
|
||||
setValue(v);
|
||||
onValueChange?.(v);
|
||||
if (!loaded) return;
|
||||
setDirty(true);
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
@@ -757,6 +796,7 @@ function CredentialField({ label, account, placeholder, mono, mask, defaultValue
|
||||
const fillDefault = async () => {
|
||||
if (!loaded || !defaultValue) return;
|
||||
setValue(defaultValue);
|
||||
onValueChange?.(defaultValue);
|
||||
setDirty(true);
|
||||
await save(defaultValue, true);
|
||||
};
|
||||
|
||||
@@ -199,19 +199,45 @@ export const inputStyle: CSSProperties = {
|
||||
"background 0.16s var(--ol-motion-quick), border-color 0.16s var(--ol-motion-quick)",
|
||||
}
|
||||
|
||||
// ASR provider id 集合,跟 ProvidersSection.tsx::ASR_PRESETS 一一对应。
|
||||
// 拆成独立类型让 LocalModelSection / ProvidersSection 都能用同一份不互相依赖。
|
||||
export type AsrPresetId =
|
||||
| "volcengine"
|
||||
| "bailian"
|
||||
| "bailian-qwen3-realtime"
|
||||
| "siliconflow"
|
||||
| "zhipu"
|
||||
| "groq"
|
||||
| "whisper"
|
||||
| "openrouter"
|
||||
| "xiaomi-mimo-asr"
|
||||
| "foundry-local-whisper"
|
||||
| "sherpa-onnx-local"
|
||||
| "local-qwen3"
|
||||
| "apple-speech"
|
||||
// ASR provider preset 清单 —— 单一来源,放这里让 ProvidersSection /
|
||||
// LocalModelSection / Overview 都从同一处取,不再各维护一份 id 列表导致漂移。
|
||||
// `AsrPresetId` 由此派生(与 ProvidersSection 的 LLM_PRESETS→LlmPresetId 同构)。
|
||||
//
|
||||
// 新增兼容厂商:
|
||||
// 1. 这里加一项 `{ id, nameKey, baseUrl, model }`;
|
||||
// 2. 后端 `coordinator.rs::active_asr_provider_kind` 加 id→kind 映射 —— 之后
|
||||
// `preflight_credential` / `configured_fields` 等穷尽 match 会被编译器逐个
|
||||
// 报错逼你补齐;走 Whisper 协议再加进 `is_whisper_compatible_provider`,
|
||||
// 专有协议另配独立 ASR client 与 provider kind;
|
||||
// 3. i18n 的 `settings.providers.presets.<nameKey>` 补各语言文案。
|
||||
export const ASR_PRESETS = [
|
||||
{ id: 'volcengine', nameKey: 'asrVolcengine', baseUrl: '', model: '' },
|
||||
{ id: 'bailian', nameKey: 'asrBailian', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/inference/', model: 'fun-asr-realtime' },
|
||||
// Qwen3-ASR-Flash 实时:OpenAI Realtime 风格 WS(/api-ws/v1/realtime),
|
||||
// 与上面经典 inference 协议不同,由 asr/qwen_realtime.rs 专用 client 处理。
|
||||
// 业务空间专属域名(wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime)同样可用。
|
||||
{ id: 'bailian-qwen3-realtime', nameKey: 'asrBailianQwen3', baseUrl: 'wss://dashscope.aliyuncs.com/api-ws/v1/realtime', model: 'qwen3-asr-flash-realtime' },
|
||||
// Fun-ASR-Flash 录音文件识别:非实时,走 DashScope 私有的
|
||||
// multimodal-generation HTTP 接口(既非实时 WS,也非 OpenAI /audio/transcriptions),
|
||||
// 由 asr/dashscope_multimodal.rs 专用批量 client 处理。API key 与百炼同一把。
|
||||
{ id: 'bailian-fun-asr-flash', nameKey: 'asrBailianFunAsrFlash', baseUrl: 'https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation', model: 'fun-asr-flash-2026-06-15' },
|
||||
{ id: 'siliconflow', nameKey: 'asrSiliconflow', baseUrl: 'https://api.siliconflow.cn/v1', model: 'FunAudioLLM/SenseVoiceSmall' },
|
||||
{ id: 'zhipu', nameKey: 'asrZhipu', baseUrl: 'https://open.bigmodel.cn/api/paas/v4', model: 'glm-asr-2512' },
|
||||
{ id: 'groq', nameKey: 'asrGroq', baseUrl: 'https://api.groq.com/openai/v1', model: 'whisper-large-v3-turbo' },
|
||||
{ id: 'whisper', nameKey: 'asrWhisper', baseUrl: 'https://api.openai.com/v1', model: 'whisper-1' },
|
||||
// OpenRouter 的 /audio/transcriptions 走 application/json + base64(issue #582),
|
||||
// 后端 coordinator.rs::whisper_request_format 对该 id 切换到 OpenRouterJson 编码。
|
||||
{ id: 'openrouter', nameKey: 'asrOpenrouter', baseUrl: 'https://openrouter.ai/api/v1', model: 'openai/whisper-large-v3-turbo' },
|
||||
// 小米 MiMo ASR 按官方文档走 /chat/completions + input_audio,不是
|
||||
// Whisper /audio/transcriptions;后端由 asr/mimo.rs 专用 client 处理。
|
||||
{ id: 'xiaomi-mimo-asr', nameKey: 'asrXiaomiMimo', baseUrl: 'https://api.xiaomimimo.com/v1', model: 'mimo-v2.5-asr' },
|
||||
{ id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' },
|
||||
// 本地引擎(Foundry / sherpa-onnx / Qwen3):无 baseUrl/model 配置,
|
||||
// 模型在「高级 → 本地模型」里下载与切换。
|
||||
{ id: 'sherpa-onnx-local', nameKey: 'asrSherpaOnnxLocal', baseUrl: '', model: '' },
|
||||
{ id: 'local-qwen3', nameKey: 'asrLocalQwen3', baseUrl: '', model: '' },
|
||||
// Apple 系统语音识别(macOS):无 baseUrl/model、无下载、无凭据。
|
||||
{ id: 'apple-speech', nameKey: 'asrAppleSpeech', baseUrl: '', model: '' },
|
||||
] as const;
|
||||
|
||||
export type AsrPresetId = typeof ASR_PRESETS[number]['id'];
|
||||
Reference in new issue
Block a user