feat(asr): add Xiaomi MiMo provider
This commit is contained in:
1 parent
61b99ad593
commit
a79b7429fe
14 files changed
+623
-2
No files matched your search
@@ -0,0 +1,466 @@
|
||||
//! Xiaomi MiMo ASR client.
|
||||
//!
|
||||
//! MiMo ASR uses the official OpenAI-compatible `/chat/completions` endpoint
|
||||
//! with `messages[].content[].input_audio`, not Whisper's
|
||||
//! `/audio/transcriptions` protocol.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use base64::Engine;
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::asr::wav::encode_wav_16k_mono;
|
||||
use crate::asr::RawTranscript;
|
||||
|
||||
const PCM_SAMPLE_RATE_HZ: u64 = 16_000;
|
||||
const PCM_BYTES_PER_SAMPLE: usize = 2;
|
||||
// 官方限制:Base64 后的音频数据不能超过 10MB。180s 的 16k/16-bit/mono WAV
|
||||
// Base64 后约 7.7MB,给 JSON/data-url 前缀和厂商侧 MB 口径差异留余量。
|
||||
const MIMO_MAX_CHUNK_DURATION_MS: u64 = 180_000;
|
||||
pub const PROVIDER_ID: &str = "xiaomi-mimo-asr";
|
||||
pub const DEFAULT_ENDPOINT: &str = "https://api.xiaomimimo.com/v1";
|
||||
pub const DEFAULT_MODEL: &str = "mimo-v2.5-asr";
|
||||
|
||||
pub struct MimoBatchASR {
|
||||
api_key: String,
|
||||
base_url: String,
|
||||
model: String,
|
||||
buffer: Mutex<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl MimoBatchASR {
|
||||
pub fn new(api_key: String, base_url: String, model: String) -> Self {
|
||||
Self {
|
||||
api_key,
|
||||
base_url,
|
||||
model,
|
||||
buffer: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn transcribe(&self) -> Result<RawTranscript> {
|
||||
let pcm = self.buffer.lock().clone();
|
||||
if pcm.is_empty() {
|
||||
return Ok(RawTranscript {
|
||||
text: String::new(),
|
||||
duration_ms: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let result = self.transcribe_inner(&pcm).await;
|
||||
if result.is_ok() {
|
||||
self.buffer.lock().clear();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn transcribe_inner(&self, pcm: &[u8]) -> Result<RawTranscript> {
|
||||
if self.api_key.trim().is_empty() {
|
||||
anyhow::bail!("MiMo API key missing");
|
||||
}
|
||||
|
||||
let duration_ms = pcm_duration_ms(pcm);
|
||||
let chunks = split_pcm_by_duration(pcm, MIMO_MAX_CHUNK_DURATION_MS);
|
||||
let mut texts = Vec::with_capacity(chunks.len());
|
||||
for chunk in chunks {
|
||||
texts.push(self.transcribe_chunk(chunk).await?);
|
||||
}
|
||||
|
||||
Ok(RawTranscript {
|
||||
text: join_transcript_chunks(&texts),
|
||||
duration_ms,
|
||||
})
|
||||
}
|
||||
|
||||
async fn transcribe_chunk(&self, pcm: &[u8]) -> Result<String> {
|
||||
let samples: Vec<i16> = pcm
|
||||
.chunks_exact(2)
|
||||
.map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
|
||||
.collect();
|
||||
let wav = encode_wav_16k_mono(&samples);
|
||||
let body = mimo_chat_body(&self.model, &wav);
|
||||
let url = mimo_chat_completions_url(&self.base_url)?;
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.api_key.trim()))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.context("MiMo ASR HTTP request failed")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("MiMo ASR API error {}: {}", status, body);
|
||||
}
|
||||
|
||||
let json: Value = resp.json().await.context("parse MiMo ASR response")?;
|
||||
Ok(extract_mimo_text(&json).trim().to_string())
|
||||
}
|
||||
|
||||
pub fn cancel(&self) {
|
||||
self.buffer.lock().clear();
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::recorder::AudioConsumer for MimoBatchASR {
|
||||
fn consume_pcm_chunk(&self, pcm: &[u8]) {
|
||||
self.buffer.lock().extend_from_slice(pcm);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mimo_chat_completions_url(base_url: &str) -> Result<String> {
|
||||
let parsed = reqwest::Url::parse(base_url.trim()).context("parse MiMo base URL")?;
|
||||
let mut url = parsed.clone();
|
||||
let path = parsed.path().trim_end_matches('/');
|
||||
let next_path = if path.ends_with("/chat/completions") {
|
||||
path.to_string()
|
||||
} else if path.ends_with("/chat") {
|
||||
format!("{path}/completions")
|
||||
} else {
|
||||
format!("{path}/chat/completions")
|
||||
};
|
||||
url.set_path(&next_path);
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
pub fn mimo_chat_body(model: &str, wav: &[u8]) -> Value {
|
||||
let audio_data = format!(
|
||||
"data:audio/wav;base64,{}",
|
||||
base64::engine::general_purpose::STANDARD.encode(wav)
|
||||
);
|
||||
serde_json::json!({
|
||||
"model": model,
|
||||
"stream": false,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_audio",
|
||||
"input_audio": {
|
||||
"data": audio_data,
|
||||
"format": "wav",
|
||||
},
|
||||
}],
|
||||
}],
|
||||
})
|
||||
}
|
||||
|
||||
pub fn extract_mimo_text(json: &Value) -> String {
|
||||
let Some(content) = json
|
||||
.get("choices")
|
||||
.and_then(|v| v.as_array())
|
||||
.and_then(|choices| choices.first())
|
||||
.and_then(|choice| choice.get("message"))
|
||||
.and_then(|message| message.get("content"))
|
||||
else {
|
||||
return String::new();
|
||||
};
|
||||
|
||||
if let Some(text) = content.as_str() {
|
||||
return text.to_string();
|
||||
}
|
||||
|
||||
content
|
||||
.as_array()
|
||||
.map(|items| {
|
||||
items
|
||||
.iter()
|
||||
.filter_map(|item| {
|
||||
item.get("text")
|
||||
.and_then(|text| text.as_str())
|
||||
.or_else(|| item.get("content").and_then(|text| text.as_str()))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("")
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn pcm_duration_ms(pcm: &[u8]) -> u64 {
|
||||
(pcm.len() as u64 / PCM_BYTES_PER_SAMPLE as u64) * 1000 / PCM_SAMPLE_RATE_HZ
|
||||
}
|
||||
|
||||
fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
|
||||
if max_chunk_duration_ms == 0 {
|
||||
return vec![pcm];
|
||||
}
|
||||
|
||||
let samples_per_chunk = PCM_SAMPLE_RATE_HZ * max_chunk_duration_ms / 1000;
|
||||
let bytes_per_chunk = samples_per_chunk as usize * PCM_BYTES_PER_SAMPLE;
|
||||
if bytes_per_chunk == 0 || pcm.len() <= bytes_per_chunk {
|
||||
return vec![pcm];
|
||||
}
|
||||
|
||||
pcm.chunks(bytes_per_chunk).collect()
|
||||
}
|
||||
|
||||
fn join_transcript_chunks(chunks: &[String]) -> String {
|
||||
let mut joined = String::new();
|
||||
for chunk in chunks.iter().map(|chunk| chunk.trim()) {
|
||||
if chunk.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if needs_chunk_separator(&joined, chunk) {
|
||||
joined.push(' ');
|
||||
}
|
||||
joined.push_str(chunk);
|
||||
}
|
||||
joined
|
||||
}
|
||||
|
||||
fn needs_chunk_separator(current: &str, next: &str) -> bool {
|
||||
let Some(prev) = current.chars().last() else {
|
||||
return false;
|
||||
};
|
||||
let Some(first) = next.chars().next() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if is_closing_punctuation(first) || is_opening_punctuation(prev) {
|
||||
return false;
|
||||
}
|
||||
if is_cjk(prev) && (is_cjk(first) || is_opening_punctuation(first)) {
|
||||
return false;
|
||||
}
|
||||
if is_cjk(first) && is_closing_punctuation(prev) {
|
||||
return false;
|
||||
}
|
||||
if is_cjk_punctuation(prev) && is_cjk(first) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn is_cjk(c: char) -> bool {
|
||||
matches!(
|
||||
c as u32,
|
||||
0x4E00..=0x9FFF
|
||||
| 0x3400..=0x4DBF
|
||||
| 0x3040..=0x30FF
|
||||
| 0xAC00..=0xD7AF
|
||||
)
|
||||
}
|
||||
|
||||
fn is_cjk_punctuation(c: char) -> bool {
|
||||
matches!(
|
||||
c,
|
||||
'。' | ',' | '、' | '!' | '?' | ';' | ':' | '」' | '』' | ')' | '》'
|
||||
)
|
||||
}
|
||||
|
||||
fn is_closing_punctuation(c: char) -> bool {
|
||||
c.is_ascii_punctuation() && !matches!(c, '(' | '[' | '{' | '"' | '\'') || is_cjk_punctuation(c)
|
||||
}
|
||||
|
||||
fn is_opening_punctuation(c: char) -> bool {
|
||||
matches!(c, '(' | '[' | '{' | '"' | '\'' | '「' | '『' | '(' | '《')
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::recorder::AudioConsumer;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[test]
|
||||
fn mimo_url_targets_chat_completions() {
|
||||
assert_eq!(
|
||||
mimo_chat_completions_url("https://api.xiaomimimo.com/v1").unwrap(),
|
||||
"https://api.xiaomimimo.com/v1/chat/completions"
|
||||
);
|
||||
assert_eq!(
|
||||
mimo_chat_completions_url("https://api.xiaomimimo.com/v1/chat/completions").unwrap(),
|
||||
"https://api.xiaomimimo.com/v1/chat/completions"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mimo_body_uses_official_input_audio_shape() {
|
||||
let body = mimo_chat_body(DEFAULT_MODEL, b"wav");
|
||||
assert_eq!(body["model"], DEFAULT_MODEL);
|
||||
assert_eq!(body["stream"], false);
|
||||
let audio = &body["messages"][0]["content"][0];
|
||||
assert_eq!(audio["type"], "input_audio");
|
||||
assert_eq!(audio["input_audio"]["format"], "wav");
|
||||
assert!(audio["input_audio"]["data"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("data:audio/wav;base64,"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mimo_response_extracts_message_content() {
|
||||
let json = serde_json::json!({
|
||||
"choices": [{ "message": { "content": "你好 MiMo" } }]
|
||||
});
|
||||
assert_eq!(extract_mimo_text(&json), "你好 MiMo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mimo_response_accepts_content_array_text() {
|
||||
let json = serde_json::json!({
|
||||
"choices": [{
|
||||
"message": {
|
||||
"content": [
|
||||
{ "type": "text", "text": "你好" },
|
||||
{ "type": "text", "text": "MiMo" }
|
||||
]
|
||||
}
|
||||
}]
|
||||
});
|
||||
assert_eq!(extract_mimo_text(&json), "你好MiMo");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mimo_posts_chat_completion_audio_request() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let mut stream = loop {
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => break stream,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"timed out waiting for MiMo ASR test request"
|
||||
);
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Err(err) => panic!("accept MiMo ASR test request failed: {err}"),
|
||||
}
|
||||
};
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
let request = read_http_request(&mut stream);
|
||||
let request_text = String::from_utf8_lossy(&request);
|
||||
let lower = request_text.to_ascii_lowercase();
|
||||
assert!(request_text.starts_with("POST /chat/completions HTTP/1.1"));
|
||||
assert!(lower.contains("authorization: bearer key"));
|
||||
assert!(lower.contains("content-type: application/json"));
|
||||
assert!(request_text.contains(r#""model":"mimo-v2.5-asr""#));
|
||||
assert!(request_text.contains(r#""type":"input_audio""#));
|
||||
assert!(request_text.contains("data:audio/wav;base64,"));
|
||||
write_json_response(
|
||||
&mut stream,
|
||||
r#"{"choices":[{"message":{"content":"mimo ok"}}]}"#,
|
||||
);
|
||||
});
|
||||
|
||||
let asr = MimoBatchASR::new(
|
||||
"key".to_string(),
|
||||
format!("http://{}", addr),
|
||||
DEFAULT_MODEL.to_string(),
|
||||
);
|
||||
asr.consume_pcm_chunk(&vec![0u8; 32_000]);
|
||||
let transcript = asr.transcribe().await.unwrap();
|
||||
|
||||
assert_eq!(transcript.text, "mimo ok");
|
||||
assert_eq!(transcript.duration_ms, 1_000);
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mimo_splits_audio_before_base64_limit() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
for (index, text) in ["hello", "world"].into_iter().enumerate() {
|
||||
let mut stream = loop {
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => break stream,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"timed out waiting for MiMo ASR chunk request"
|
||||
);
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Err(err) => panic!("accept MiMo ASR chunk request failed: {err}"),
|
||||
}
|
||||
};
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(5)))
|
||||
.unwrap();
|
||||
let request = read_http_request(&mut stream);
|
||||
let request_text = String::from_utf8_lossy(&request);
|
||||
assert!(request_text.starts_with("POST /chat/completions HTTP/1.1"));
|
||||
assert!(request_text.contains(r#""model":"mimo-v2.5-asr""#));
|
||||
assert!(request_text.contains("data:audio/wav;base64,"));
|
||||
assert!(
|
||||
request_text.len() < 10 * 1024 * 1024,
|
||||
"chunk {index} exceeded MiMo 10MB base64 request budget"
|
||||
);
|
||||
write_json_response(
|
||||
&mut stream,
|
||||
&format!(r#"{{"choices":[{{"message":{{"content":"{text}"}}}}]}}"#),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let asr = MimoBatchASR::new(
|
||||
"key".to_string(),
|
||||
format!("http://{}", addr),
|
||||
DEFAULT_MODEL.to_string(),
|
||||
);
|
||||
let pcm = vec![0u8; 32_000 * 181];
|
||||
asr.consume_pcm_chunk(&pcm);
|
||||
let transcript = asr.transcribe().await.unwrap();
|
||||
|
||||
assert_eq!(transcript.text, "hello world");
|
||||
assert_eq!(transcript.duration_ms, 181_000);
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
fn read_http_request(stream: &mut std::net::TcpStream) -> Vec<u8> {
|
||||
let mut request = Vec::new();
|
||||
let mut buf = [0u8; 4096];
|
||||
let mut expected_len = None;
|
||||
loop {
|
||||
let read = stream.read(&mut buf).unwrap();
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
request.extend_from_slice(&buf[..read]);
|
||||
if expected_len.is_none() {
|
||||
expected_len = parse_expected_request_len(&request);
|
||||
}
|
||||
if expected_len.is_some_and(|len| request.len() >= len) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
fn parse_expected_request_len(request: &[u8]) -> Option<usize> {
|
||||
let header_end = request.windows(4).position(|w| w == b"\r\n\r\n")? + 4;
|
||||
let headers = String::from_utf8_lossy(&request[..header_end]);
|
||||
let content_len = headers.lines().find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
if name.eq_ignore_ascii_case("content-length") {
|
||||
value.trim().parse::<usize>().ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})?;
|
||||
Some(header_end + content_len)
|
||||
}
|
||||
|
||||
fn write_json_response(stream: &mut std::net::TcpStream, body: &str) {
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
stream.write_all(response.as_bytes()).unwrap();
|
||||
stream.flush().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,13 @@
|
||||
pub mod bailian;
|
||||
mod frame;
|
||||
pub mod local;
|
||||
pub mod mimo;
|
||||
pub mod volcengine;
|
||||
pub mod wav;
|
||||
pub mod whisper;
|
||||
|
||||
pub use bailian::{BailianCredentials, BailianRealtimeASR};
|
||||
pub use mimo::MimoBatchASR;
|
||||
pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR};
|
||||
pub use whisper::WhisperBatchASR;
|
||||
|
||||
|
||||
@@ -684,6 +684,11 @@ fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bo
|
||||
if provider == crate::asr::bailian::PROVIDER_ID {
|
||||
return configured(&snap.asr_api_key);
|
||||
}
|
||||
if provider == crate::asr::mimo::PROVIDER_ID {
|
||||
return configured(&snap.asr_api_key)
|
||||
&& configured(&snap.asr_endpoint)
|
||||
&& configured(&snap.asr_model);
|
||||
}
|
||||
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
|
||||
}
|
||||
|
||||
@@ -882,6 +887,11 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
|
||||
models: vec![crate::asr::bailian::DEFAULT_MODEL.to_string()],
|
||||
});
|
||||
}
|
||||
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::mimo::PROVIDER_ID {
|
||||
return Ok(ProviderModelsResult {
|
||||
models: vec![crate::asr::mimo::DEFAULT_MODEL.to_string()],
|
||||
});
|
||||
}
|
||||
if kind == "llm" && CredentialsVault::get_active_llm() == CODEX_OAUTH_PROVIDER_ID {
|
||||
return Ok(ProviderModelsResult {
|
||||
models: vec![
|
||||
@@ -1014,6 +1024,9 @@ async fn validate_asr_provider() -> Result<(), String> {
|
||||
if active_asr == crate::asr::bailian::PROVIDER_ID {
|
||||
return validate_bailian_asr_provider().await;
|
||||
}
|
||||
if active_asr == crate::asr::mimo::PROVIDER_ID {
|
||||
return validate_mimo_asr_provider().await;
|
||||
}
|
||||
|
||||
let config = read_openai_provider_config("asr")?;
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
@@ -1023,6 +1036,23 @@ async fn validate_asr_provider() -> Result<(), String> {
|
||||
validate_asr_transcription(&config, model.trim()).await
|
||||
}
|
||||
|
||||
async fn validate_mimo_asr_provider() -> Result<(), String> {
|
||||
let config = read_openai_provider_config("asr")?;
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.map_err(|e| e.to_string())?
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::mimo::DEFAULT_MODEL.to_string());
|
||||
let asr = crate::asr::MimoBatchASR::new(config.api_key, config.base_url, model);
|
||||
crate::recorder::AudioConsumer::consume_pcm_chunk(
|
||||
&asr,
|
||||
&encode_wav_16k_mono_silence(250)[44..],
|
||||
);
|
||||
asr.transcribe()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
async fn validate_bailian_asr_provider() -> Result<(), String> {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.map_err(|e| e.to_string())?
|
||||
|
||||
@@ -22,7 +22,7 @@ use crate::asr::local::{
|
||||
foundry, sherpa, FoundryLocalRuntime, FoundryLocalWhisperAsr, SherpaOnnxAsr, SherpaOnnxRuntime,
|
||||
};
|
||||
use crate::asr::{
|
||||
BailianCredentials, BailianRealtimeASR, DictionaryHotword, RawTranscript,
|
||||
BailianCredentials, BailianRealtimeASR, DictionaryHotword, MimoBatchASR, RawTranscript,
|
||||
VolcengineCredentials, VolcengineStreamingASR, WhisperBatchASR,
|
||||
};
|
||||
use crate::combo_hotkey::{ComboHotkeyError, ComboHotkeyEvent, ComboHotkeyMonitor};
|
||||
@@ -144,6 +144,7 @@ fn show_capsule_window_for_recording<R: tauri::Runtime>(
|
||||
enum ActiveAsr {
|
||||
Volcengine(Arc<VolcengineStreamingASR>),
|
||||
Whisper(Arc<WhisperBatchASR>),
|
||||
Mimo(Arc<MimoBatchASR>),
|
||||
Bailian(Arc<BailianRealtimeASR>),
|
||||
#[cfg(target_os = "windows")]
|
||||
FoundryLocalWhisper(Arc<FoundryLocalWhisperAsr>),
|
||||
@@ -170,6 +171,7 @@ fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ActiveAsrProviderKind {
|
||||
Bailian,
|
||||
Mimo,
|
||||
WhisperCompatible,
|
||||
Volcengine,
|
||||
}
|
||||
@@ -177,6 +179,8 @@ enum ActiveAsrProviderKind {
|
||||
fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind {
|
||||
if is_bailian_provider(id) {
|
||||
ActiveAsrProviderKind::Bailian
|
||||
} else if is_mimo_provider(id) {
|
||||
ActiveAsrProviderKind::Mimo
|
||||
} else if is_whisper_compatible_provider(id) {
|
||||
ActiveAsrProviderKind::WhisperCompatible
|
||||
} else {
|
||||
@@ -2536,6 +2540,10 @@ fn is_bailian_provider(id: &str) -> bool {
|
||||
id == crate::asr::bailian::PROVIDER_ID
|
||||
}
|
||||
|
||||
fn is_mimo_provider(id: &str) -> bool {
|
||||
id == crate::asr::mimo::PROVIDER_ID
|
||||
}
|
||||
|
||||
fn apply_chinese_script_preference(text: &str, pref: ChineseScriptPreference) -> String {
|
||||
if text.is_empty() {
|
||||
return String::new();
|
||||
@@ -2688,6 +2696,13 @@ async fn build_qa_asr_start(inner: &Arc<Inner>, active_asr: &str) -> Result<QaAs
|
||||
asr: Arc::new(BailianRealtimeASR::new(read_bailian_credentials())),
|
||||
bridge: Arc::new(DeferredAsrBridge::new()),
|
||||
}),
|
||||
ActiveAsrProviderKind::Mimo => {
|
||||
let (api_key, base_url, model) = read_mimo_credentials();
|
||||
let mimo = Arc::new(MimoBatchASR::new(api_key, base_url, model));
|
||||
let active = ActiveAsr::Mimo(Arc::clone(&mimo));
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = mimo;
|
||||
Ok(QaAsrStart::Ready { active, consumer })
|
||||
}
|
||||
ActiveAsrProviderKind::WhisperCompatible => {
|
||||
let (api_key, base_url, model) = read_whisper_credentials();
|
||||
let whisper_prompt =
|
||||
@@ -3088,6 +3103,24 @@ fn read_whisper_credentials() -> (String, String, String) {
|
||||
(api_key, base_url, model)
|
||||
}
|
||||
|
||||
fn read_mimo_credentials() -> (String, String, String) {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let base_url = CredentialsVault::get(CredentialAccount::AsrEndpoint)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::mimo::DEFAULT_ENDPOINT.to_string());
|
||||
let model = CredentialsVault::get(CredentialAccount::AsrModel)
|
||||
.ok()
|
||||
.flatten()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| crate::asr::mimo::DEFAULT_MODEL.to_string());
|
||||
(api_key, base_url, model)
|
||||
}
|
||||
|
||||
fn read_bailian_credentials() -> BailianCredentials {
|
||||
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
|
||||
.ok()
|
||||
@@ -3440,6 +3473,26 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
}
|
||||
ActiveAsr::Mimo(m) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS);
|
||||
match tokio::time::timeout(timeout_duration, m.transcribe()).await {
|
||||
Ok(Ok(r)) => r,
|
||||
Ok(Err(e)) => {
|
||||
log::error!("[coord] QA: MiMo ASR transcribe failed: {e}");
|
||||
finish_qa_with_error(inner, format!("识别失败: {e}"));
|
||||
return Err(e.to_string());
|
||||
}
|
||||
Err(_) => {
|
||||
log::error!(
|
||||
"[coord] QA: MiMo ASR 全局超时 {} 秒",
|
||||
COORDINATOR_GLOBAL_TIMEOUT_SECS
|
||||
);
|
||||
finish_qa_with_error(inner, "识别超时".to_string());
|
||||
return Err("mimo global timeout".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "windows")]
|
||||
ActiveAsr::FoundryLocalWhisper(local) => {
|
||||
debug_assert!(!uses_global_timeout);
|
||||
@@ -4075,6 +4128,9 @@ mod tests {
|
||||
assert!(!is_whisper_compatible_provider(
|
||||
crate::asr::local::sherpa::PROVIDER_ID
|
||||
));
|
||||
assert!(!is_whisper_compatible_provider(
|
||||
crate::asr::mimo::PROVIDER_ID
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4118,6 +4174,10 @@ mod tests {
|
||||
active_asr_provider_kind("whisper"),
|
||||
ActiveAsrProviderKind::WhisperCompatible
|
||||
);
|
||||
assert_eq!(
|
||||
active_asr_provider_kind(crate::asr::mimo::PROVIDER_ID),
|
||||
ActiveAsrProviderKind::Mimo
|
||||
);
|
||||
assert_eq!(
|
||||
active_asr_provider_kind("volcengine"),
|
||||
ActiveAsrProviderKind::Volcengine
|
||||
|
||||
@@ -800,6 +800,17 @@ pub(super) async fn begin_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
let flushed_bytes = bridge.attach(target);
|
||||
log::info!("[coord] Bailian ASR connected; flushed {flushed_bytes} deferred audio bytes");
|
||||
finish_starting_session(inner, current_session_id).await;
|
||||
} else if is_mimo_provider(&active_asr) {
|
||||
let (api_key, base_url, model) = read_mimo_credentials();
|
||||
let mimo = Arc::new(MimoBatchASR::new(api_key, base_url, model));
|
||||
store_asr_for_session(
|
||||
inner,
|
||||
current_session_id,
|
||||
ActiveAsr::Mimo(Arc::clone(&mimo)),
|
||||
);
|
||||
let consumer: Arc<dyn crate::recorder::AudioConsumer> = mimo;
|
||||
start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer)
|
||||
.await?;
|
||||
} else if is_whisper_compatible_provider(&active_asr) {
|
||||
let (api_key, base_url, model) = read_whisper_credentials();
|
||||
// 用户辞書の有効フレーズを Whisper の `prompt` に流し込む。固有名詞や
|
||||
@@ -1239,6 +1250,46 @@ pub(super) async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
}
|
||||
ActiveAsr::Mimo(m) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS);
|
||||
match tokio::time::timeout(timeout_duration, m.transcribe()).await {
|
||||
Ok(Ok(r)) => r,
|
||||
Ok(Err(e)) => {
|
||||
log::error!("[coord] MiMo ASR transcribe failed: {e}");
|
||||
emit_capsule(
|
||||
inner,
|
||||
CapsuleState::Error,
|
||||
0.0,
|
||||
elapsed,
|
||||
Some(format!("识别失败: {e}")),
|
||||
None,
|
||||
);
|
||||
restore_prepared_windows_ime_session(inner, current_session_id);
|
||||
inner.state.lock().phase = SessionPhase::Idle;
|
||||
schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS);
|
||||
return Err(e.to_string());
|
||||
}
|
||||
Err(_) => {
|
||||
log::error!(
|
||||
"[coord] MiMo ASR 全局超时 {} 秒",
|
||||
COORDINATOR_GLOBAL_TIMEOUT_SECS
|
||||
);
|
||||
emit_capsule(
|
||||
inner,
|
||||
CapsuleState::Error,
|
||||
0.0,
|
||||
elapsed,
|
||||
Some("识别超时".to_string()),
|
||||
None,
|
||||
);
|
||||
restore_prepared_windows_ime_session(inner, current_session_id);
|
||||
inner.state.lock().phase = SessionPhase::Idle;
|
||||
schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS);
|
||||
return Err("mimo global timeout".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ActiveAsr::Bailian(asr) => {
|
||||
debug_assert!(uses_global_timeout);
|
||||
if let Err(e) = asr.send_last_frame().await {
|
||||
|
||||
@@ -67,6 +67,7 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) {
|
||||
match asr {
|
||||
ActiveAsr::Volcengine(v) => v.cancel(),
|
||||
ActiveAsr::Whisper(w) => w.cancel(),
|
||||
ActiveAsr::Mimo(m) => m.cancel(),
|
||||
ActiveAsr::Bailian(b) => b.cancel(),
|
||||
#[cfg(target_os = "windows")]
|
||||
ActiveAsr::FoundryLocalWhisper(local) => local.cancel(),
|
||||
|
||||
@@ -619,6 +619,7 @@ export const en: typeof zhCN = {
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
asrWhisper: 'OpenAI Whisper (compatible)',
|
||||
asrOpenrouter: 'OpenRouter Whisper',
|
||||
asrXiaomiMimo: 'Xiaomi MiMo ASR',
|
||||
asrSherpaOnnxLocal: 'Local sherpa-onnx (experimental)',
|
||||
asrFoundryLocalWhisper: 'Local Whisper (Foundry Local)',
|
||||
asrLocalQwen3: 'Local Qwen3-ASR',
|
||||
|
||||
@@ -621,6 +621,7 @@ export const ja: typeof zhCN = {
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
asrWhisper: 'OpenAI Whisper(互換)',
|
||||
asrOpenrouter: 'OpenRouter Whisper',
|
||||
asrXiaomiMimo: 'Xiaomi MiMo ASR',
|
||||
asrSherpaOnnxLocal: 'ローカル sherpa-onnx(実験的)',
|
||||
asrFoundryLocalWhisper: 'ローカル Whisper(Foundry Local)',
|
||||
asrLocalQwen3: 'ローカル Qwen3-ASR',
|
||||
|
||||
@@ -621,6 +621,7 @@ export const ko: typeof zhCN = {
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
asrWhisper: 'OpenAI Whisper(호환)',
|
||||
asrOpenrouter: 'OpenRouter Whisper',
|
||||
asrXiaomiMimo: 'Xiaomi MiMo ASR',
|
||||
asrSherpaOnnxLocal: '로컬 sherpa-onnx(실험적)',
|
||||
asrFoundryLocalWhisper: '로컬 Whisper(Foundry Local)',
|
||||
asrLocalQwen3: '로컬 Qwen3-ASR',
|
||||
|
||||
@@ -617,6 +617,7 @@ export const zhCN = {
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
asrWhisper: 'OpenAI Whisper(兼容)',
|
||||
asrOpenrouter: 'OpenRouter Whisper',
|
||||
asrXiaomiMimo: '小米 MiMo ASR',
|
||||
asrSherpaOnnxLocal: '本地 sherpa-onnx(实验性)',
|
||||
asrFoundryLocalWhisper: '本地 Whisper(Foundry Local)',
|
||||
asrLocalQwen3: '本地 Qwen3-ASR',
|
||||
|
||||
@@ -619,6 +619,7 @@ export const zhTW: typeof zhCN = {
|
||||
asrGroq: 'Groq Whisper-large-v3',
|
||||
asrWhisper: 'OpenAI Whisper(兼容)',
|
||||
asrOpenrouter: 'OpenRouter Whisper',
|
||||
asrXiaomiMimo: '小米 MiMo ASR',
|
||||
asrSherpaOnnxLocal: '本地 sherpa-onnx(實驗性)',
|
||||
asrFoundryLocalWhisper: '本地 Whisper(Foundry Local)',
|
||||
asrLocalQwen3: '本地 Qwen3-ASR',
|
||||
|
||||
@@ -31,6 +31,7 @@ const ASR_NAME_KEY_BY_ID: Record<string, string> = {
|
||||
groq: 'asrGroq',
|
||||
whisper: 'asrWhisper',
|
||||
openrouter: 'asrOpenrouter',
|
||||
'xiaomi-mimo-asr': 'asrXiaomiMimo',
|
||||
'foundry-local-whisper': 'asrFoundryLocalWhisper',
|
||||
'sherpa-onnx-local': 'asrSherpaOnnxLocal',
|
||||
'local-qwen3': 'asrLocalQwen3',
|
||||
|
||||
@@ -132,7 +132,8 @@ const ASR_DEFAULT_RESOURCE_ID = 'volc.seedasr.sauc.duration';
|
||||
// `/audio/transcriptions`(`coordinator.rs::is_whisper_compatible_provider`)。
|
||||
// 新增兼容厂商:
|
||||
// 1. 在这里加一项 `{ id, nameKey, baseUrl, model }`;
|
||||
// 2. `coordinator.rs::is_whisper_compatible_provider` 加同名 id;
|
||||
// 2. 若走 Whisper 协议,`coordinator.rs::is_whisper_compatible_provider` 加同名 id;
|
||||
// 若是专有协议,新增独立 ASR client 与 provider kind;
|
||||
// 3. 在 i18n 的 `settings.providers.presets.<nameKey>` 加文案。
|
||||
// `AsrPresetId` 定义在 settings/shared.tsx,LocalModelSection / ProvidersSection 共用同一份。
|
||||
const ASR_PRESETS: ReadonlyArray<{ id: AsrPresetId; nameKey: string; baseUrl: string; model: string }> = [
|
||||
@@ -145,6 +146,9 @@ const ASR_PRESETS: ReadonlyArray<{ id: AsrPresetId; nameKey: string; baseUrl: st
|
||||
// OpenRouter 的 /audio/transcriptions 走 application/json + base64(issue #582),
|
||||
// 后端 coordinator.rs::whisper_request_format 对该 id 切换到 OpenRouterJson 编码。
|
||||
{ id: 'openrouter', nameKey: 'asrOpenrouter', baseUrl: 'https://openrouter.ai/api/v1', model: 'openai/whisper-large-v3-turbo' },
|
||||
// 小米 MiMo ASR 按官方文档走 /chat/completions + input_audio,不是
|
||||
// Whisper /audio/transcriptions;后端由 asr/mimo.rs 专用 client 处理。
|
||||
{ id: 'xiaomi-mimo-asr', nameKey: 'asrXiaomiMimo', baseUrl: 'https://api.xiaomimimo.com/v1', model: 'mimo-v2.5-asr' },
|
||||
{ id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' },
|
||||
// 本地引擎(Foundry / sherpa-onnx / Qwen3):无 baseUrl/model 配置,
|
||||
// 模型在「高级 → 本地模型」里下载与切换。
|
||||
|
||||
@@ -180,6 +180,7 @@ export type AsrPresetId =
|
||||
| "groq"
|
||||
| "whisper"
|
||||
| "openrouter"
|
||||
| "xiaomi-mimo-asr"
|
||||
| "foundry-local-whisper"
|
||||
| "sherpa-onnx-local"
|
||||
| "local-qwen3"
|
||||
Reference in new issue
Block a user