From a79b7429fe3e989643e3239f86ac67ff16a333ca Mon Sep 17 00:00:00 2001 From: H-Chris233 Date: Sat, 6 Jun 2026 17:58:23 +0800 Subject: [PATCH] feat(asr): add Xiaomi MiMo provider --- openless-all/app/src-tauri/src/asr/mimo.rs | 466 ++++++++++++++++++ openless-all/app/src-tauri/src/asr/mod.rs | 2 + openless-all/app/src-tauri/src/commands.rs | 30 ++ openless-all/app/src-tauri/src/coordinator.rs | 62 ++- .../src-tauri/src/coordinator/dictation.rs | 51 ++ .../src-tauri/src/coordinator/resources.rs | 1 + openless-all/app/src/i18n/en.ts | 1 + openless-all/app/src/i18n/ja.ts | 1 + openless-all/app/src/i18n/ko.ts | 1 + openless-all/app/src/i18n/zh-CN.ts | 1 + openless-all/app/src/i18n/zh-TW.ts | 1 + openless-all/app/src/pages/Overview.tsx | 1 + .../src/pages/settings/ProvidersSection.tsx | 6 +- .../app/src/pages/settings/shared.tsx | 1 + 14 files changed, 623 insertions(+), 2 deletions(-) create mode 100644 openless-all/app/src-tauri/src/asr/mimo.rs diff --git a/openless-all/app/src-tauri/src/asr/mimo.rs b/openless-all/app/src-tauri/src/asr/mimo.rs new file mode 100644 index 00000000..0cc1759f --- /dev/null +++ b/openless-all/app/src-tauri/src/asr/mimo.rs @@ -0,0 +1,466 @@ +//! Xiaomi MiMo ASR client. +//! +//! MiMo ASR uses the official OpenAI-compatible `/chat/completions` endpoint +//! with `messages[].content[].input_audio`, not Whisper's +//! `/audio/transcriptions` protocol. + +use anyhow::{Context, Result}; +use base64::Engine; +use parking_lot::Mutex; +use serde_json::Value; + +use crate::asr::wav::encode_wav_16k_mono; +use crate::asr::RawTranscript; + +const PCM_SAMPLE_RATE_HZ: u64 = 16_000; +const PCM_BYTES_PER_SAMPLE: usize = 2; +// 官方限制：Base64 后的音频数据不能超过 10MB。180s 的 16k/16-bit/mono WAV +// Base64 后约 7.7MB，给 JSON/data-url 前缀和厂商侧 MB 口径差异留余量。 +const MIMO_MAX_CHUNK_DURATION_MS: u64 = 180_000; +pub const PROVIDER_ID: &str = "xiaomi-mimo-asr"; +pub const DEFAULT_ENDPOINT: &str = "https://api.xiaomimimo.com/v1"; +pub const DEFAULT_MODEL: &str = "mimo-v2.5-asr"; + +pub struct MimoBatchASR { + api_key: String, + base_url: String, + model: String, + buffer: Mutex>, +} + +impl MimoBatchASR { + pub fn new(api_key: String, base_url: String, model: String) -> Self { + Self { + api_key, + base_url, + model, + buffer: Mutex::new(Vec::new()), + } + } + + pub async fn transcribe(&self) -> Result { + let pcm = self.buffer.lock().clone(); + if pcm.is_empty() { + return Ok(RawTranscript { + text: String::new(), + duration_ms: 0, + }); + } + + let result = self.transcribe_inner(&pcm).await; + if result.is_ok() { + self.buffer.lock().clear(); + } + result + } + + async fn transcribe_inner(&self, pcm: &[u8]) -> Result { + if self.api_key.trim().is_empty() { + anyhow::bail!("MiMo API key missing"); + } + + let duration_ms = pcm_duration_ms(pcm); + let chunks = split_pcm_by_duration(pcm, MIMO_MAX_CHUNK_DURATION_MS); + let mut texts = Vec::with_capacity(chunks.len()); + for chunk in chunks { + texts.push(self.transcribe_chunk(chunk).await?); + } + + Ok(RawTranscript { + text: join_transcript_chunks(&texts), + duration_ms, + }) + } + + async fn transcribe_chunk(&self, pcm: &[u8]) -> Result { + let samples: Vec = pcm + .chunks_exact(2) + .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + let wav = encode_wav_16k_mono(&samples); + let body = mimo_chat_body(&self.model, &wav); + let url = mimo_chat_completions_url(&self.base_url)?; + let client = reqwest::Client::new(); + let resp = client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key.trim())) + .json(&body) + .send() + .await + .context("MiMo ASR HTTP request failed")?; + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + anyhow::bail!("MiMo ASR API error {}: {}", status, body); + } + + let json: Value = resp.json().await.context("parse MiMo ASR response")?; + Ok(extract_mimo_text(&json).trim().to_string()) + } + + pub fn cancel(&self) { + self.buffer.lock().clear(); + } +} + +impl crate::recorder::AudioConsumer for MimoBatchASR { + fn consume_pcm_chunk(&self, pcm: &[u8]) { + self.buffer.lock().extend_from_slice(pcm); + } +} + +pub fn mimo_chat_completions_url(base_url: &str) -> Result { + let parsed = reqwest::Url::parse(base_url.trim()).context("parse MiMo base URL")?; + let mut url = parsed.clone(); + let path = parsed.path().trim_end_matches('/'); + let next_path = if path.ends_with("/chat/completions") { + path.to_string() + } else if path.ends_with("/chat") { + format!("{path}/completions") + } else { + format!("{path}/chat/completions") + }; + url.set_path(&next_path); + Ok(url.to_string()) +} + +pub fn mimo_chat_body(model: &str, wav: &[u8]) -> Value { + let audio_data = format!( + "data:audio/wav;base64,{}", + base64::engine::general_purpose::STANDARD.encode(wav) + ); + serde_json::json!({ + "model": model, + "stream": false, + "messages": [{ + "role": "user", + "content": [{ + "type": "input_audio", + "input_audio": { + "data": audio_data, + "format": "wav", + }, + }], + }], + }) +} + +pub fn extract_mimo_text(json: &Value) -> String { + let Some(content) = json + .get("choices") + .and_then(|v| v.as_array()) + .and_then(|choices| choices.first()) + .and_then(|choice| choice.get("message")) + .and_then(|message| message.get("content")) + else { + return String::new(); + }; + + if let Some(text) = content.as_str() { + return text.to_string(); + } + + content + .as_array() + .map(|items| { + items + .iter() + .filter_map(|item| { + item.get("text") + .and_then(|text| text.as_str()) + .or_else(|| item.get("content").and_then(|text| text.as_str())) + }) + .collect::>() + .join("") + }) + .unwrap_or_default() +} + +fn pcm_duration_ms(pcm: &[u8]) -> u64 { + (pcm.len() as u64 / PCM_BYTES_PER_SAMPLE as u64) * 1000 / PCM_SAMPLE_RATE_HZ +} + +fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> { + if max_chunk_duration_ms == 0 { + return vec![pcm]; + } + + let samples_per_chunk = PCM_SAMPLE_RATE_HZ * max_chunk_duration_ms / 1000; + let bytes_per_chunk = samples_per_chunk as usize * PCM_BYTES_PER_SAMPLE; + if bytes_per_chunk == 0 || pcm.len() <= bytes_per_chunk { + return vec![pcm]; + } + + pcm.chunks(bytes_per_chunk).collect() +} + +fn join_transcript_chunks(chunks: &[String]) -> String { + let mut joined = String::new(); + for chunk in chunks.iter().map(|chunk| chunk.trim()) { + if chunk.is_empty() { + continue; + } + if needs_chunk_separator(&joined, chunk) { + joined.push(' '); + } + joined.push_str(chunk); + } + joined +} + +fn needs_chunk_separator(current: &str, next: &str) -> bool { + let Some(prev) = current.chars().last() else { + return false; + }; + let Some(first) = next.chars().next() else { + return false; + }; + + if is_closing_punctuation(first) || is_opening_punctuation(prev) { + return false; + } + if is_cjk(prev) && (is_cjk(first) || is_opening_punctuation(first)) { + return false; + } + if is_cjk(first) && is_closing_punctuation(prev) { + return false; + } + if is_cjk_punctuation(prev) && is_cjk(first) { + return false; + } + true +} + +fn is_cjk(c: char) -> bool { + matches!( + c as u32, + 0x4E00..=0x9FFF + | 0x3400..=0x4DBF + | 0x3040..=0x30FF + | 0xAC00..=0xD7AF + ) +} + +fn is_cjk_punctuation(c: char) -> bool { + matches!( + c, + '。' | '，' | '、' | '！' | '？' | '；' | '：' | '」' | '』' | '）' | '》' + ) +} + +fn is_closing_punctuation(c: char) -> bool { + c.is_ascii_punctuation() && !matches!(c, '(' | '[' | '{' | '"' | '\'') || is_cjk_punctuation(c) +} + +fn is_opening_punctuation(c: char) -> bool { + matches!(c, '(' | '[' | '{' | '"' | '\'' | '「' | '『' | '（' | '《') +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::recorder::AudioConsumer; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn mimo_url_targets_chat_completions() { + assert_eq!( + mimo_chat_completions_url("https://api.xiaomimimo.com/v1").unwrap(), + "https://api.xiaomimimo.com/v1/chat/completions" + ); + assert_eq!( + mimo_chat_completions_url("https://api.xiaomimimo.com/v1/chat/completions").unwrap(), + "https://api.xiaomimimo.com/v1/chat/completions" + ); + } + + #[test] + fn mimo_body_uses_official_input_audio_shape() { + let body = mimo_chat_body(DEFAULT_MODEL, b"wav"); + assert_eq!(body["model"], DEFAULT_MODEL); + assert_eq!(body["stream"], false); + let audio = &body["messages"][0]["content"][0]; + assert_eq!(audio["type"], "input_audio"); + assert_eq!(audio["input_audio"]["format"], "wav"); + assert!(audio["input_audio"]["data"] + .as_str() + .unwrap() + .starts_with("data:audio/wav;base64,")); + } + + #[test] + fn mimo_response_extracts_message_content() { + let json = serde_json::json!({ + "choices": [{ "message": { "content": "你好 MiMo" } }] + }); + assert_eq!(extract_mimo_text(&json), "你好 MiMo"); + } + + #[test] + fn mimo_response_accepts_content_array_text() { + let json = serde_json::json!({ + "choices": [{ + "message": { + "content": [ + { "type": "text", "text": "你好" }, + { "type": "text", "text": "MiMo" } + ] + } + }] + }); + assert_eq!(extract_mimo_text(&json), "你好MiMo"); + } + + #[tokio::test] + async fn mimo_posts_chat_completion_audio_request() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "timed out waiting for MiMo ASR test request" + ); + thread::sleep(Duration::from_millis(10)); + } + Err(err) => panic!("accept MiMo ASR test request failed: {err}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let request = read_http_request(&mut stream); + let request_text = String::from_utf8_lossy(&request); + let lower = request_text.to_ascii_lowercase(); + assert!(request_text.starts_with("POST /chat/completions HTTP/1.1")); + assert!(lower.contains("authorization: bearer key")); + assert!(lower.contains("content-type: application/json")); + assert!(request_text.contains(r#""model":"mimo-v2.5-asr""#)); + assert!(request_text.contains(r#""type":"input_audio""#)); + assert!(request_text.contains("data:audio/wav;base64,")); + write_json_response( + &mut stream, + r#"{"choices":[{"message":{"content":"mimo ok"}}]}"#, + ); + }); + + let asr = MimoBatchASR::new( + "key".to_string(), + format!("http://{}", addr), + DEFAULT_MODEL.to_string(), + ); + asr.consume_pcm_chunk(&vec![0u8; 32_000]); + let transcript = asr.transcribe().await.unwrap(); + + assert_eq!(transcript.text, "mimo ok"); + assert_eq!(transcript.duration_ms, 1_000); + server.join().unwrap(); + } + + #[tokio::test] + async fn mimo_splits_audio_before_base64_limit() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + for (index, text) in ["hello", "world"].into_iter().enumerate() { + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + Instant::now() < deadline, + "timed out waiting for MiMo ASR chunk request" + ); + thread::sleep(Duration::from_millis(10)); + } + Err(err) => panic!("accept MiMo ASR chunk request failed: {err}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let request = read_http_request(&mut stream); + let request_text = String::from_utf8_lossy(&request); + assert!(request_text.starts_with("POST /chat/completions HTTP/1.1")); + assert!(request_text.contains(r#""model":"mimo-v2.5-asr""#)); + assert!(request_text.contains("data:audio/wav;base64,")); + assert!( + request_text.len() < 10 * 1024 * 1024, + "chunk {index} exceeded MiMo 10MB base64 request budget" + ); + write_json_response( + &mut stream, + &format!(r#"{{"choices":[{{"message":{{"content":"{text}"}}}}]}}"#), + ); + } + }); + + let asr = MimoBatchASR::new( + "key".to_string(), + format!("http://{}", addr), + DEFAULT_MODEL.to_string(), + ); + let pcm = vec![0u8; 32_000 * 181]; + asr.consume_pcm_chunk(&pcm); + let transcript = asr.transcribe().await.unwrap(); + + assert_eq!(transcript.text, "hello world"); + assert_eq!(transcript.duration_ms, 181_000); + server.join().unwrap(); + } + + fn read_http_request(stream: &mut std::net::TcpStream) -> Vec { + let mut request = Vec::new(); + let mut buf = [0u8; 4096]; + let mut expected_len = None; + loop { + let read = stream.read(&mut buf).unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&buf[..read]); + if expected_len.is_none() { + expected_len = parse_expected_request_len(&request); + } + if expected_len.is_some_and(|len| request.len() >= len) { + break; + } + } + request + } + + fn parse_expected_request_len(request: &[u8]) -> Option { + let header_end = request.windows(4).position(|w| w == b"\r\n\r\n")? + 4; + let headers = String::from_utf8_lossy(&request[..header_end]); + let content_len = headers.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + if name.eq_ignore_ascii_case("content-length") { + value.trim().parse::().ok() + } else { + None + } + })?; + Some(header_end + content_len) + } + + fn write_json_response(stream: &mut std::net::TcpStream, body: &str) { + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.flush().unwrap(); + } +} diff --git a/openless-all/app/src-tauri/src/asr/mod.rs b/openless-all/app/src-tauri/src/asr/mod.rs index 4433556d..1b51857f 100644 --- a/openless-all/app/src-tauri/src/asr/mod.rs +++ b/openless-all/app/src-tauri/src/asr/mod.rs @@ -8,11 +8,13 @@ pub mod bailian; mod frame; pub mod local; +pub mod mimo; pub mod volcengine; pub mod wav; pub mod whisper; pub use bailian::{BailianCredentials, BailianRealtimeASR}; +pub use mimo::MimoBatchASR; pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR}; pub use whisper::WhisperBatchASR; diff --git a/openless-all/app/src-tauri/src/commands.rs b/openless-all/app/src-tauri/src/commands.rs index 1ed9af5e..9ea29c18 100644 --- a/openless-all/app/src-tauri/src/commands.rs +++ b/openless-all/app/src-tauri/src/commands.rs @@ -684,6 +684,11 @@ fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bo if provider == crate::asr::bailian::PROVIDER_ID { return configured(&snap.asr_api_key); } + if provider == crate::asr::mimo::PROVIDER_ID { + return configured(&snap.asr_api_key) + && configured(&snap.asr_endpoint) + && configured(&snap.asr_model); + } configured(&snap.asr_endpoint) && configured(&snap.asr_model) } @@ -882,6 +887,11 @@ pub async fn list_provider_models(kind: String) -> Result Result<(), String> { if active_asr == crate::asr::bailian::PROVIDER_ID { return validate_bailian_asr_provider().await; } + if active_asr == crate::asr::mimo::PROVIDER_ID { + return validate_mimo_asr_provider().await; + } let config = read_openai_provider_config("asr")?; let model = CredentialsVault::get(CredentialAccount::AsrModel) @@ -1023,6 +1036,23 @@ async fn validate_asr_provider() -> Result<(), String> { validate_asr_transcription(&config, model.trim()).await } +async fn validate_mimo_asr_provider() -> Result<(), String> { + let config = read_openai_provider_config("asr")?; + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .map_err(|e| e.to_string())? + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::mimo::DEFAULT_MODEL.to_string()); + let asr = crate::asr::MimoBatchASR::new(config.api_key, config.base_url, model); + crate::recorder::AudioConsumer::consume_pcm_chunk( + &asr, + &encode_wav_16k_mono_silence(250)[44..], + ); + asr.transcribe() + .await + .map(|_| ()) + .map_err(|e| e.to_string()) +} + async fn validate_bailian_asr_provider() -> Result<(), String> { let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) .map_err(|e| e.to_string())? diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index f8c58e6c..c9a7417d 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -22,7 +22,7 @@ use crate::asr::local::{ foundry, sherpa, FoundryLocalRuntime, FoundryLocalWhisperAsr, SherpaOnnxAsr, SherpaOnnxRuntime, }; use crate::asr::{ - BailianCredentials, BailianRealtimeASR, DictionaryHotword, RawTranscript, + BailianCredentials, BailianRealtimeASR, DictionaryHotword, MimoBatchASR, RawTranscript, VolcengineCredentials, VolcengineStreamingASR, WhisperBatchASR, }; use crate::combo_hotkey::{ComboHotkeyError, ComboHotkeyEvent, ComboHotkeyMonitor}; @@ -144,6 +144,7 @@ fn show_capsule_window_for_recording( enum ActiveAsr { Volcengine(Arc), Whisper(Arc), + Mimo(Arc), Bailian(Arc), #[cfg(target_os = "windows")] FoundryLocalWhisper(Arc), @@ -170,6 +171,7 @@ fn asr_transcribe_uses_global_timeout(asr: &ActiveAsr) -> bool { #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ActiveAsrProviderKind { Bailian, + Mimo, WhisperCompatible, Volcengine, } @@ -177,6 +179,8 @@ enum ActiveAsrProviderKind { fn active_asr_provider_kind(id: &str) -> ActiveAsrProviderKind { if is_bailian_provider(id) { ActiveAsrProviderKind::Bailian + } else if is_mimo_provider(id) { + ActiveAsrProviderKind::Mimo } else if is_whisper_compatible_provider(id) { ActiveAsrProviderKind::WhisperCompatible } else { @@ -2536,6 +2540,10 @@ fn is_bailian_provider(id: &str) -> bool { id == crate::asr::bailian::PROVIDER_ID } +fn is_mimo_provider(id: &str) -> bool { + id == crate::asr::mimo::PROVIDER_ID +} + fn apply_chinese_script_preference(text: &str, pref: ChineseScriptPreference) -> String { if text.is_empty() { return String::new(); @@ -2688,6 +2696,13 @@ async fn build_qa_asr_start(inner: &Arc, active_asr: &str) -> Result { + let (api_key, base_url, model) = read_mimo_credentials(); + let mimo = Arc::new(MimoBatchASR::new(api_key, base_url, model)); + let active = ActiveAsr::Mimo(Arc::clone(&mimo)); + let consumer: Arc = mimo; + Ok(QaAsrStart::Ready { active, consumer }) + } ActiveAsrProviderKind::WhisperCompatible => { let (api_key, base_url, model) = read_whisper_credentials(); let whisper_prompt = @@ -3088,6 +3103,24 @@ fn read_whisper_credentials() -> (String, String, String) { (api_key, base_url, model) } +fn read_mimo_credentials() -> (String, String, String) { + let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) + .ok() + .flatten() + .unwrap_or_default(); + let base_url = CredentialsVault::get(CredentialAccount::AsrEndpoint) + .ok() + .flatten() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::mimo::DEFAULT_ENDPOINT.to_string()); + let model = CredentialsVault::get(CredentialAccount::AsrModel) + .ok() + .flatten() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| crate::asr::mimo::DEFAULT_MODEL.to_string()); + (api_key, base_url, model) +} + fn read_bailian_credentials() -> BailianCredentials { let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey) .ok() @@ -3440,6 +3473,26 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { } } } + ActiveAsr::Mimo(m) => { + debug_assert!(uses_global_timeout); + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, m.transcribe()).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + log::error!("[coord] QA: MiMo ASR transcribe failed: {e}"); + finish_qa_with_error(inner, format!("识别失败: {e}")); + return Err(e.to_string()); + } + Err(_) => { + log::error!( + "[coord] QA: MiMo ASR 全局超时 {} 秒", + COORDINATOR_GLOBAL_TIMEOUT_SECS + ); + finish_qa_with_error(inner, "识别超时".to_string()); + return Err("mimo global timeout".to_string()); + } + } + } #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => { debug_assert!(!uses_global_timeout); @@ -4075,6 +4128,9 @@ mod tests { assert!(!is_whisper_compatible_provider( crate::asr::local::sherpa::PROVIDER_ID )); + assert!(!is_whisper_compatible_provider( + crate::asr::mimo::PROVIDER_ID + )); } #[test] @@ -4118,6 +4174,10 @@ mod tests { active_asr_provider_kind("whisper"), ActiveAsrProviderKind::WhisperCompatible ); + assert_eq!( + active_asr_provider_kind(crate::asr::mimo::PROVIDER_ID), + ActiveAsrProviderKind::Mimo + ); assert_eq!( active_asr_provider_kind("volcengine"), ActiveAsrProviderKind::Volcengine diff --git a/openless-all/app/src-tauri/src/coordinator/dictation.rs b/openless-all/app/src-tauri/src/coordinator/dictation.rs index dfbbe3a5..e3371177 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation.rs @@ -800,6 +800,17 @@ pub(super) async fn begin_session(inner: &Arc) -> Result<(), String> { let flushed_bytes = bridge.attach(target); log::info!("[coord] Bailian ASR connected; flushed {flushed_bytes} deferred audio bytes"); finish_starting_session(inner, current_session_id).await; + } else if is_mimo_provider(&active_asr) { + let (api_key, base_url, model) = read_mimo_credentials(); + let mimo = Arc::new(MimoBatchASR::new(api_key, base_url, model)); + store_asr_for_session( + inner, + current_session_id, + ActiveAsr::Mimo(Arc::clone(&mimo)), + ); + let consumer: Arc = mimo; + start_recorder_and_enter_listening(inner, current_session_id, &active_asr, consumer) + .await?; } else if is_whisper_compatible_provider(&active_asr) { let (api_key, base_url, model) = read_whisper_credentials(); // 用户辞書の有効フレーズを Whisper の `prompt` に流し込む。固有名詞や @@ -1239,6 +1250,46 @@ pub(super) async fn end_session(inner: &Arc) -> Result<(), String> { } } } + ActiveAsr::Mimo(m) => { + debug_assert!(uses_global_timeout); + let timeout_duration = std::time::Duration::from_secs(COORDINATOR_GLOBAL_TIMEOUT_SECS); + match tokio::time::timeout(timeout_duration, m.transcribe()).await { + Ok(Ok(r)) => r, + Ok(Err(e)) => { + log::error!("[coord] MiMo ASR transcribe failed: {e}"); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some(format!("识别失败: {e}")), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err(e.to_string()); + } + Err(_) => { + log::error!( + "[coord] MiMo ASR 全局超时 {} 秒", + COORDINATOR_GLOBAL_TIMEOUT_SECS + ); + emit_capsule( + inner, + CapsuleState::Error, + 0.0, + elapsed, + Some("识别超时".to_string()), + None, + ); + restore_prepared_windows_ime_session(inner, current_session_id); + inner.state.lock().phase = SessionPhase::Idle; + schedule_capsule_idle(inner, CAPSULE_AUTO_HIDE_DELAY_MS); + return Err("mimo global timeout".to_string()); + } + } + } ActiveAsr::Bailian(asr) => { debug_assert!(uses_global_timeout); if let Err(e) = asr.send_last_frame().await { diff --git a/openless-all/app/src-tauri/src/coordinator/resources.rs b/openless-all/app/src-tauri/src/coordinator/resources.rs index b92299ea..b9562c01 100644 --- a/openless-all/app/src-tauri/src/coordinator/resources.rs +++ b/openless-all/app/src-tauri/src/coordinator/resources.rs @@ -67,6 +67,7 @@ pub(super) fn cancel_active_asr(asr: ActiveAsr) { match asr { ActiveAsr::Volcengine(v) => v.cancel(), ActiveAsr::Whisper(w) => w.cancel(), + ActiveAsr::Mimo(m) => m.cancel(), ActiveAsr::Bailian(b) => b.cancel(), #[cfg(target_os = "windows")] ActiveAsr::FoundryLocalWhisper(local) => local.cancel(), diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 507ce497..3a0e4c45 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -619,6 +619,7 @@ export const en: typeof zhCN = { asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper (compatible)', asrOpenrouter: 'OpenRouter Whisper', + asrXiaomiMimo: 'Xiaomi MiMo ASR', asrSherpaOnnxLocal: 'Local sherpa-onnx (experimental)', asrFoundryLocalWhisper: 'Local Whisper (Foundry Local)', asrLocalQwen3: 'Local Qwen3-ASR', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index fa83f525..dfdb8b31 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -621,6 +621,7 @@ export const ja: typeof zhCN = { asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper（互換）', asrOpenrouter: 'OpenRouter Whisper', + asrXiaomiMimo: 'Xiaomi MiMo ASR', asrSherpaOnnxLocal: 'ローカル sherpa-onnx（実験的）', asrFoundryLocalWhisper: 'ローカル Whisper（Foundry Local）', asrLocalQwen3: 'ローカル Qwen3-ASR', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 3e3c0887..3287d126 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -621,6 +621,7 @@ export const ko: typeof zhCN = { asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper(호환)', asrOpenrouter: 'OpenRouter Whisper', + asrXiaomiMimo: 'Xiaomi MiMo ASR', asrSherpaOnnxLocal: '로컬 sherpa-onnx(실험적)', asrFoundryLocalWhisper: '로컬 Whisper(Foundry Local)', asrLocalQwen3: '로컬 Qwen3-ASR', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 14289086..01c1664c 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -617,6 +617,7 @@ export const zhCN = { asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper（兼容）', asrOpenrouter: 'OpenRouter Whisper', + asrXiaomiMimo: '小米 MiMo ASR', asrSherpaOnnxLocal: '本地 sherpa-onnx（实验性）', asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）', asrLocalQwen3: '本地 Qwen3-ASR', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 0aafd1b4..e35b7338 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -619,6 +619,7 @@ export const zhTW: typeof zhCN = { asrGroq: 'Groq Whisper-large-v3', asrWhisper: 'OpenAI Whisper（兼容）', asrOpenrouter: 'OpenRouter Whisper', + asrXiaomiMimo: '小米 MiMo ASR', asrSherpaOnnxLocal: '本地 sherpa-onnx（實驗性）', asrFoundryLocalWhisper: '本地 Whisper（Foundry Local）', asrLocalQwen3: '本地 Qwen3-ASR', diff --git a/openless-all/app/src/pages/Overview.tsx b/openless-all/app/src/pages/Overview.tsx index 71101040..118ef703 100644 --- a/openless-all/app/src/pages/Overview.tsx +++ b/openless-all/app/src/pages/Overview.tsx @@ -31,6 +31,7 @@ const ASR_NAME_KEY_BY_ID: Record = { groq: 'asrGroq', whisper: 'asrWhisper', openrouter: 'asrOpenrouter', + 'xiaomi-mimo-asr': 'asrXiaomiMimo', 'foundry-local-whisper': 'asrFoundryLocalWhisper', 'sherpa-onnx-local': 'asrSherpaOnnxLocal', 'local-qwen3': 'asrLocalQwen3', diff --git a/openless-all/app/src/pages/settings/ProvidersSection.tsx b/openless-all/app/src/pages/settings/ProvidersSection.tsx index dae82737..6521617f 100644 --- a/openless-all/app/src/pages/settings/ProvidersSection.tsx +++ b/openless-all/app/src/pages/settings/ProvidersSection.tsx @@ -132,7 +132,8 @@ const ASR_DEFAULT_RESOURCE_ID = 'volc.seedasr.sauc.duration'; // `/audio/transcriptions`（`coordinator.rs::is_whisper_compatible_provider`）。 // 新增兼容厂商： // 1. 在这里加一项 `{ id, nameKey, baseUrl, model }`； -// 2. `coordinator.rs::is_whisper_compatible_provider` 加同名 id； +// 2. 若走 Whisper 协议，`coordinator.rs::is_whisper_compatible_provider` 加同名 id； +// 若是专有协议，新增独立 ASR client 与 provider kind； // 3. 在 i18n 的 `settings.providers.presets.` 加文案。 // `AsrPresetId` 定义在 settings/shared.tsx，LocalModelSection / ProvidersSection 共用同一份。 const ASR_PRESETS: ReadonlyArray<{ id: AsrPresetId; nameKey: string; baseUrl: string; model: string }> = [ @@ -145,6 +146,9 @@ const ASR_PRESETS: ReadonlyArray<{ id: AsrPresetId; nameKey: string; baseUrl: st // OpenRouter 的 /audio/transcriptions 走 application/json + base64（issue #582）， // 后端 coordinator.rs::whisper_request_format 对该 id 切换到 OpenRouterJson 编码。 { id: 'openrouter', nameKey: 'asrOpenrouter', baseUrl: 'https://openrouter.ai/api/v1', model: 'openai/whisper-large-v3-turbo' }, + // 小米 MiMo ASR 按官方文档走 /chat/completions + input_audio，不是 + // Whisper /audio/transcriptions；后端由 asr/mimo.rs 专用 client 处理。 + { id: 'xiaomi-mimo-asr', nameKey: 'asrXiaomiMimo', baseUrl: 'https://api.xiaomimimo.com/v1', model: 'mimo-v2.5-asr' }, { id: 'foundry-local-whisper', nameKey: 'asrFoundryLocalWhisper', baseUrl: '', model: '' }, // 本地引擎（Foundry / sherpa-onnx / Qwen3）：无 baseUrl/model 配置， // 模型在「高级 → 本地模型」里下载与切换。 diff --git a/openless-all/app/src/pages/settings/shared.tsx b/openless-all/app/src/pages/settings/shared.tsx index 18eb644d..0cf13433 100644 --- a/openless-all/app/src/pages/settings/shared.tsx +++ b/openless-all/app/src/pages/settings/shared.tsx @@ -180,6 +180,7 @@ export type AsrPresetId = | "groq" | "whisper" | "openrouter" + | "xiaomi-mimo-asr" | "foundry-local-whisper" | "sherpa-onnx-local" | "local-qwen3"