Tighten Qwen3 tokenizer cache validation

This commit is contained in:
weikeyi committed 2026-05-23 16:18:05 +08:00
1 parent cd9696d561
commit 75a5082dc7
1 file changed
+33 -1
@@ -130,7 +130,7 @@ pub fn required_files_for_alias(alias: &str) -> Result<&'static [&'static str]>
pub fn required_path_is_valid(alias: &str, required: &str, path: &Path) -> bool {
if required_path_is_dir(alias, required) {
path.is_dir()
required_dir_is_valid(alias, required, path)
} else {
path.is_file()
}
@@ -140,6 +140,13 @@ fn required_path_is_dir(alias: &str, required: &str) -> bool {
matches!((alias, required), ("qwen3-asr-0.6b-int8", "tokenizer"))
}
fn required_dir_is_valid(alias: &str, required: &str, path: &Path) -> bool {
match (alias, required) {
("qwen3-asr-0.6b-int8", "tokenizer") => path.join("tokenizer.json").is_file(),
_ => false,
}
}
pub fn download_files_for_alias(alias: &str) -> Result<&'static [(&'static str, &'static str)]> {
match alias {
"sense-voice-small-zh" => Ok(&[
@@ -308,6 +315,7 @@ impl SherpaRuntimeStatus {
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
#[test]
fn provider_id_is_stable() {
@@ -336,6 +344,30 @@ mod tests {
assert!(catalog.iter().all(|m| !m.cached));
}
#[test]
fn qwen3_tokenizer_requires_tokenizer_json() {
let dir = std::env::temp_dir().join(format!(
"openless-sherpa-tokenizer-{}",
uuid::Uuid::new_v4()
));
fs::create_dir_all(&dir).expect("create tokenizer dir");
assert!(!required_path_is_valid(
"qwen3-asr-0.6b-int8",
"tokenizer",
&dir
));
fs::write(dir.join("tokenizer.json"), b"{}").expect("write tokenizer json");
assert!(required_path_is_valid(
"qwen3-asr-0.6b-int8",
"tokenizer",
&dir
));
let _ = fs::remove_dir_all(dir);
}
#[test]
fn unavailable_status_uses_provider_id() {
let status = SherpaRuntimeStatus::unavailable("paraformer-zh".into(), "not ready");