Tighten Qwen3 tokenizer cache validation
This commit is contained in:
1 parent
cd9696d561
commit
75a5082dc7
1 file changed
+33
-1
@@ -130,7 +130,7 @@ pub fn required_files_for_alias(alias: &str) -> Result<&'static [&'static str]>
|
||||
|
||||
pub fn required_path_is_valid(alias: &str, required: &str, path: &Path) -> bool {
|
||||
if required_path_is_dir(alias, required) {
|
||||
path.is_dir()
|
||||
required_dir_is_valid(alias, required, path)
|
||||
} else {
|
||||
path.is_file()
|
||||
}
|
||||
@@ -140,6 +140,13 @@ fn required_path_is_dir(alias: &str, required: &str) -> bool {
|
||||
matches!((alias, required), ("qwen3-asr-0.6b-int8", "tokenizer"))
|
||||
}
|
||||
|
||||
fn required_dir_is_valid(alias: &str, required: &str, path: &Path) -> bool {
|
||||
match (alias, required) {
|
||||
("qwen3-asr-0.6b-int8", "tokenizer") => path.join("tokenizer.json").is_file(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn download_files_for_alias(alias: &str) -> Result<&'static [(&'static str, &'static str)]> {
|
||||
match alias {
|
||||
"sense-voice-small-zh" => Ok(&[
|
||||
@@ -308,6 +315,7 @@ impl SherpaRuntimeStatus {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
#[test]
|
||||
fn provider_id_is_stable() {
|
||||
@@ -336,6 +344,30 @@ mod tests {
|
||||
assert!(catalog.iter().all(|m| !m.cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qwen3_tokenizer_requires_tokenizer_json() {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"openless-sherpa-tokenizer-{}",
|
||||
uuid::Uuid::new_v4()
|
||||
));
|
||||
fs::create_dir_all(&dir).expect("create tokenizer dir");
|
||||
|
||||
assert!(!required_path_is_valid(
|
||||
"qwen3-asr-0.6b-int8",
|
||||
"tokenizer",
|
||||
&dir
|
||||
));
|
||||
|
||||
fs::write(dir.join("tokenizer.json"), b"{}").expect("write tokenizer json");
|
||||
assert!(required_path_is_valid(
|
||||
"qwen3-asr-0.6b-int8",
|
||||
"tokenizer",
|
||||
&dir
|
||||
));
|
||||
|
||||
let _ = fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_status_uses_provider_id() {
|
||||
let status = SherpaRuntimeStatus::unavailable("paraformer-zh".into(), "not ready");
|
||||
|
||||
Reference in new issue
Block a user