fix: normalize OCR images and validate native text recognition

This commit is contained in:
Codex committed 2026-10-12 12:03:43 +00:00
1 parent 2561db83a7
commit bf75be270a
12 files changed
+292 -28

No files matched your search

+2
View File
@@ -51,6 +51,8 @@ jobs:
~/.cargo/git
src-tauri/target
key: ${{ runner.os }}-community-cargo-${{ hashFiles('src-tauri/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-community-cargo-
- name: Install frontend dependencies
run: npm run ci:install
+7 -4
View File
@@ -9,7 +9,7 @@
## ✨ 功能
- **快捷键呼出翻译** — 默认 `⌘⇧Space` 呼出/收起窗口,输入文本回车翻译,`Esc` 收起,失焦自动隐藏
- **截图 OCR 翻译** — `⌘⇧S` 框选屏幕区域,调用 macOS 原生 **Vision** 框架识别文字并自动翻译(按系统实际支持配置中/英/日等语言,失败时自动兼容重试)
- **截图 OCR 翻译** — `⌘⇧S` 框选屏幕区域,调用 macOS 原生 **Vision** 框架识别文字并自动翻译(按系统实际支持配置中/英/日等语言,截图先规范化为 8-bit sRGB 位图,失败时自动兼容重试)
- **划词翻译 / 划词 AI** — 选中任意应用内的文本,一键翻译或发送给 AI(基于 macOS Accessibility,未授权时回退到临时复制并恢复剪贴板)
- **AI 对话模式** — 独立于翻译的多轮对话,支持自定义 system 提示词
- **自动免费翻译** — Google / 微软必应无需 key,健康服务轮询分流;超时、限流、空响应或服务异常时自动切换,故障服务冷却 30–300 秒后恢复尝试,长文本自动分块
@@ -92,7 +92,8 @@ src-tauri/src/ Rust 后端
├── window.rs 窗口显示/隐藏/居中/聚焦/固定
├── tray.rs 系统托盘菜单
├── screenshot.rs screencapture -i 区域截图
├── ocr/vision.rs Apple Vision OCR(VNRecognizeTextRequest)
├── ocr/image.rs ImageIO 解码 / 8-bit sRGB 规范化 / 小截图边距
├── ocr/vision.rs Apple Vision OCR(有效语言组合 / 旧模型回退)
├── selection/macos.rs 划词捕获(Accessibility + 剪贴板回退)
└── translate/
├── mod.rs 引擎分发 + 共享 HTTP 客户端
@@ -114,7 +115,7 @@ src/ Svelte 5 前端(runes)
```bash
# Rust
cd src-tauri && cargo test # 翻译解析 / SSE 解析 / 快捷键 / 划词归一化单测
cd src-tauri && cargo test # 翻译解析 / SSE / 快捷键 / 划词 / 真实 PNG Vision OCR
cd src-tauri && cargo clippy -- -D warnings
# 前端
@@ -122,10 +123,12 @@ npm test # Vitest(快捷键录制器)
npm run check # svelte-check 类型检查
```
macOS OCR 回归测试会通过 Xcode Command Line Tools 的 Swift 在临时目录生成样例,并真正执行 ImageIO 和 Vision;覆盖单行、灰度、16-bit、透明图片以及中英混排。
## ⚠️ 说明与限制
- 免费翻译自动在谷歌 / 微软必应之间分流和切换;两者都是**非官方免费端点**,同时不可用时会提示稍后重试。OpenAI 兼容接口需自备 API Key。
- OCR 保留 Apple 原生自由框选;截图期间临时隐藏主窗口,重复触发会被忽略。识别失败或无文字时用新 Vision 请求在 CPU 上重试,取消框选不会显示为识别失败。
- OCR 保留 Apple 原生自由框选;截图期间临时隐藏主窗口,重复触发会被忽略。中文识别只与英文组合;截图经 ImageIO 解码后绘制成 8-bit sRGB 位图,小截图加边距。识别失败或无文字时用旧版 Vision 模型在 CPU 上重试,取消框选不会显示为识别失败。
- API Key 存储在 macOS Keychain,不写入明文 Tauri store。
- 仅支持 **macOS 11.0+**(依赖 Vision、Accessibility、screencapture 等系统框架)。
- GitHub Release 是社区签名、非 Apple 公证包;首次安装需要在“隐私与安全性”中手动选择“仍要打开”。
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "ttree",
"version": "0.1.4",
"version": "0.1.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "ttree",
"version": "0.1.4",
"version": "0.1.5",
"license": "MIT",
"dependencies": {
"@tauri-apps/api": "^2",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "ttree",
"version": "0.1.4",
"version": "0.1.5",
"description": "轻量 macOS AI 翻译器",
"type": "module",
"scripts": {
+3 -1
View File
@@ -4463,15 +4463,17 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "ttree"
version = "0.1.4"
version = "0.1.5"
dependencies = [
"anyhow",
"core-foundation 0.10.1",
"futures-util",
"objc2",
"objc2-app-kit",
"objc2-core-foundation",
"objc2-core-graphics",
"objc2-foundation",
"objc2-image-io",
"objc2-vision",
"reqwest 0.12.28",
"security-framework",
+8 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "ttree"
version = "0.1.4"
version = "0.1.5"
description = "轻量 macOS AI 翻译器"
authors = ["you"]
edition = "2021"
@@ -37,14 +37,20 @@ core-foundation = "0.10"
# macOS Keychain —— 存放 API key,避免明文落盘
security-framework = "3"
objc2 = "0.6"
objc2-core-foundation = "0.3"
objc2-image-io = { version = "0.3", features = ["CGImageSource"] }
objc2-app-kit = { version = "0.3", features = [
"NSPasteboard",
"NSPasteboardItem",
] }
objc2-core-graphics = { version = "0.3", features = [
"CGBitmapContext",
"CGColorSpace",
"CGContext",
"CGEvent",
"CGEventTypes",
"CGRemoteOperation",
"CGImage",
] }
objc2-foundation = { version = "0.3", features = [
"NSString",
@@ -55,6 +61,7 @@ objc2-foundation = { version = "0.3", features = [
"NSObject",
] }
objc2-vision = { version = "0.3", features = [
"objc2-core-graphics",
"VNRequest",
"VNRecognizeTextRequest",
"VNObservation",
+111
View File
@@ -0,0 +1,111 @@
//! 将系统 PNG 解码并绘制成 Vision 可以稳定读取的 8-bit sRGB 位图。
#![cfg(target_os = "macos")]
use objc2_core_foundation::{CFData, CFRetained, CGPoint, CGRect, CGSize};
use objc2_core_graphics::{
kCGColorSpaceSRGB, CGBitmapContextCreate, CGBitmapContextCreateImage, CGBitmapInfo,
CGColorSpace, CGContext, CGImage, CGImageAlphaInfo,
};
use objc2_image_io::CGImageSource;
const MIN_DIMENSION: usize = 128;
const BORDER: usize = 16;
const MAX_PIXELS: usize = 32_000_000;
pub(super) fn prepare_png(bytes: &[u8]) -> Result<CFRetained<CGImage>, String> {
if !bytes.starts_with(b"\x89PNG\r\n\x1a\n") || bytes.len() < 24 {
return Err("截图不是有效的 PNG 图片,请重新框选。".to_string());
}
// 在原生解码前限制 IHDR 尺寸,避免异常文件触发巨量分配。
if &bytes[12..16] != b"IHDR" {
return Err("截图 PNG 缺少图片尺寸信息,请重新框选。".to_string());
}
let width = u32::from_be_bytes(bytes[16..20].try_into().unwrap()) as usize;
let height = u32::from_be_bytes(bytes[20..24].try_into().unwrap()) as usize;
canvas_size(width, height)?;
let data = CFData::from_bytes(bytes);
// SAFETY: 使用 ImageIO 默认选项,data 的内容在整个解码期间不变。
let source = unsafe { CGImageSource::with_data(&data, None) }
.ok_or_else(|| "系统无法读取截图 PNG,请重新框选。".to_string())?;
let image = unsafe { source.image_at_index(0, None) }
.ok_or_else(|| "截图 PNG 解码失败,请重新框选。".to_string())?;
let width = CGImage::width(Some(&image));
let height = CGImage::height(Some(&image));
let (canvas_width, canvas_height) = canvas_size(width, height)?;
// 显示器截图可能包含灰度、透明度、广色域或 16-bit 像素。统一为
// 8-bit sRGB、32-bit RGBX,并用白色背景消除透明通道的歧义。
let color_space = CGColorSpace::with_name(Some(unsafe { kCGColorSpaceSRGB }))
.ok_or_else(|| "创建 OCR 色彩空间失败".to_string())?;
let bitmap_info = CGImageAlphaInfo::NoneSkipLast.0 | CGBitmapInfo::ByteOrder32Big.bits();
// SAFETY: null 让 CoreGraphics 自行管理内存;尺寸已检查且每像素 4 字节。
let context = unsafe {
CGBitmapContextCreate(
std::ptr::null_mut(),
canvas_width,
canvas_height,
8,
canvas_width * 4,
Some(&color_space),
bitmap_info,
)
}
.ok_or_else(|| "创建 OCR 位图失败".to_string())?;
CGContext::set_rgb_fill_color(Some(&context), 1.0, 1.0, 1.0, 1.0);
CGContext::fill_rect(Some(&context), rect(0, 0, canvas_width, canvas_height));
// 保持原始像素大小;窄单行截图加边距,避免极小图像进入文字模型。
CGContext::draw_image(
Some(&context),
rect((canvas_width - width) / 2, (canvas_height - height) / 2, width, height),
Some(&image),
);
CGBitmapContextCreateImage(Some(&context))
.ok_or_else(|| "生成 OCR 位图失败".to_string())
}
fn canvas_size(width: usize, height: usize) -> Result<(usize, usize), String> {
if width == 0 || height == 0 {
return Err("截图尺寸为空,请重新框选。".to_string());
}
let padded = width
.checked_add(BORDER * 2)
.zip(height.checked_add(BORDER * 2))
.map(|(width, height)| (width.max(MIN_DIMENSION), height.max(MIN_DIMENSION)))
.filter(|(width, height)| {
width.checked_mul(*height).is_some_and(|pixels| pixels <= MAX_PIXELS)
});
padded.ok_or_else(|| "截图区域过大,请缩小框选范围后重试。".to_string())
}
fn rect(x: usize, y: usize, width: usize, height: usize) -> CGRect {
CGRect {
origin: CGPoint { x: x as _, y: y as _ },
size: CGSize { width: width as _, height: height as _ },
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pads_single_line_captures_without_stretching_text() {
assert_eq!(canvas_size(400, 20).unwrap(), (432, 128));
assert_eq!(canvas_size(1, 1).unwrap(), (128, 128));
}
#[test]
fn refuses_zero_or_excessive_dimensions() {
assert!(canvas_size(0, 20).is_err());
assert!(canvas_size(20, 0).is_err());
assert!(canvas_size(usize::MAX, 20).is_err());
assert!(canvas_size(10_000, 10_000).is_err());
}
#[test]
fn rejects_truncated_png_without_entering_vision() {
assert!(prepare_png(b"\x89PNG\r\n\x1a\n").is_err());
}
}
+6
View File
@@ -1,8 +1,14 @@
//! OCR 模块入口。
#[cfg(target_os = "macos")]
mod image;
#[cfg(target_os = "macos")]
pub mod vision;
#[cfg(all(test, target_os = "macos"))]
mod tests;
/// 对图片文件执行 OCR,返回识别文本。
#[cfg(target_os = "macos")]
pub fn recognize_file(path: &str) -> Result<String, String> {
+75
View File
@@ -0,0 +1,75 @@
//! 真正执行 ImageIO + Vision 的 macOS 回归测试,不能只检查 request 属性。
use std::path::Path;
use std::process::Command;
use std::sync::OnceLock;
use objc2::rc::autoreleasepool;
use objc2_core_graphics::CGImage;
use tempfile::TempDir;
use super::{image, vision};
fn fixtures() -> &'static Path {
static FIXTURES: OnceLock<TempDir> = OnceLock::new();
FIXTURES
.get_or_init(|| {
let directory = TempDir::new().unwrap();
let script = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("src/ocr/tests/generate-fixtures.swift");
let output = Command::new("/usr/bin/swift")
.arg(script)
.arg(directory.path())
.output()
.expect("macOS OCR tests require the Xcode Command Line Tools");
assert!(
output.status.success(),
"生成 OCR 样例失败: {}",
String::from_utf8_lossy(&output.stderr)
);
directory
})
.path()
}
fn fixture(name: &str) -> Vec<u8> {
std::fs::read(fixtures().join(format!("{name}.png"))).unwrap()
}
#[test]
fn native_ocr_reads_real_png_formats_and_single_line_captures() {
for name in ["english", "single-line", "grayscale", "grayscale-16bit", "transparent"] {
let text = vision::recognize_bytes(&fixture(name))
.unwrap_or_else(|error| panic!("{name}: {error}"));
assert!(text.contains("Hello OCR"), "{name}: {text:?}");
}
let text = vision::recognize_bytes(&fixture("chinese")).unwrap();
let compact: String = text.chars().filter(|ch| !ch.is_whitespace()).collect();
assert!(compact.contains("你好世界"), "chinese: {text:?}");
assert!(text.contains("Hello OCR"), "mixed Chinese/English: {text:?}");
if objc2::available!(macos = 13.0) {
let text = vision::recognize_bytes(&fixture("japanese")).unwrap();
assert!(text.contains("テキスト"), "japanese: {text:?}");
}
}
#[test]
fn imageio_decodes_16bit_png_and_normalizes_pixels_before_vision() {
autoreleasepool(|_| {
let bytes = fixture("grayscale-16bit");
assert_eq!(bytes[24], 16, "fixture must actually have 16-bit PNG samples");
let image = image::prepare_png(&bytes).unwrap();
assert_eq!(CGImage::bits_per_component(Some(&image)), 8);
assert_eq!(CGImage::bits_per_pixel(Some(&image)), 32);
assert!(CGImage::height(Some(&image)) >= 128);
});
}
#[test]
fn compatibility_retry_really_recognizes_text_on_cpu() {
autoreleasepool(|_| {
let image = image::prepare_png(&fixture("english")).unwrap();
let text = vision::recognize_once(&image, true).unwrap();
assert!(text.contains("Hello OCR"), "CPU retry: {text:?}");
});
}
@@ -0,0 +1,45 @@
// Deterministic OCR fixtures generated with macOS system fonts, entirely offline.
import AppKit
let directory = URL(fileURLWithPath: CommandLine.arguments[1], isDirectory: true)
func writeFixture(
_ name: String, text: String, width: Int = 440, height: Int = 80,
fontSize: CGFloat = 32, bits: Int = 8, grayscale: Bool = false,
transparent: Bool = false
) throws {
guard let bitmap = NSBitmapImageRep(
bitmapDataPlanes: nil, pixelsWide: width, pixelsHigh: height,
bitsPerSample: bits, samplesPerPixel: grayscale ? 1 : 4,
hasAlpha: !grayscale, isPlanar: false,
colorSpaceName: grayscale ? .deviceWhite : .deviceRGB,
bytesPerRow: 0, bitsPerPixel: 0
), let context = NSGraphicsContext(bitmapImageRep: bitmap) else {
fatalError("Could not create fixture bitmap: \(name)")
}
NSGraphicsContext.saveGraphicsState()
NSGraphicsContext.current = context
context.cgContext.setFillColor((transparent ? NSColor.clear : NSColor.white).cgColor)
context.cgContext.setBlendMode(.copy)
context.cgContext.fill(CGRect(x: 0, y: 0, width: CGFloat(width), height: CGFloat(height)))
context.cgContext.setBlendMode(.normal)
(text as NSString).draw(
at: NSPoint(x: 4, y: 2),
withAttributes: [.font: NSFont.systemFont(ofSize: fontSize),
.foregroundColor: NSColor.black]
)
context.flushGraphics()
NSGraphicsContext.restoreGraphicsState()
guard let png = bitmap.representation(using: .png, properties: [:]) else {
fatalError("Could not encode fixture PNG: \(name)")
}
try png.write(to: directory.appendingPathComponent(name + ".png"))
}
try writeFixture("english", text: "Hello OCR 123")
try writeFixture("single-line", text: "Hello OCR 123", width: 160, height: 22, fontSize: 14)
try writeFixture("grayscale", text: "Hello OCR 123", grayscale: true)
try writeFixture("grayscale-16bit", text: "Hello OCR 123", bits: 16, grayscale: true)
try writeFixture("transparent", text: "Hello OCR 123", transparent: true)
try writeFixture("chinese", text: "你好世界 Hello OCR")
try writeFixture("japanese", text: "日本語テキスト")
+31 -18
View File
@@ -1,13 +1,14 @@
//! macOS 原生 OCR —— 通过 Apple Vision 框架的 VNRecognizeTextRequest 识别图片文字。
//!
//! 仅 macOS。加载 PNG 为 NSData → VNImageRequestHandler → 同步 perform →
//! 仅 macOS。PNG → ImageIO 解码 → 8-bit RGB CGImage → Vision 同步 perform →
//! 读取 VNRecognizedTextObservation 的 topCandidates(1) 拼接。
#![cfg(target_os = "macos")]
use objc2::rc::{autoreleasepool, Retained};
use objc2::AnyThread;
use objc2_foundation::{NSArray, NSData, NSString};
use objc2_core_graphics::CGImage;
use objc2_foundation::{NSArray, NSString};
use objc2_vision::{VNImageRequestHandler, VNRecognizeTextRequest, VNRequestTextRecognitionLevel};
/// 对指定图片文件路径执行 OCR,返回识别文本(按行拼接)。
@@ -18,20 +19,18 @@ pub fn recognize_file(path: &str) -> Result<String, String> {
/// 对内存中的图片字节执行 OCR。
pub fn recognize_bytes(bytes: &[u8]) -> Result<String, String> {
if !bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
return Err("截图不是有效的 PNG 图片,请重新框选。".to_string());
}
// Tokio 阻塞线程没有 AppKit 事件循环的自动释放池;每次识别单独释放
// Vision 创建的临时原生对象,防止多次截图后内存和计算资源累积。
autoreleasepool(|_| {
let first_error = match recognize_once(bytes, false) {
let image = super::image::prepare_png(bytes)?;
let first_error = match recognize_once(&image, false) {
Ok(text) if !text.trim().is_empty() => return Ok(text),
Ok(_) => "未识别到文字".to_string(),
Err(error) => error,
};
// 复用同一截图,但创建全新的 request/handler。CPU 回退避开临时
// GPU/Metal 故障,关闭语言自动检测和纠错以覆盖模型判断失败。
match recognize_once(bytes, true) {
match recognize_once(&image, true) {
Ok(text) if !text.trim().is_empty() => Ok(text),
Ok(_) => Err("截图中未识别到文字,请框选清晰的文字区域后重试。".to_string()),
Err(error) => Err(format!("OCR 识别及兼容重试失败: {first_error}; {error}")),
@@ -39,19 +38,17 @@ pub fn recognize_bytes(bytes: &[u8]) -> Result<String, String> {
})
}
fn recognize_once(bytes: &[u8], compatibility: bool) -> Result<String, String> {
pub(super) fn recognize_once(image: &CGImage, compatibility: bool) -> Result<String, String> {
// SAFETY: Vision API 须在持有有效图像数据期间调用;以下对象均由我们创建并在本函数内存活。
unsafe {
let data = NSData::with_bytes(bytes);
// 创建并配置多语言文字识别请求。
let request: Retained<VNRecognizeTextRequest> = VNRecognizeTextRequest::new();
configure_request(&request, compatibility)?;
// 基于图像数据构建 handler 并同步执行
let handler = VNImageRequestHandler::initWithData_options(
// 直接提供已规范化的 CGImage,避免 Vision 再走压缩数据解码路径。
let handler = VNImageRequestHandler::initWithCGImage_options(
VNImageRequestHandler::alloc(),
&data,
image,
&objc2_foundation::NSDictionary::new(),
);
@@ -60,7 +57,14 @@ fn recognize_once(bytes: &[u8], compatibility: bool) -> Result<String, String> {
))]);
handler
.performRequests_error(&requests)
.map_err(|e| format!("OCR 执行失败: {e}"))?;
.map_err(|e| {
format!(
"OCR 执行失败: {e}(图像 {}×{},{} bit)",
CGImage::width(Some(image)),
CGImage::height(Some(image)),
CGImage::bits_per_component(Some(image)),
)
})?;
Ok(collect_text(&request))
}
@@ -71,11 +75,12 @@ fn configure_request(request: &VNRecognizeTextRequest, compatibility: bool) -> R
request.setUsesLanguageCorrection(!compatibility);
// 自动检测需要 revision 3,不能只开启属性而依赖系统默认 revision。
let revision = if objc2::available!(macos = 13.0) {
let revision = if !compatibility && objc2::available!(macos = 13.0) {
3
} else {
2
};
// 兼容重试使用 revision 2,避开新版识别模型与图像读取器的同一失败路径。
// SAFETY: revision 2 在最低支持版本 macOS 11 可用;revision 3 经版本判断。
unsafe {
request.setRevision(revision);
@@ -115,9 +120,9 @@ fn configure_request(request: &VNRecognizeTextRequest, compatibility: bool) -> R
}
fn preferred_languages(supported: &[String]) -> Vec<&str> {
[
"zh-Hans", "zh-Hant", "en-US", "ja-JP", "ko-KR", "fr-FR", "de-DE", "es-ES", "ru-RU",
]
// Apple 的语言支持查询只说明各语言能单独识别。中文模型只能与
// 英文配对,不能混入日语、韩语或欧洲语言;自动检测负责选择脚本。
["zh-Hans", "zh-Hant", "en-US"]
.into_iter()
.filter(|code| supported.iter().any(|supported| supported.as_str() == *code))
.collect()
@@ -180,6 +185,7 @@ mod tests {
let request = VNRecognizeTextRequest::new();
configure_request(&request, true).unwrap();
assert!(!request.usesLanguageCorrection());
assert_eq!(unsafe { request.revision() }, 2);
if objc2::available!(macos = 13.0) {
assert!(!request.automaticallyDetectsLanguage());
}
@@ -192,6 +198,13 @@ mod tests {
assert!(preferred_languages(&[]).is_empty());
}
#[test]
fn supported_languages_do_not_form_an_invalid_chinese_combination() {
let supported = ["zh-Hans", "zh-Hant", "en-US", "ja-JP", "ko-KR", "fr-FR"]
.map(String::from);
assert_eq!(preferred_languages(&supported), ["zh-Hans", "zh-Hant", "en-US"]);
}
#[test]
fn rejects_invalid_image_data_before_starting_vision() {
assert!(recognize_bytes(&[]).unwrap_err().contains("PNG"));
+1 -1
View File
@@ -1,7 +1,7 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "TTREE",
"version": "0.1.4",
"version": "0.1.5",
"identifier": "com.samwstu.ttree",
"build": {
"beforeDevCommand": "npm run dev",