fix(health-check): record Gemini thought-only responses as successful usage

This commit is contained in:
lzt404 committed 2026-08-14 18:43:12 +08:00
1 parent ee41ac4922
commit 18c9546b51
12 files changed
+285 -21

No files matched your search

+1 -1
View File
@@ -593,7 +593,7 @@ checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636"
[[package]]
name = "cpa-gui"
version = "0.2.21"
version = "0.2.22"
dependencies = [
"base64 0.22.1",
"chrono",
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "cpa-gui"
# Single source of truth for the desktop app version.
version = "0.2.21"
version = "0.2.22"
description = "CPA desktop management shell"
authors = ["lzt404"]
edition = "2021"
+4 -1
View File
@@ -24,7 +24,10 @@ use management_api::{
};
use oauth_browser::*;
#[cfg(test)]
use provider_health::{provider_health_content_type_is_streaming, provider_health_stream_has_text};
use provider_health::{
provider_health_content_type_is_streaming, provider_health_stream_has_terminal_success,
provider_health_stream_has_text,
};
use agents::*;
use app_settings::*;
+177 -5
View File
@@ -1,4 +1,7 @@
use super::{build_http_client_with_proxy, is_loopback_host, GuiConfigState, APP_USER_AGENT};
use super::{
build_http_client_with_proxy, is_loopback_host, usage, GuiConfigState, APP_USER_AGENT,
};
use chrono::Local;
use futures_util::StreamExt;
use serde::{Deserialize, Serialize};
use std::{
@@ -16,12 +19,29 @@ pub(crate) struct ProviderHealthProbeRequest {
data: String,
protocol: String,
timeout_ms: Option<u64>,
#[serde(default)]
model: String,
#[serde(default)]
source: String,
#[serde(default)]
auth_index: String,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ProviderHealthProbeResponse {
first_token_latency_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
first_token_latency_ms: Option<u64>,
response_latency_ms: u64,
}
#[derive(Default)]
struct ProviderHealthUsageTokens {
input_tokens: u64,
output_tokens: u64,
reasoning_tokens: u64,
cache_read_tokens: u64,
total_tokens: u64,
}
fn provider_health_value_has_text(value: Option<&serde_json::Value>) -> bool {
@@ -122,6 +142,139 @@ pub(crate) fn provider_health_stream_has_text(protocol: &str, bytes: &[u8]) -> b
})
}
fn provider_health_json_has_terminal_success(protocol: &str, value: &serde_json::Value) -> bool {
if protocol != "gemini" {
return false;
}
let exhausted_thinking_budget = value
.get("candidates")
.and_then(serde_json::Value::as_array)
.is_some_and(|candidates| {
candidates.iter().any(|candidate| {
candidate
.get("finishReason")
.and_then(serde_json::Value::as_str)
== Some("MAX_TOKENS")
})
});
let thoughts = value
.pointer("/usageMetadata/thoughtsTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default();
let total = value
.pointer("/usageMetadata/totalTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default();
exhausted_thinking_budget && thoughts > 0 && total >= thoughts
}
pub(crate) fn provider_health_stream_has_terminal_success(protocol: &str, bytes: &[u8]) -> bool {
let text = String::from_utf8_lossy(bytes);
text.lines().any(|line| {
let line = line.trim();
let data = line.strip_prefix("data:").map(str::trim).unwrap_or(line);
if data.is_empty() || data == "[DONE]" {
return false;
}
serde_json::from_str::<serde_json::Value>(data)
.ok()
.is_some_and(|value| provider_health_json_has_terminal_success(protocol, &value))
})
}
fn provider_health_usage_tokens(protocol: &str, bytes: &[u8]) -> ProviderHealthUsageTokens {
let mut tokens = ProviderHealthUsageTokens::default();
if protocol != "gemini" {
return tokens;
}
let text = String::from_utf8_lossy(bytes);
for line in text.lines() {
let line = line.trim();
let data = line.strip_prefix("data:").map(str::trim).unwrap_or(line);
let Some(usage) = serde_json::from_str::<serde_json::Value>(data)
.ok()
.and_then(|value| value.get("usageMetadata").cloned())
else {
continue;
};
tokens.input_tokens = tokens.input_tokens.max(
usage
.get("promptTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default(),
);
tokens.output_tokens = tokens.output_tokens.max(
usage
.get("candidatesTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default(),
);
tokens.reasoning_tokens = tokens.reasoning_tokens.max(
usage
.get("thoughtsTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default(),
);
tokens.cache_read_tokens = tokens.cache_read_tokens.max(
usage
.get("cachedContentTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default(),
);
tokens.total_tokens = tokens.total_tokens.max(
usage
.get("totalTokenCount")
.and_then(serde_json::Value::as_u64)
.unwrap_or_default(),
);
}
tokens
}
fn provider_health_usage_provider(protocol: &str) -> &str {
match protocol {
"openai-responses" => "codex",
"openai-chat" => "openai",
"claude" => "claude",
"gemini" => "gemini",
_ => "unknown",
}
}
fn persist_provider_health_success(
app: &tauri::AppHandle,
request: &ProviderHealthProbeRequest,
endpoint: &str,
latency_ms: u64,
ttft_ms: Option<u64>,
received: &[u8],
) {
let tokens = provider_health_usage_tokens(&request.protocol, received);
let event = serde_json::json!({
"timestamp": Local::now().to_rfc3339(),
"latency_ms": latency_ms,
"ttft_ms": ttft_ms,
"source": request.source.as_str(),
"auth_index": request.auth_index.as_str(),
"failed": false,
"provider": provider_health_usage_provider(&request.protocol),
"model": request.model.as_str(),
"executor_type": "DesktopProviderHealthCheck",
"endpoint": endpoint,
"generate": false,
"tokens": {
"input_tokens": tokens.input_tokens,
"output_tokens": tokens.output_tokens,
"reasoning_tokens": tokens.reasoning_tokens,
"cache_read_tokens": tokens.cache_read_tokens,
"total_tokens": tokens.total_tokens,
},
});
if let Err(error) = usage::persist_local_usage_event(app, "desktop_health_check", event) {
eprintln!("保存桌面健康检测使用记录失败: {error}");
}
}
pub(crate) fn provider_health_content_type_is_streaming(content_type: &str) -> bool {
let content_type = content_type.to_ascii_lowercase();
content_type.contains("text/event-stream")
@@ -131,6 +284,7 @@ pub(crate) fn provider_health_content_type_is_streaming(content_type: &str) -> b
#[tauri::command]
pub(crate) async fn provider_health_probe(
app: tauri::AppHandle,
gui_config_state: tauri::State<'_, GuiConfigState>,
request: ProviderHealthProbeRequest,
) -> Result<ProviderHealthProbeResponse, String> {
@@ -145,6 +299,7 @@ pub(crate) async fn provider_health_probe(
) {
return Err("不支持的健康检测协议".to_string());
}
let endpoint = format!("POST {}", url.path());
if request.data.len() > 64 * 1024 {
return Err("健康检测请求体过大".to_string());
}
@@ -164,7 +319,7 @@ pub(crate) async fn provider_health_probe(
"创建健康检测客户端失败",
)?;
let mut headers = reqwest::header::HeaderMap::new();
for (name, value) in request.header {
for (name, value) in &request.header {
let name = reqwest::header::HeaderName::from_bytes(name.as_bytes())
.map_err(|error| format!("健康检测请求头名称无效: {error}"))?;
let value = reqwest::header::HeaderValue::from_str(&value)
@@ -182,7 +337,7 @@ pub(crate) async fn provider_health_probe(
let response = client
.post(url)
.headers(headers)
.body(request.data)
.body(request.data.clone())
.send()
.await
.map_err(|error| format!("健康检测请求失败: {error}"))?;
@@ -214,9 +369,26 @@ pub(crate) async fn provider_health_probe(
return Err("健康检测在限制范围内未收到模型首字".to_string());
}
received.extend_from_slice(&chunk);
let elapsed_ms = started_at.elapsed().as_millis().max(1) as u64;
if provider_health_stream_has_text(&request.protocol, &received) {
persist_provider_health_success(
&app,
&request,
&endpoint,
elapsed_ms,
Some(elapsed_ms),
&received,
);
return Ok(ProviderHealthProbeResponse {
first_token_latency_ms: started_at.elapsed().as_millis().max(1) as u64,
first_token_latency_ms: Some(elapsed_ms),
response_latency_ms: elapsed_ms,
});
}
if provider_health_stream_has_terminal_success(&request.protocol, &received) {
persist_provider_health_success(&app, &request, &endpoint, elapsed_ms, None, &received);
return Ok(ProviderHealthProbeResponse {
first_token_latency_ms: None,
response_latency_ms: elapsed_ms,
});
}
}
+33
View File
@@ -63,4 +63,37 @@ data: {"delta":{"type":"text_delta","text":"H"}}
"#;
assert!(provider_health_stream_has_text("gemini", gemini_text));
let gemini_thought_only = br#"data: {"candidates":[{"content":{},"finishReason":"MAX_TOKENS","index":0}],"usageMetadata":{"promptTokenCount":1,"totalTokenCount":14,"thoughtsTokenCount":13},"modelVersion":"gemini-3.7-flash"}
"#;
assert!(!provider_health_stream_has_text(
"gemini",
gemini_thought_only
));
assert!(provider_health_stream_has_terminal_success(
"gemini",
gemini_thought_only
));
let gemini_empty_max_tokens =
br#"data: {"candidates":[{"finishReason":"MAX_TOKENS"}],"usageMetadata":{"totalTokenCount":1,"thoughtsTokenCount":0}}
"#;
assert!(!provider_health_stream_has_terminal_success(
"gemini",
gemini_empty_max_tokens
));
let gemini_safety = br#"data: {"candidates":[{"finishReason":"SAFETY"}],"usageMetadata":{"totalTokenCount":14,"thoughtsTokenCount":13}}
"#;
assert!(!provider_health_stream_has_terminal_success(
"gemini",
gemini_safety
));
assert!(!provider_health_stream_has_terminal_success(
"openai-chat",
gemini_thought_only
));
}
+12
View File
@@ -1106,6 +1106,18 @@ fn publish_collected_records(app: &tauri::AppHandle, saved: usize, message: &str
}
}
pub(crate) fn persist_local_usage_event(
app: &tauri::AppHandle,
collector_source: &str,
value: Value,
) -> Result<usize, String> {
let config = app.state::<GuiConfigState>().snapshot()?;
let root = usage_root_dir()?;
let saved = persist_queue_items_from_source(&root, collector_source, vec![value], &config)?;
publish_collected_records(app, saved, "已保存桌面健康检测使用记录");
Ok(saved)
}
async fn wait_or_cancel(token: &CancellationToken, seconds: u64) {
tokio::select! {
_ = token.cancelled() => {},
+3 -1
View File
@@ -677,7 +677,7 @@ export const jaOverrides = {
'apiAccess.health.action': 'ヘルスチェック',
'apiAccess.health.title': 'モデルヘルスチェック',
'apiAccess.health.description': '{provider}',
'apiAccess.health.warning': '各モデルへ最小限のストリーミングリクエストを 1 回送信し、最初の出力文字を受信した時点で停止します。少額の料金が発生する場合があります。',
'apiAccess.health.warning': '各モデルへ最小限のストリーミングリクエストを 1 回送信し、最初の出力文字または検証可能な正常終了応答を受信した時点で停止します。少額の料金が発生する場合があります。',
'apiAccess.health.search': 'モデル名またはエイリアスを検索',
'apiAccess.health.loadingModels': 'モデル一覧を読み込んでいます…',
'apiAccess.health.modelLoadFailed': 'モデル一覧を更新できませんでした:{error}',
@@ -696,6 +696,8 @@ export const jaOverrides = {
'apiAccess.health.missingDirectKey': '初字遅延を測定するには、直接使用できる平文の API キーが必要です',
'apiAccess.health.firstTokenInline': '初字 {latency} ms',
'apiAccess.health.firstTokenLatencyResult': '初字遅延:{latency} ms',
'apiAccess.health.responseInline': '応答 {latency} ms',
'apiAccess.health.responseLatencyResult': '応答遅延:{latency} ms',
'common.edit': '編集',
'apiAccess.error.enterBaseUrl': '先に Base URL を入力してください',
'apiAccess.error.noAvailableModels': '利用可能なモデルが見つかりません',
+3 -1
View File
@@ -677,7 +677,7 @@ export const en: Record<MessageKey, string> = {
'apiAccess.health.action': 'Health Check',
'apiAccess.health.title': 'Model Health Check',
'apiAccess.health.description': '{provider}',
'apiAccess.health.warning': 'The check sends one minimal streaming request to each model, stops at the first output token, and may incur a small charge.',
'apiAccess.health.warning': 'The check sends one minimal streaming request to each model, stops at the first output token or a verifiable successful terminal response, and may incur a small charge.',
'apiAccess.health.search': 'Search model names or aliases',
'apiAccess.health.loadingModels': 'Loading the model list…',
'apiAccess.health.modelLoadFailed': 'Could not refresh the model list: {error}',
@@ -696,6 +696,8 @@ export const en: Record<MessageKey, string> = {
'apiAccess.health.missingDirectKey': 'A directly usable plaintext API key is required to measure first-token latency',
'apiAccess.health.firstTokenInline': 'TTFT {latency} ms',
'apiAccess.health.firstTokenLatencyResult': 'Time to first token: {latency} ms',
'apiAccess.health.responseInline': 'Response {latency} ms',
'apiAccess.health.responseLatencyResult': 'Response latency: {latency} ms',
'common.edit': 'Edit',
'apiAccess.error.enterBaseUrl': 'Enter a Base URL first',
'apiAccess.error.noAvailableModels': 'No available models found',
+3 -1
View File
@@ -675,7 +675,7 @@ export const zhCN = {
'apiAccess.health.action': '健康检测',
'apiAccess.health.title': '模型健康检测',
'apiAccess.health.description': '{provider}',
'apiAccess.health.warning': '检测会对每个模型发送一次最小流式请求,收到模型首字后立即停止,并可能产生少量费用。',
'apiAccess.health.warning': '检测会对每个模型发送一次最小流式请求,收到模型首字或可验证的正常终止响应后立即停止,并可能产生少量费用。',
'apiAccess.health.search': '搜索模型名称或别名',
'apiAccess.health.loadingModels': '正在获取模型列表…',
'apiAccess.health.modelLoadFailed': '模型列表刷新失败:{error}',
@@ -694,6 +694,8 @@ export const zhCN = {
'apiAccess.health.missingDirectKey': '缺少可直接使用的明文 API 密钥,无法测量首字延迟',
'apiAccess.health.firstTokenInline': '首字 {latency} ms',
'apiAccess.health.firstTokenLatencyResult': '首字延迟:{latency} ms',
'apiAccess.health.responseInline': '响应 {latency} ms',
'apiAccess.health.responseLatencyResult': '响应延迟:{latency} ms',
'common.edit': '编辑',
'apiAccess.error.enterBaseUrl': '请先填写 Base URL',
'apiAccess.error.noAvailableModels': '未发现可用模型',
+12 -4
View File
@@ -1467,6 +1467,16 @@ function ProviderHealthDialog({ row, onClose }: ProviderHealthDialogProps) {
? t('apiAccess.health.timeout')
: checked.error || t('apiAccess.health.failed')
: '';
const latencyTitle = checked?.firstTokenLatencyMs !== undefined
? t('apiAccess.health.firstTokenLatencyResult', { latency: checked.firstTokenLatencyMs })
: checked?.responseLatencyMs !== undefined
? t('apiAccess.health.responseLatencyResult', { latency: checked.responseLatencyMs })
: error || undefined;
const latencyInline = checked?.firstTokenLatencyMs !== undefined
? t('apiAccess.health.firstTokenInline', { latency: checked.firstTokenLatencyMs })
: checked?.responseLatencyMs !== undefined
? t('apiAccess.health.responseInline', { latency: checked.responseLatencyMs })
: '';
return (
<div className={`provider-health-model-row ${checked?.status === 'failed' ? 'failed' : ''}`} key={model.name}>
<div className="provider-health-model-name">
@@ -1479,12 +1489,10 @@ function ProviderHealthDialog({ row, onClose }: ProviderHealthDialogProps) {
</div>
<span
className={`state-pill provider-health-pill ${state?.status === 'checking' ? 'checking' : checked?.status === 'healthy' ? 'success' : checked?.status === 'failed' ? 'error' : ''}`}
title={checked?.firstTokenLatencyMs === undefined ? error || undefined : t('apiAccess.health.firstTokenLatencyResult', { latency: checked.firstTokenLatencyMs })}
title={latencyTitle}
>
{statusLabel(state)}
{checked?.firstTokenLatencyMs === undefined
? ''
: ` · ${t('apiAccess.health.firstTokenInline', { latency: checked.firstTokenLatencyMs })}`}
{latencyInline ? ` · ${latencyInline}` : ''}
</span>
<button
type="button"
+34 -6
View File
@@ -9,11 +9,15 @@ export type ProviderHealthProbe = {
header: Record<string, string>;
data: string;
protocol: 'openai-chat' | 'openai-responses' | 'claude' | 'gemini';
model: string;
source: string;
authIndex: string;
};
export type ProviderHealthProbeResult = {
success: boolean;
firstTokenLatencyMs?: number;
responseLatencyMs?: number;
error?: string;
timedOut?: boolean;
errorCode?: 'missing-direct-key';
@@ -101,13 +105,21 @@ export function buildProviderHealthProbe(
const root = endpointRoot(provider, baseUrl);
const headers = { ...customHeaders };
const key = apiKey.trim();
const normalizedModel = provider === 'gemini'
? model.trim().replace(/^models\//i, '')
: model.trim();
const metadata = {
model: normalizedModel,
source: key,
authIndex: authIndex.trim(),
};
setHeaderIfMissing(headers, 'Content-Type', 'application/json');
if (provider === 'gemini') {
if (key) setHeaderIfMissing(headers, 'x-goog-api-key', key);
else if (authIndex) setHeaderIfMissing(headers, 'x-goog-api-key', '$TOKEN$');
const normalizedModel = model.trim().replace(/^models\//i, '');
return {
...metadata,
url: `${root}/v1beta/models/${encodeURIComponent(normalizedModel)}:generateContent?alt=sse`,
header: headers,
protocol: 'gemini',
@@ -125,11 +137,12 @@ export function buildProviderHealthProbe(
else if (authIndex) setHeaderIfMissing(headers, 'x-api-key', '$TOKEN$');
setHeaderIfMissing(headers, 'anthropic-version', '2023-06-01');
return {
...metadata,
url: `${root}/v1/messages`,
header: headers,
protocol: 'claude',
data: JSON.stringify({
model: model.trim(),
model: normalizedModel,
max_tokens: 16,
stream: true,
messages: [{ role: 'user', content: 'hi' }],
@@ -142,11 +155,12 @@ export function buildProviderHealthProbe(
if (provider === 'codex') {
return {
...metadata,
url: `${root}/v1/responses`,
header: headers,
protocol: 'openai-responses',
data: JSON.stringify({
model: model.trim(),
model: normalizedModel,
input: 'hi',
stream: true,
}),
@@ -154,11 +168,12 @@ export function buildProviderHealthProbe(
}
return {
...metadata,
url: openAIChatCompletionsEndpoint(baseUrl),
header: headers,
protocol: 'openai-chat',
data: JSON.stringify({
model: model.trim(),
model: normalizedModel,
messages: [{ role: 'user', content: 'hi' }],
stream: true,
}),
@@ -197,18 +212,31 @@ export async function checkProviderHealthProbe(
errorCode: 'missing-direct-key',
};
}
const response = await invoke<{ firstTokenLatencyMs: number }>('provider_health_probe', {
const response = await invoke<{
firstTokenLatencyMs?: number;
responseLatencyMs: number;
}>('provider_health_probe', {
request: {
protocol: probe.protocol,
timeoutMs,
data: probe.data,
header: probe.header,
url: probe.url,
model: probe.model,
source: probe.source,
authIndex: probe.authIndex,
},
});
const firstTokenLatencyMs = Number.isFinite(response.firstTokenLatencyMs)
? Math.max(1, Math.round(response.firstTokenLatencyMs as number))
: undefined;
const responseLatencyMs = Number.isFinite(response.responseLatencyMs)
? Math.max(1, Math.round(response.responseLatencyMs))
: firstTokenLatencyMs;
return {
success: true,
firstTokenLatencyMs: Math.max(1, Math.round(response.firstTokenLatencyMs)),
...(firstTokenLatencyMs === undefined ? {} : { firstTokenLatencyMs }),
...(responseLatencyMs === undefined ? {} : { responseLatencyMs }),
};
} catch (error) {
const message = errorMessage(error);
+2
View File
@@ -154,6 +154,8 @@ describe('API 接入健康检测', () => {
);
expect(probe.header['x-goog-api-key']).toBe('gemini-key');
expect(probe.protocol).toBe('gemini');
expect(probe.model).toBe('gemini-2.5-flash');
expect(probe.source).toBe('gemini-key');
});
it('检测全部时每个模型只请求一次并保持结果顺序', async () => {