feat: improve model health and usage summary

.gitignore

@@ -7,5 +7,5 @@ apps/ui/dist/
 apps/worker/.wrangler/
 tests/
 
-helloagents/plan
-helloagents/archive
+.helloagents/plan
+.helloagents/archive

.helloagents/CHANGELOG.md

@@ -20,6 +20,15 @@
 - **[worker/models]**: 引入通道模型能力表并基于“模型广场测试结果+可配置TTL（默认2小时）”进行模型发布与路由匹配
   - 类型: 微调（无方案包）
   - 文件: apps/worker/migrations/0004_add_channel_model_capabilities.sql, apps/worker/src/services/channel-model-capabilities.ts, apps/worker/src/services/channel-testing.ts, apps/worker/src/services/settings.ts, apps/worker/src/routes/settings.ts, apps/worker/src/routes/models.ts, apps/worker/src/routes/newapiUsers.ts, apps/worker/src/routes/proxy.ts, tests/worker/channel-model-capabilities.test.ts, tests/worker/newapi.test.ts, helloagents/modules/settings.md, helloagents/modules/models.md, helloagents/modules/proxy.md
+- **[worker/models]**: 运行时失败冷却与成功刷新能力表，TTL 过期回退模型列表
+  - 方案: [202603141718_model-health](archive/2026-03/202603141718_model-health/)
+  - 文件: apps/worker/src/routes/proxy.ts, apps/worker/src/services/channel-model-capabilities.ts, apps/worker/src/routes/models.ts, apps/worker/src/routes/newapiUsers.ts, apps/worker/src/services/settings.ts, apps/worker/src/routes/settings.ts, tests/worker/settings.test.ts, tests/worker/channel-model-capabilities.test.ts, .helloagents/modules/proxy.md, .helloagents/modules/models.md, .helloagents/modules/settings.md
+- **[proxy]**: 失败判定简化为非 2xx/超时即失败
+  - 方案: [202603141749_usage-summary](archive/2026-03/202603141749_usage-summary/)
+  - 文件: apps/worker/src/routes/proxy.ts
+- **[admin-ui/usage]**: 使用日志新增概述卡片（总量/成功/失败/错误率/平均延迟）
+  - 方案: [202603141749_usage-summary](archive/2026-03/202603141749_usage-summary/)
+  - 文件: apps/ui/src/core/utils.ts, apps/ui/src/features/UsageView.tsx, tests/ui/usage.test.ts, .helloagents/modules/usage.md, .helloagents/modules/admin-ui.md
 - **[worker/proxy]**: 移除临时调试日志（请求入口/出口与上游响应汇总）
   - 类型: 微调（无方案包）
   - 文件: apps/worker/src/index.ts, apps/worker/src/routes/proxy.ts

.helloagents/modules/admin-ui.md

@@ -25,6 +25,7 @@
 - done-hub 站点仅使用 base_url，不再维护多地址输入
 - 站点连通测试提示令牌成功/失败统计
 - 使用日志支持本地分页（默认 50 条）与固定高度滚动，并展示令牌、输入/输出 tokens、首 token 延迟、流式与推理强度
+- 使用日志顶部展示概述（总量/成功/失败/错误率/平均延迟）
 - 令牌管理采用列表视图（默认每页 10 条），生成令牌使用弹窗表单，并展示创建时间与前缀
 - 站点管理内置一键签到入口，展示今日签到结果（基于后台写回的 `last_checkin_*`）
 - 系统设置增加定时签到开关与时间（中国时间）

.helloagents/modules/models.md

@@ -11,7 +11,7 @@
 ## 行为规范
 - 仅聚合启用状态的渠道
 - 使用“模型广场测试结果”作为可用模型来源
-- 模型结果带 TTL（默认 2 小时，来自 settings），过期后不再返回
+- 模型结果带 TTL（默认 2 小时，来自 settings），过期后回退到渠道声明模型列表（避免模型广场清空）
 - 模型归一化支持字符串数组与对象数组（对象取 `id` 字段）
 
 ## 依赖关系

.helloagents/modules/proxy.md

@@ -27,6 +27,8 @@
 - 允许 `metadata_json.header_override` 与 `metadata_json.query_override` 注入额外上游请求头/查询参数
 - Anthropic 默认注入 `anthropic-version=2023-06-01`，可通过 `header_override` 覆盖
 - 路由时优先使用模型能力表（TTL 内测试成功的模型），无能力记录时回退渠道声明模型列表
+- 上游失败/超时会记录到能力表并进入冷却期，冷却期内跳过对应渠道/模型
+- 上游成功响应会刷新能力表，清空错误并延长可用窗口
 
 ## 依赖关系
 - `channels` / `tokens` / `usage_logs` 表

.helloagents/modules/settings.md

@@ -5,6 +5,7 @@
 - 为使用日志与面板提供统一配置读取
 - 维护定时签到执行时间
 - 维护模型能力 TTL（模型广场与路由过滤窗口）
+- 维护模型失败冷却时间（运行时失败的渠道/模型冷却窗口）
 
 ## 接口定义
 - `GET /api/settings` 读取设置
@@ -17,6 +18,7 @@
 - 更新签到时间后会触发 `CheckinScheduler` 重新调度
 - 若更新后当前时间已超过新签到时间，当日可再次触发自动签到
 - 模型能力 TTL 以小时计（默认 2 小时），用于控制“模型广场测试结果”的有效期
+- 模型失败冷却以分钟计（默认 10 分钟），用于跳过最近失败的渠道/模型
 
 ## 依赖关系
 - `settings` 表

.helloagents/modules/usage.md

@@ -14,6 +14,7 @@
 - 查询结果附带渠道与令牌名称
 - usage_logs 记录输入/输出 tokens、首 token 延迟、流式标记与推理强度
 - 推理强度来自请求体 `reasoning` / `reasoning_effort` 字段
+- 前端使用日志列表计算概述（总量/成功/失败/错误率/平均延迟）
 
 ## 依赖关系
 - `usage_logs` 表

apps/ui/src/core/utils.ts

@@ -1,3 +1,5 @@
+import type { UsageLog } from "./types";
+
 /**
  * Formats a datetime string for display.
  *
@@ -76,3 +78,41 @@ export const buildPageItems = (current: number, total: number): PageItem[] => {
 		return true;
 	});
 };
+
+export type UsageSummary = {
+	total: number;
+	success: number;
+	failed: number;
+	errorRate: number;
+	avgLatencyMs: number | null;
+	totalTokens: number;
+};
+
+export const summarizeUsageLogs = (logs: UsageLog[]): UsageSummary => {
+	const total = logs.length;
+	const success = logs.filter((log) => log.status === "ok").length;
+	const failed = total - success;
+	let latencySum = 0;
+	let latencyCount = 0;
+	let totalTokens = 0;
+	for (const log of logs) {
+		const latency = log.latency_ms;
+		if (latency !== null && latency !== undefined && !Number.isNaN(latency)) {
+			latencySum += latency;
+			latencyCount += 1;
+		}
+		if (log.total_tokens !== null && log.total_tokens !== undefined) {
+			totalTokens += log.total_tokens;
+		} else {
+			totalTokens += (log.prompt_tokens ?? 0) + (log.completion_tokens ?? 0);
+		}
+	}
+	return {
+		total,
+		success,
+		failed,
+		errorRate: total > 0 ? (failed / total) * 100 : 0,
+		avgLatencyMs: latencyCount > 0 ? latencySum / latencyCount : null,
+		totalTokens,
+	};
+};

apps/ui/src/features/UsageView.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useMemo, useState } from "hono/jsx/dom";
 import type { UsageLog } from "../core/types";
-import { buildPageItems, formatDateTime } from "../core/utils";
+import { buildPageItems, formatDateTime, summarizeUsageLogs } from "../core/utils";
 
 type UsageViewProps = {
 	usage: UsageLog[];
@@ -42,6 +42,7 @@ export const UsageView = ({ usage, onRefresh }: UsageViewProps) => {
 	const [pageSize, setPageSize] = useState(50);
 	const [page, setPage] = useState(1);
 	const total = usage.length;
+	const summary = useMemo(() => summarizeUsageLogs(usage), [usage]);
 	const totalPages = useMemo(
 		() => Math.max(1, Math.ceil(total / pageSize)),
 		[total, pageSize],
@@ -85,6 +86,31 @@ export const UsageView = ({ usage, onRefresh }: UsageViewProps) => {
 					</button>
 				</div>
 			</div>
+			<div class="mt-4 rounded-xl border border-stone-200 bg-stone-50 px-4 py-3">
+				<div class="flex flex-wrap items-center gap-3 text-xs text-stone-600">
+					<span class="font-semibold text-stone-700">
+						概述（当前展示 {summary.total} 条）
+					</span>
+					<span class="rounded-full bg-emerald-100 px-3 py-1 font-semibold text-emerald-700">
+						成功 {summary.success}
+					</span>
+					<span class="rounded-full bg-rose-100 px-3 py-1 font-semibold text-rose-700">
+						失败 {summary.failed}
+					</span>
+					<span class="rounded-full bg-stone-200 px-3 py-1 font-semibold text-stone-700">
+						错误率 {summary.total === 0 ? "-" : `${summary.errorRate.toFixed(1)}%`}
+					</span>
+					<span class="rounded-full bg-stone-200 px-3 py-1 font-semibold text-stone-700">
+						平均延迟{" "}
+						{summary.avgLatencyMs === null
+							? "-"
+							: `${summary.avgLatencyMs.toFixed(0)} ms`}
+					</span>
+					<span class="rounded-full bg-stone-200 px-3 py-1 font-semibold text-stone-700">
+						总 Tokens {summary.totalTokens}
+					</span>
+				</div>
+			</div>
 			<div class="mt-4 overflow-hidden rounded-xl border border-stone-200">
 				<div class="h-[420px] overflow-auto sm:h-[520px]">
 					<table class="min-w-[960px] w-full border-collapse text-xs sm:text-sm">

apps/worker/src/routes/models.ts

@@ -1,6 +1,6 @@
 import { Hono } from "hono";
 import type { AppEnv } from "../env";
-import { listVerifiedModelEntries } from "../services/channel-model-capabilities";
+import { listModelEntriesWithFallback } from "../services/channel-model-capabilities";
 import { listActiveChannels } from "../services/channel-repo";
 import { getModelCapabilityTtlHours } from "../services/settings";
 
@@ -13,9 +13,13 @@ models.get("/", async (c) => {
 	const channels = await listActiveChannels(c.env.DB);
 	const ttlHours = await getModelCapabilityTtlHours(c.env.DB);
 	const ttlSeconds = Math.max(1, Math.floor(ttlHours)) * 60 * 60;
-	const entries = await listVerifiedModelEntries(
+	const entries = await listModelEntriesWithFallback(
 		c.env.DB,
-		channels.map((channel) => ({ id: channel.id, name: channel.name })),
+		channels.map((channel) => ({
+			id: channel.id,
+			name: channel.name,
+			models_json: channel.models_json,
+		})),
 		ttlSeconds,
 	);
 

apps/worker/src/routes/newapiUsers.ts

@@ -2,7 +2,7 @@ import { Hono } from "hono";
 import type { AppEnv } from "../env";
 import { newApiAuth } from "../middleware/newApiAuth";
 import {
-	listVerifiedModelsByChannel,
+	listModelsByChannelWithFallback,
 } from "../services/channel-model-capabilities";
 import { listActiveChannels } from "../services/channel-repo";
 import { getModelCapabilityTtlHours } from "../services/settings";
@@ -15,9 +15,13 @@ users.get("/models", async (c) => {
 	const channels = await listActiveChannels(c.env.DB);
 	const ttlHours = await getModelCapabilityTtlHours(c.env.DB);
 	const ttlSeconds = Math.max(1, Math.floor(ttlHours)) * 60 * 60;
-	const map = await listVerifiedModelsByChannel(
+	const map = await listModelsByChannelWithFallback(
 		c.env.DB,
-		channels.map((channel) => channel.id),
+		channels.map((channel) => ({
+			id: channel.id,
+			name: channel.name,
+			models_json: channel.models_json,
+		})),
 		ttlSeconds,
 	);
 	const modelSet = new Set<string>();

apps/worker/src/routes/proxy.ts

@@ -17,8 +17,16 @@ import {
 	createWeightedOrder,
 	extractModels,
 } from "../services/channels";
-import { listVerifiedModelsByChannel } from "../services/channel-model-capabilities";
-import { getModelCapabilityTtlHours } from "../services/settings";
+import {
+	listCoolingDownChannelsForModel,
+	listVerifiedModelsByChannel,
+	recordChannelModelError,
+	upsertChannelModelCapabilities,
+} from "../services/channel-model-capabilities";
+import {
+	getModelCapabilityTtlHours,
+	getModelFailureCooldownMinutes,
+} from "../services/settings";
 import {
 	applyGeminiModelToPath,
 	buildUpstreamChatRequest,
@@ -55,6 +63,17 @@ type ExecutionContextLike = {
 	waitUntil: (promise: Promise<unknown>) => void;
 };
 
+function scheduleDbWrite(
+	c: { executionCtx?: ExecutionContextLike },
+	task: Promise<void>,
+): void {
+	if (c.executionCtx?.waitUntil) {
+		c.executionCtx.waitUntil(task);
+	} else {
+		task.catch(() => undefined);
+	}
+}
+
 function channelSupportsModel(
 	channel: ChannelRecord,
 	model: string | null,
@@ -64,19 +83,24 @@ function channelSupportsModel(
 		return true;
 	}
 	const verified = verifiedModelsByChannel.get(channel.id);
+	const declaredModels = extractModels(channel).map((entry) => entry.id);
 	const metadata = parseChannelMetadata(channel.metadata_json);
 	const mapped = resolveMappedModel(metadata.model_mapping, model);
+	const declaredAllows =
+		declaredModels.length === 0
+			? true
+			: (mapped ? declaredModels.includes(mapped) : false) ||
+				declaredModels.includes(model);
+	if (!declaredAllows) {
+		return false;
+	}
 	if (verified && verified.size > 0) {
 		if (mapped && verified.has(mapped)) {
 			return true;
 		}
 		return verified.has(model);
 	}
-	const declaredModels = extractModels(channel).map((entry) => entry.id);
-	if (mapped && declaredModels.includes(mapped)) {
-		return true;
-	}
-	return declaredModels.includes(model);
+	return true;
 }
 
 export function selectCandidateChannels(
@@ -318,11 +342,40 @@ proxy.all("/*", tokenAuth, async (c) => {
 		allowedChannels.map((channel) => channel.id),
 		ttlSeconds,
 	);
-	const candidates = selectCandidateChannels(
+	let candidates = selectCandidateChannels(
 		allowedChannels,
 		downstreamModel,
 		verifiedModelsByChannel,
 	);
+	const cooldownMinutes = await getModelFailureCooldownMinutes(c.env.DB);
+	const cooldownSeconds = Math.max(0, Math.floor(cooldownMinutes)) * 60;
+	if (downstreamModel && cooldownSeconds > 0 && candidates.length > 0) {
+		const coolingChannels = await listCoolingDownChannelsForModel(
+			c.env.DB,
+			candidates.map((channel) => channel.id),
+			downstreamModel,
+			cooldownSeconds,
+		);
+		if (coolingChannels.size > 0) {
+			candidates = candidates.filter(
+				(channel) => !coolingChannels.has(channel.id),
+			);
+			if (candidates.length === 0) {
+				console.warn("[proxy:model_cooldown]", {
+					path: c.req.path,
+					model: downstreamModel,
+					cooldown_minutes: cooldownMinutes,
+					blocked_channels: coolingChannels.size,
+				});
+				return jsonError(
+					c,
+					503,
+					"upstream_cooldown",
+					"upstream_cooldown",
+				);
+			}
+		}
+	}
 
 	if (candidates.length === 0 && allowedChannels.length > 0) {
 		console.warn("[proxy:no_compatible_channels]", {
@@ -345,6 +398,7 @@ proxy.all("/*", tokenAuth, async (c) => {
 		1000,
 		Number(c.env.PROXY_UPSTREAM_TIMEOUT_MS ?? "30000"),
 	);
+	const nowSeconds = Math.floor(Date.now() / 1000);
 	let lastResponse: Response | null = null;
 	let lastChannel: ChannelRecord | null = null;
 	let lastRequestPath = targetPath;
@@ -363,6 +417,7 @@ proxy.all("/*", tokenAuth, async (c) => {
 			verifiedModelsByChannel,
 		);
 		const upstreamModel = resolvedModel.model;
+		const recordModel = upstreamModel ?? downstreamModel;
 		if (
 			upstreamProvider === "gemini" &&
 			!upstreamModel &&
@@ -594,8 +649,31 @@ proxy.all("/*", tokenAuth, async (c) => {
 				selectedChannel = channel;
 				selectedUpstreamProvider = upstreamProvider;
 				selectedUpstreamModel = upstreamModel;
+				if (recordModel) {
+					scheduleDbWrite(
+						c,
+						upsertChannelModelCapabilities(
+							c.env.DB,
+							channel.id,
+							[recordModel],
+							nowSeconds,
+						),
+					);
+				}
 				break;
 			}
+			if (recordModel) {
+				scheduleDbWrite(
+					c,
+					recordChannelModelError(
+						c.env.DB,
+						channel.id,
+						recordModel,
+						String(response.status),
+						nowSeconds,
+					),
+				);
+			}
 		} catch (error) {
 			const isTimeout =
 				error instanceof Error &&
@@ -610,6 +688,18 @@ proxy.all("/*", tokenAuth, async (c) => {
 				reason: isTimeout ? "timeout" : "exception",
 				error: error instanceof Error ? error.message : String(error),
 			});
+			if (recordModel) {
+				scheduleDbWrite(
+					c,
+					recordChannelModelError(
+						c.env.DB,
+						channel.id,
+						recordModel,
+						isTimeout ? "timeout" : "exception",
+						nowSeconds,
+					),
+				);
+			}
 			lastResponse = null;
 		}
 	}

apps/worker/src/routes/settings.ts

@@ -9,12 +9,14 @@ import {
 	getRetentionDays,
 	getSessionTtlHours,
 	getModelCapabilityTtlHours,
+	getModelFailureCooldownMinutes,
 	isAdminPasswordSet,
 	setAdminPasswordHash,
 	setCheckinScheduleTime,
 	setRetentionDays,
 	setSessionTtlHours,
 	setModelCapabilityTtlHours,
+	setModelFailureCooldownMinutes,
 } from "../services/settings";
 import { sha256Hex } from "../utils/crypto";
 import { jsonError } from "../utils/http";
@@ -30,12 +32,16 @@ settings.get("/", async (c) => {
 	const adminPasswordSet = await isAdminPasswordSet(c.env.DB);
 	const checkinScheduleTime = await getCheckinScheduleTime(c.env.DB);
 	const modelCapabilityTtlHours = await getModelCapabilityTtlHours(c.env.DB);
+	const modelFailureCooldownMinutes = await getModelFailureCooldownMinutes(
+		c.env.DB,
+	);
 	return c.json({
 		log_retention_days: retention,
 		session_ttl_hours: sessionTtlHours,
 		admin_password_set: adminPasswordSet,
 		checkin_schedule_time: checkinScheduleTime,
 		model_capability_ttl_hours: modelCapabilityTtlHours,
+		model_failure_cooldown_minutes: modelFailureCooldownMinutes,
 	});
 });
 
@@ -94,6 +100,20 @@ settings.put("/", async (c) => {
 		touched = true;
 	}
 
+	if (body.model_failure_cooldown_minutes !== undefined) {
+		const minutes = Number(body.model_failure_cooldown_minutes);
+		if (Number.isNaN(minutes) || minutes < 1) {
+			return jsonError(
+				c,
+				400,
+				"invalid_model_failure_cooldown_minutes",
+				"invalid_model_failure_cooldown_minutes",
+			);
+		}
+		await setModelFailureCooldownMinutes(c.env.DB, minutes);
+		touched = true;
+	}
+
 	if (typeof body.admin_password === "string" && body.admin_password.trim()) {
 		const hash = await sha256Hex(body.admin_password.trim());
 		await setAdminPasswordHash(c.env.DB, hash);

apps/worker/src/services/channel-model-capabilities.ts

@@ -1,5 +1,6 @@
 import type { D1Database } from "@cloudflare/workers-types";
 import { nowIso } from "../utils/time";
+import { extractModelIds } from "./channel-models";
 import type { ModelEntry } from "./channel-models";
 
 export const DEFAULT_MODEL_CAPABILITY_TTL_SEC = 2 * 60 * 60;
@@ -67,6 +68,95 @@ export async function listVerifiedModelEntries(
 	return entries;
 }
 
+export async function listModelsByChannelWithFallback(
+	db: D1Database,
+	channels: Array<{ id: string; name: string; models_json?: string | null }>,
+	ttlSeconds: number = DEFAULT_MODEL_CAPABILITY_TTL_SEC,
+): Promise<Map<string, Set<string>>> {
+	const ids = channels.map((channel) => channel.id);
+	const verified = await listVerifiedModelsByChannel(db, ids, ttlSeconds);
+	const map = new Map<string, Set<string>>();
+	for (const channel of channels) {
+		const verifiedModels = verified.get(channel.id);
+		if (verifiedModels && verifiedModels.size > 0) {
+			map.set(channel.id, new Set(verifiedModels));
+			continue;
+		}
+		const declaredModels = extractModelIds(channel);
+		if (declaredModels.length > 0) {
+			map.set(channel.id, new Set(declaredModels));
+		}
+	}
+	return map;
+}
+
+export async function listModelEntriesWithFallback(
+	db: D1Database,
+	channels: Array<{ id: string; name: string; models_json?: string | null }>,
+	ttlSeconds: number = DEFAULT_MODEL_CAPABILITY_TTL_SEC,
+): Promise<ModelEntry[]> {
+	const map = await listModelsByChannelWithFallback(db, channels, ttlSeconds);
+	const entries: ModelEntry[] = [];
+	for (const channel of channels) {
+		const models = map.get(channel.id);
+		if (!models) {
+			continue;
+		}
+		for (const id of models) {
+			entries.push({ id, label: id, channelId: channel.id, channelName: channel.name });
+		}
+	}
+	return entries;
+}
+
+export async function listCoolingDownChannelsForModel(
+	db: D1Database,
+	channelIds: string[],
+	model: string | null,
+	cooldownSeconds: number,
+): Promise<Set<string>> {
+	if (!model || channelIds.length === 0 || cooldownSeconds <= 0) {
+		return new Set();
+	}
+	const now = Math.floor(Date.now() / 1000);
+	const cutoff = now - cooldownSeconds;
+	const placeholders = channelIds.map(() => "?").join(", ");
+	const rows = await db
+		.prepare(
+			`SELECT channel_id, last_err_at, last_ok_at FROM channel_model_capabilities WHERE model = ? AND channel_id IN (${placeholders}) AND last_err_at IS NOT NULL AND last_err_at >= ?`,
+		)
+		.bind(model, ...channelIds, cutoff)
+		.all<{ channel_id: string; last_err_at: number | null; last_ok_at: number | null }>();
+	const blocked = new Set<string>();
+	for (const row of rows.results ?? []) {
+		const lastErr = Number(row.last_err_at ?? 0);
+		const lastOk = Number(row.last_ok_at ?? 0);
+		if (lastErr && lastErr >= lastOk) {
+			blocked.add(row.channel_id);
+		}
+	}
+	return blocked;
+}
+
+export async function recordChannelModelError(
+	db: D1Database,
+	channelId: string,
+	model: string | null,
+	errorCode: string,
+	nowSeconds: number = Math.floor(Date.now() / 1000),
+): Promise<void> {
+	if (!model) {
+		return;
+	}
+	const timestamp = nowIso();
+	await db
+		.prepare(
+			"INSERT INTO channel_model_capabilities (channel_id, model, last_ok_at, last_err_at, last_err_code, created_at, updated_at) VALUES (?, ?, 0, ?, ?, ?, ?) ON CONFLICT(channel_id, model) DO UPDATE SET last_err_at = excluded.last_err_at, last_err_code = excluded.last_err_code, updated_at = excluded.updated_at",
+		)
+		.bind(channelId, model, nowSeconds, errorCode, timestamp, timestamp)
+		.run();
+}
+
 export async function upsertChannelModelCapabilities(
 	db: D1Database,
 	channelId: string,

apps/worker/src/services/settings.ts

@@ -5,11 +5,13 @@ const DEFAULT_LOG_RETENTION_DAYS = 30;
 const DEFAULT_SESSION_TTL_HOURS = 12;
 const DEFAULT_CHECKIN_SCHEDULE_TIME = "00:10";
 const DEFAULT_MODEL_CAPABILITY_TTL_HOURS = 2;
+const DEFAULT_MODEL_FAILURE_COOLDOWN_MINUTES = 10;
 const RETENTION_KEY = "log_retention_days";
 const SESSION_TTL_KEY = "session_ttl_hours";
 const ADMIN_PASSWORD_HASH_KEY = "admin_password_hash";
 const CHECKIN_SCHEDULE_TIME_KEY = "checkin_schedule_time";
 const MODEL_CAPABILITY_TTL_KEY = "model_capability_ttl_hours";
+const MODEL_FAILURE_COOLDOWN_KEY = "model_failure_cooldown_minutes";
 
 async function readSetting(
 	db: D1Database,
@@ -143,6 +145,21 @@ export async function setModelCapabilityTtlHours(
 	await upsertSetting(db, MODEL_CAPABILITY_TTL_KEY, value);
 }
 
+export async function getModelFailureCooldownMinutes(
+	db: D1Database,
+): Promise<number> {
+	const value = await readSetting(db, MODEL_FAILURE_COOLDOWN_KEY);
+	return parsePositiveNumber(value, DEFAULT_MODEL_FAILURE_COOLDOWN_MINUTES);
+}
+
+export async function setModelFailureCooldownMinutes(
+	db: D1Database,
+	minutes: number,
+): Promise<void> {
+	const value = Math.max(1, Math.floor(minutes)).toString();
+	await upsertSetting(db, MODEL_FAILURE_COOLDOWN_KEY, value);
+}
+
 /**
  * Loads generic settings as a key/value map.
  */

tests/ui/usage.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+import { summarizeUsageLogs } from "../../apps/ui/src/core/utils";
+
+const buildLog = (partial: Partial<{ status: string; total_tokens: number | null; prompt_tokens: number | null; completion_tokens: number | null; latency_ms: number | null }>) => ({
+	id: "1",
+	model: "gpt-4",
+	channel_id: "c1",
+	token_id: "t1",
+	total_tokens: partial.total_tokens ?? null,
+	prompt_tokens: partial.prompt_tokens ?? null,
+	completion_tokens: partial.completion_tokens ?? null,
+	latency_ms: partial.latency_ms ?? null,
+	status: partial.status ?? "ok",
+	created_at: "2026-03-14T00:00:00Z",
+});
+
+describe("usage summary", () => {
+	it("summarizes success, failure, tokens, and latency", () => {
+		const logs = [
+			buildLog({ status: "ok", total_tokens: 20, latency_ms: 1000 }),
+			buildLog({ status: "error", total_tokens: 10, latency_ms: 3000 }),
+			buildLog({ status: "ok", total_tokens: null, prompt_tokens: 5, completion_tokens: 5, latency_ms: null }),
+		];
+		const summary = summarizeUsageLogs(logs as any);
+		expect(summary.total).toBe(3);
+		expect(summary.success).toBe(2);
+		expect(summary.failed).toBe(1);
+		expect(summary.totalTokens).toBe(40);
+		expect(Math.round(summary.errorRate)).toBe(33);
+		expect(summary.avgLatencyMs).toBe(2000);
+	});
+});