diff --git a/.gitignore b/.gitignore index 5a68340..1c0908e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,5 @@ apps/ui/dist/ apps/worker/.wrangler/ tests/ -helloagents/plan -helloagents/archive +.helloagents/plan +.helloagents/archive diff --git a/helloagents/CHANGELOG.md b/.helloagents/CHANGELOG.md similarity index 95% rename from helloagents/CHANGELOG.md rename to .helloagents/CHANGELOG.md index ee6edd4..17bddc9 100644 --- a/helloagents/CHANGELOG.md +++ b/.helloagents/CHANGELOG.md @@ -20,6 +20,15 @@ - **[worker/models]**: 引入通道模型能力表并基于“模型广场测试结果+可配置TTL（默认2小时）”进行模型发布与路由匹配 - 类型: 微调（无方案包） - 文件: apps/worker/migrations/0004_add_channel_model_capabilities.sql, apps/worker/src/services/channel-model-capabilities.ts, apps/worker/src/services/channel-testing.ts, apps/worker/src/services/settings.ts, apps/worker/src/routes/settings.ts, apps/worker/src/routes/models.ts, apps/worker/src/routes/newapiUsers.ts, apps/worker/src/routes/proxy.ts, tests/worker/channel-model-capabilities.test.ts, tests/worker/newapi.test.ts, helloagents/modules/settings.md, helloagents/modules/models.md, helloagents/modules/proxy.md +- **[worker/models]**: 运行时失败冷却与成功刷新能力表，TTL 过期回退模型列表 + - 方案: [202603141718_model-health](archive/2026-03/202603141718_model-health/) + - 文件: apps/worker/src/routes/proxy.ts, apps/worker/src/services/channel-model-capabilities.ts, apps/worker/src/routes/models.ts, apps/worker/src/routes/newapiUsers.ts, apps/worker/src/services/settings.ts, apps/worker/src/routes/settings.ts, tests/worker/settings.test.ts, tests/worker/channel-model-capabilities.test.ts, .helloagents/modules/proxy.md, .helloagents/modules/models.md, .helloagents/modules/settings.md +- **[proxy]**: 失败判定简化为非 2xx/超时即失败 + - 方案: [202603141749_usage-summary](archive/2026-03/202603141749_usage-summary/) + - 文件: apps/worker/src/routes/proxy.ts +- **[admin-ui/usage]**: 使用日志新增概述卡片（总量/成功/失败/错误率/平均延迟） + - 方案: [202603141749_usage-summary](archive/2026-03/202603141749_usage-summary/) + - 文件: apps/ui/src/core/utils.ts, apps/ui/src/features/UsageView.tsx, tests/ui/usage.test.ts, .helloagents/modules/usage.md, .helloagents/modules/admin-ui.md - **[worker/proxy]**: 移除临时调试日志（请求入口/出口与上游响应汇总） - 类型: 微调（无方案包） - 文件: apps/worker/src/index.ts, apps/worker/src/routes/proxy.ts diff --git a/helloagents/INDEX.md b/.helloagents/INDEX.md similarity index 100% rename from helloagents/INDEX.md rename to .helloagents/INDEX.md diff --git a/helloagents/config.toml b/.helloagents/config.toml similarity index 100% rename from helloagents/config.toml rename to .helloagents/config.toml diff --git a/helloagents/context.md b/.helloagents/context.md similarity index 100% rename from helloagents/context.md rename to .helloagents/context.md diff --git a/helloagents/modules/_index.md b/.helloagents/modules/_index.md similarity index 100% rename from helloagents/modules/_index.md rename to .helloagents/modules/_index.md diff --git a/helloagents/modules/admin-ui.md b/.helloagents/modules/admin-ui.md similarity index 96% rename from helloagents/modules/admin-ui.md rename to .helloagents/modules/admin-ui.md index c68c182..acb7126 100644 --- a/helloagents/modules/admin-ui.md +++ b/.helloagents/modules/admin-ui.md @@ -25,6 +25,7 @@ - done-hub 站点仅使用 base_url，不再维护多地址输入 - 站点连通测试提示令牌成功/失败统计 - 使用日志支持本地分页（默认 50 条）与固定高度滚动，并展示令牌、输入/输出 tokens、首 token 延迟、流式与推理强度 +- 使用日志顶部展示概述（总量/成功/失败/错误率/平均延迟） - 令牌管理采用列表视图（默认每页 10 条），生成令牌使用弹窗表单，并展示创建时间与前缀 - 站点管理内置一键签到入口，展示今日签到结果（基于后台写回的 `last_checkin_*`） - 系统设置增加定时签到开关与时间（中国时间） diff --git a/helloagents/modules/auth.md b/.helloagents/modules/auth.md similarity index 100% rename from helloagents/modules/auth.md rename to .helloagents/modules/auth.md diff --git a/helloagents/modules/channels.md b/.helloagents/modules/channels.md similarity index 100% rename from helloagents/modules/channels.md rename to .helloagents/modules/channels.md diff --git a/helloagents/modules/checkin.md b/.helloagents/modules/checkin.md similarity index 100% rename from helloagents/modules/checkin.md rename to .helloagents/modules/checkin.md diff --git a/helloagents/modules/dashboard.md b/.helloagents/modules/dashboard.md similarity index 100% rename from helloagents/modules/dashboard.md rename to .helloagents/modules/dashboard.md diff --git a/helloagents/modules/deploy-workflow.md b/.helloagents/modules/deploy-workflow.md similarity index 100% rename from helloagents/modules/deploy-workflow.md rename to .helloagents/modules/deploy-workflow.md diff --git a/helloagents/modules/models.md b/.helloagents/modules/models.md similarity index 88% rename from helloagents/modules/models.md rename to .helloagents/modules/models.md index dd8ab05..78cc506 100644 --- a/helloagents/modules/models.md +++ b/.helloagents/modules/models.md @@ -11,7 +11,7 @@ ## 行为规范 - 仅聚合启用状态的渠道 - 使用“模型广场测试结果”作为可用模型来源 -- 模型结果带 TTL（默认 2 小时，来自 settings），过期后不再返回 +- 模型结果带 TTL（默认 2 小时，来自 settings），过期后回退到渠道声明模型列表（避免模型广场清空） - 模型归一化支持字符串数组与对象数组（对象取 `id` 字段） ## 依赖关系 diff --git a/helloagents/modules/proxy.md b/.helloagents/modules/proxy.md similarity index 90% rename from helloagents/modules/proxy.md rename to .helloagents/modules/proxy.md index c84bebd..fda1332 100644 --- a/helloagents/modules/proxy.md +++ b/.helloagents/modules/proxy.md @@ -27,6 +27,8 @@ - 允许 `metadata_json.header_override` 与 `metadata_json.query_override` 注入额外上游请求头/查询参数 - Anthropic 默认注入 `anthropic-version=2023-06-01`，可通过 `header_override` 覆盖 - 路由时优先使用模型能力表（TTL 内测试成功的模型），无能力记录时回退渠道声明模型列表 +- 上游失败/超时会记录到能力表并进入冷却期，冷却期内跳过对应渠道/模型 +- 上游成功响应会刷新能力表，清空错误并延长可用窗口 ## 依赖关系 - `channels` / `tokens` / `usage_logs` 表 diff --git a/helloagents/modules/settings.md b/.helloagents/modules/settings.md similarity index 81% rename from helloagents/modules/settings.md rename to .helloagents/modules/settings.md index c915542..632ed29 100644 --- a/helloagents/modules/settings.md +++ b/.helloagents/modules/settings.md @@ -5,6 +5,7 @@ - 为使用日志与面板提供统一配置读取 - 维护定时签到执行时间 - 维护模型能力 TTL（模型广场与路由过滤窗口） +- 维护模型失败冷却时间（运行时失败的渠道/模型冷却窗口） ## 接口定义 - `GET /api/settings` 读取设置 @@ -17,6 +18,7 @@ - 更新签到时间后会触发 `CheckinScheduler` 重新调度 - 若更新后当前时间已超过新签到时间，当日可再次触发自动签到 - 模型能力 TTL 以小时计（默认 2 小时），用于控制“模型广场测试结果”的有效期 +- 模型失败冷却以分钟计（默认 10 分钟），用于跳过最近失败的渠道/模型 ## 依赖关系 - `settings` 表 diff --git a/helloagents/modules/sites.md b/.helloagents/modules/sites.md similarity index 100% rename from helloagents/modules/sites.md rename to .helloagents/modules/sites.md diff --git a/helloagents/modules/tokens.md b/.helloagents/modules/tokens.md similarity index 100% rename from helloagents/modules/tokens.md rename to .helloagents/modules/tokens.md diff --git a/helloagents/modules/usage.md b/.helloagents/modules/usage.md similarity index 85% rename from helloagents/modules/usage.md rename to .helloagents/modules/usage.md index ab9726f..afd03fa 100644 --- a/helloagents/modules/usage.md +++ b/.helloagents/modules/usage.md @@ -14,6 +14,7 @@ - 查询结果附带渠道与令牌名称 - usage_logs 记录输入/输出 tokens、首 token 延迟、流式标记与推理强度 - 推理强度来自请求体 `reasoning` / `reasoning_effort` 字段 +- 前端使用日志列表计算概述（总量/成功/失败/错误率/平均延迟） ## 依赖关系 - `usage_logs` 表 diff --git a/apps/ui/src/core/utils.ts b/apps/ui/src/core/utils.ts index 96dbece..1eb7861 100644 --- a/apps/ui/src/core/utils.ts +++ b/apps/ui/src/core/utils.ts @@ -1,3 +1,5 @@ +import type { UsageLog } from "./types"; + /** * Formats a datetime string for display. * @@ -76,3 +78,41 @@ export const buildPageItems = (current: number, total: number): PageItem[] => { return true; }); }; + +export type UsageSummary = { + total: number; + success: number; + failed: number; + errorRate: number; + avgLatencyMs: number | null; + totalTokens: number; +}; + +export const summarizeUsageLogs = (logs: UsageLog[]): UsageSummary => { + const total = logs.length; + const success = logs.filter((log) => log.status === "ok").length; + const failed = total - success; + let latencySum = 0; + let latencyCount = 0; + let totalTokens = 0; + for (const log of logs) { + const latency = log.latency_ms; + if (latency !== null && latency !== undefined && !Number.isNaN(latency)) { + latencySum += latency; + latencyCount += 1; + } + if (log.total_tokens !== null && log.total_tokens !== undefined) { + totalTokens += log.total_tokens; + } else { + totalTokens += (log.prompt_tokens ?? 0) + (log.completion_tokens ?? 0); + } + } + return { + total, + success, + failed, + errorRate: total > 0 ? (failed / total) * 100 : 0, + avgLatencyMs: latencyCount > 0 ? latencySum / latencyCount : null, + totalTokens, + }; +}; diff --git a/apps/ui/src/features/UsageView.tsx b/apps/ui/src/features/UsageView.tsx index b09e420..7370246 100644 --- a/apps/ui/src/features/UsageView.tsx +++ b/apps/ui/src/features/UsageView.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from "hono/jsx/dom"; import type { UsageLog } from "../core/types"; -import { buildPageItems, formatDateTime } from "../core/utils"; +import { buildPageItems, formatDateTime, summarizeUsageLogs } from "../core/utils"; type UsageViewProps = { usage: UsageLog[]; @@ -42,6 +42,7 @@ export const UsageView = ({ usage, onRefresh }: UsageViewProps) => { const [pageSize, setPageSize] = useState(50); const [page, setPage] = useState(1); const total = usage.length; + const summary = useMemo(() => summarizeUsageLogs(usage), [usage]); const totalPages = useMemo( () => Math.max(1, Math.ceil(total / pageSize)), [total, pageSize], @@ -85,6 +86,31 @@ export const UsageView = ({ usage, onRefresh }: UsageViewProps) => { +
+
+ + 概述（当前展示 {summary.total} 条） + + + 成功 {summary.success} + + + 失败 {summary.failed} + + + 错误率 {summary.total === 0 ? "-" : `${summary.errorRate.toFixed(1)}%`} + + + 平均延迟{" "} + {summary.avgLatencyMs === null + ? "-" + : `${summary.avgLatencyMs.toFixed(0)} ms`} + + + 总 Tokens {summary.totalTokens} + +
+
diff --git a/apps/worker/src/routes/models.ts b/apps/worker/src/routes/models.ts index eff8de1..6dab3be 100644 --- a/apps/worker/src/routes/models.ts +++ b/apps/worker/src/routes/models.ts @@ -1,6 +1,6 @@ import { Hono } from "hono"; import type { AppEnv } from "../env"; -import { listVerifiedModelEntries } from "../services/channel-model-capabilities"; +import { listModelEntriesWithFallback } from "../services/channel-model-capabilities"; import { listActiveChannels } from "../services/channel-repo"; import { getModelCapabilityTtlHours } from "../services/settings"; @@ -13,9 +13,13 @@ models.get("/", async (c) => { const channels = await listActiveChannels(c.env.DB); const ttlHours = await getModelCapabilityTtlHours(c.env.DB); const ttlSeconds = Math.max(1, Math.floor(ttlHours)) * 60 * 60; - const entries = await listVerifiedModelEntries( + const entries = await listModelEntriesWithFallback( c.env.DB, - channels.map((channel) => ({ id: channel.id, name: channel.name })), + channels.map((channel) => ({ + id: channel.id, + name: channel.name, + models_json: channel.models_json, + })), ttlSeconds, ); diff --git a/apps/worker/src/routes/newapiUsers.ts b/apps/worker/src/routes/newapiUsers.ts index 0fa80d0..3de087d 100644 --- a/apps/worker/src/routes/newapiUsers.ts +++ b/apps/worker/src/routes/newapiUsers.ts @@ -2,7 +2,7 @@ import { Hono } from "hono"; import type { AppEnv } from "../env"; import { newApiAuth } from "../middleware/newApiAuth"; import { - listVerifiedModelsByChannel, + listModelsByChannelWithFallback, } from "../services/channel-model-capabilities"; import { listActiveChannels } from "../services/channel-repo"; import { getModelCapabilityTtlHours } from "../services/settings"; @@ -15,9 +15,13 @@ users.get("/models", async (c) => { const channels = await listActiveChannels(c.env.DB); const ttlHours = await getModelCapabilityTtlHours(c.env.DB); const ttlSeconds = Math.max(1, Math.floor(ttlHours)) * 60 * 60; - const map = await listVerifiedModelsByChannel( + const map = await listModelsByChannelWithFallback( c.env.DB, - channels.map((channel) => channel.id), + channels.map((channel) => ({ + id: channel.id, + name: channel.name, + models_json: channel.models_json, + })), ttlSeconds, ); const modelSet = new Set(); diff --git a/apps/worker/src/routes/proxy.ts b/apps/worker/src/routes/proxy.ts index 3dad0f6..2c33a82 100644 --- a/apps/worker/src/routes/proxy.ts +++ b/apps/worker/src/routes/proxy.ts @@ -17,8 +17,16 @@ import { createWeightedOrder, extractModels, } from "../services/channels"; -import { listVerifiedModelsByChannel } from "../services/channel-model-capabilities"; -import { getModelCapabilityTtlHours } from "../services/settings"; +import { + listCoolingDownChannelsForModel, + listVerifiedModelsByChannel, + recordChannelModelError, + upsertChannelModelCapabilities, +} from "../services/channel-model-capabilities"; +import { + getModelCapabilityTtlHours, + getModelFailureCooldownMinutes, +} from "../services/settings"; import { applyGeminiModelToPath, buildUpstreamChatRequest, @@ -55,6 +63,17 @@ type ExecutionContextLike = { waitUntil: (promise: Promise) => void; }; +function scheduleDbWrite( + c: { executionCtx?: ExecutionContextLike }, + task: Promise, +): void { + if (c.executionCtx?.waitUntil) { + c.executionCtx.waitUntil(task); + } else { + task.catch(() => undefined); + } +} + function channelSupportsModel( channel: ChannelRecord, model: string | null, @@ -64,19 +83,24 @@ function channelSupportsModel( return true; } const verified = verifiedModelsByChannel.get(channel.id); + const declaredModels = extractModels(channel).map((entry) => entry.id); const metadata = parseChannelMetadata(channel.metadata_json); const mapped = resolveMappedModel(metadata.model_mapping, model); + const declaredAllows = + declaredModels.length === 0 + ? true + : (mapped ? declaredModels.includes(mapped) : false) || + declaredModels.includes(model); + if (!declaredAllows) { + return false; + } if (verified && verified.size > 0) { if (mapped && verified.has(mapped)) { return true; } return verified.has(model); } - const declaredModels = extractModels(channel).map((entry) => entry.id); - if (mapped && declaredModels.includes(mapped)) { - return true; - } - return declaredModels.includes(model); + return true; } export function selectCandidateChannels( @@ -318,11 +342,40 @@ proxy.all("/*", tokenAuth, async (c) => { allowedChannels.map((channel) => channel.id), ttlSeconds, ); - const candidates = selectCandidateChannels( + let candidates = selectCandidateChannels( allowedChannels, downstreamModel, verifiedModelsByChannel, ); + const cooldownMinutes = await getModelFailureCooldownMinutes(c.env.DB); + const cooldownSeconds = Math.max(0, Math.floor(cooldownMinutes)) * 60; + if (downstreamModel && cooldownSeconds > 0 && candidates.length > 0) { + const coolingChannels = await listCoolingDownChannelsForModel( + c.env.DB, + candidates.map((channel) => channel.id), + downstreamModel, + cooldownSeconds, + ); + if (coolingChannels.size > 0) { + candidates = candidates.filter( + (channel) => !coolingChannels.has(channel.id), + ); + if (candidates.length === 0) { + console.warn("[proxy:model_cooldown]", { + path: c.req.path, + model: downstreamModel, + cooldown_minutes: cooldownMinutes, + blocked_channels: coolingChannels.size, + }); + return jsonError( + c, + 503, + "upstream_cooldown", + "upstream_cooldown", + ); + } + } + } if (candidates.length === 0 && allowedChannels.length > 0) { console.warn("[proxy:no_compatible_channels]", { @@ -345,6 +398,7 @@ proxy.all("/*", tokenAuth, async (c) => { 1000, Number(c.env.PROXY_UPSTREAM_TIMEOUT_MS ?? "30000"), ); + const nowSeconds = Math.floor(Date.now() / 1000); let lastResponse: Response | null = null; let lastChannel: ChannelRecord | null = null; let lastRequestPath = targetPath; @@ -363,6 +417,7 @@ proxy.all("/*", tokenAuth, async (c) => { verifiedModelsByChannel, ); const upstreamModel = resolvedModel.model; + const recordModel = upstreamModel ?? downstreamModel; if ( upstreamProvider === "gemini" && !upstreamModel && @@ -594,8 +649,31 @@ proxy.all("/*", tokenAuth, async (c) => { selectedChannel = channel; selectedUpstreamProvider = upstreamProvider; selectedUpstreamModel = upstreamModel; + if (recordModel) { + scheduleDbWrite( + c, + upsertChannelModelCapabilities( + c.env.DB, + channel.id, + [recordModel], + nowSeconds, + ), + ); + } break; } + if (recordModel) { + scheduleDbWrite( + c, + recordChannelModelError( + c.env.DB, + channel.id, + recordModel, + String(response.status), + nowSeconds, + ), + ); + } } catch (error) { const isTimeout = error instanceof Error && @@ -610,6 +688,18 @@ proxy.all("/*", tokenAuth, async (c) => { reason: isTimeout ? "timeout" : "exception", error: error instanceof Error ? error.message : String(error), }); + if (recordModel) { + scheduleDbWrite( + c, + recordChannelModelError( + c.env.DB, + channel.id, + recordModel, + isTimeout ? "timeout" : "exception", + nowSeconds, + ), + ); + } lastResponse = null; } } diff --git a/apps/worker/src/routes/settings.ts b/apps/worker/src/routes/settings.ts index 1445d18..89857ac 100644 --- a/apps/worker/src/routes/settings.ts +++ b/apps/worker/src/routes/settings.ts @@ -9,12 +9,14 @@ import { getRetentionDays, getSessionTtlHours, getModelCapabilityTtlHours, + getModelFailureCooldownMinutes, isAdminPasswordSet, setAdminPasswordHash, setCheckinScheduleTime, setRetentionDays, setSessionTtlHours, setModelCapabilityTtlHours, + setModelFailureCooldownMinutes, } from "../services/settings"; import { sha256Hex } from "../utils/crypto"; import { jsonError } from "../utils/http"; @@ -30,12 +32,16 @@ settings.get("/", async (c) => { const adminPasswordSet = await isAdminPasswordSet(c.env.DB); const checkinScheduleTime = await getCheckinScheduleTime(c.env.DB); const modelCapabilityTtlHours = await getModelCapabilityTtlHours(c.env.DB); + const modelFailureCooldownMinutes = await getModelFailureCooldownMinutes( + c.env.DB, + ); return c.json({ log_retention_days: retention, session_ttl_hours: sessionTtlHours, admin_password_set: adminPasswordSet, checkin_schedule_time: checkinScheduleTime, model_capability_ttl_hours: modelCapabilityTtlHours, + model_failure_cooldown_minutes: modelFailureCooldownMinutes, }); }); @@ -94,6 +100,20 @@ settings.put("/", async (c) => { touched = true; } + if (body.model_failure_cooldown_minutes !== undefined) { + const minutes = Number(body.model_failure_cooldown_minutes); + if (Number.isNaN(minutes) || minutes < 1) { + return jsonError( + c, + 400, + "invalid_model_failure_cooldown_minutes", + "invalid_model_failure_cooldown_minutes", + ); + } + await setModelFailureCooldownMinutes(c.env.DB, minutes); + touched = true; + } + if (typeof body.admin_password === "string" && body.admin_password.trim()) { const hash = await sha256Hex(body.admin_password.trim()); await setAdminPasswordHash(c.env.DB, hash); diff --git a/apps/worker/src/services/channel-model-capabilities.ts b/apps/worker/src/services/channel-model-capabilities.ts index 389d08d..e4d9b0d 100644 --- a/apps/worker/src/services/channel-model-capabilities.ts +++ b/apps/worker/src/services/channel-model-capabilities.ts @@ -1,5 +1,6 @@ import type { D1Database } from "@cloudflare/workers-types"; import { nowIso } from "../utils/time"; +import { extractModelIds } from "./channel-models"; import type { ModelEntry } from "./channel-models"; export const DEFAULT_MODEL_CAPABILITY_TTL_SEC = 2 * 60 * 60; @@ -67,6 +68,95 @@ export async function listVerifiedModelEntries( return entries; } +export async function listModelsByChannelWithFallback( + db: D1Database, + channels: Array<{ id: string; name: string; models_json?: string | null }>, + ttlSeconds: number = DEFAULT_MODEL_CAPABILITY_TTL_SEC, +): Promise>> { + const ids = channels.map((channel) => channel.id); + const verified = await listVerifiedModelsByChannel(db, ids, ttlSeconds); + const map = new Map>(); + for (const channel of channels) { + const verifiedModels = verified.get(channel.id); + if (verifiedModels && verifiedModels.size > 0) { + map.set(channel.id, new Set(verifiedModels)); + continue; + } + const declaredModels = extractModelIds(channel); + if (declaredModels.length > 0) { + map.set(channel.id, new Set(declaredModels)); + } + } + return map; +} + +export async function listModelEntriesWithFallback( + db: D1Database, + channels: Array<{ id: string; name: string; models_json?: string | null }>, + ttlSeconds: number = DEFAULT_MODEL_CAPABILITY_TTL_SEC, +): Promise { + const map = await listModelsByChannelWithFallback(db, channels, ttlSeconds); + const entries: ModelEntry[] = []; + for (const channel of channels) { + const models = map.get(channel.id); + if (!models) { + continue; + } + for (const id of models) { + entries.push({ id, label: id, channelId: channel.id, channelName: channel.name }); + } + } + return entries; +} + +export async function listCoolingDownChannelsForModel( + db: D1Database, + channelIds: string[], + model: string | null, + cooldownSeconds: number, +): Promise> { + if (!model || channelIds.length === 0 || cooldownSeconds <= 0) { + return new Set(); + } + const now = Math.floor(Date.now() / 1000); + const cutoff = now - cooldownSeconds; + const placeholders = channelIds.map(() => "?").join(", "); + const rows = await db + .prepare( + `SELECT channel_id, last_err_at, last_ok_at FROM channel_model_capabilities WHERE model = ? AND channel_id IN (${placeholders}) AND last_err_at IS NOT NULL AND last_err_at >= ?`, + ) + .bind(model, ...channelIds, cutoff) + .all<{ channel_id: string; last_err_at: number | null; last_ok_at: number | null }>(); + const blocked = new Set(); + for (const row of rows.results ?? []) { + const lastErr = Number(row.last_err_at ?? 0); + const lastOk = Number(row.last_ok_at ?? 0); + if (lastErr && lastErr >= lastOk) { + blocked.add(row.channel_id); + } + } + return blocked; +} + +export async function recordChannelModelError( + db: D1Database, + channelId: string, + model: string | null, + errorCode: string, + nowSeconds: number = Math.floor(Date.now() / 1000), +): Promise { + if (!model) { + return; + } + const timestamp = nowIso(); + await db + .prepare( + "INSERT INTO channel_model_capabilities (channel_id, model, last_ok_at, last_err_at, last_err_code, created_at, updated_at) VALUES (?, ?, 0, ?, ?, ?, ?) ON CONFLICT(channel_id, model) DO UPDATE SET last_err_at = excluded.last_err_at, last_err_code = excluded.last_err_code, updated_at = excluded.updated_at", + ) + .bind(channelId, model, nowSeconds, errorCode, timestamp, timestamp) + .run(); +} + export async function upsertChannelModelCapabilities( db: D1Database, channelId: string, diff --git a/apps/worker/src/services/settings.ts b/apps/worker/src/services/settings.ts index 8d62e61..650b799 100644 --- a/apps/worker/src/services/settings.ts +++ b/apps/worker/src/services/settings.ts @@ -5,11 +5,13 @@ const DEFAULT_LOG_RETENTION_DAYS = 30; const DEFAULT_SESSION_TTL_HOURS = 12; const DEFAULT_CHECKIN_SCHEDULE_TIME = "00:10"; const DEFAULT_MODEL_CAPABILITY_TTL_HOURS = 2; +const DEFAULT_MODEL_FAILURE_COOLDOWN_MINUTES = 10; const RETENTION_KEY = "log_retention_days"; const SESSION_TTL_KEY = "session_ttl_hours"; const ADMIN_PASSWORD_HASH_KEY = "admin_password_hash"; const CHECKIN_SCHEDULE_TIME_KEY = "checkin_schedule_time"; const MODEL_CAPABILITY_TTL_KEY = "model_capability_ttl_hours"; +const MODEL_FAILURE_COOLDOWN_KEY = "model_failure_cooldown_minutes"; async function readSetting( db: D1Database, @@ -143,6 +145,21 @@ export async function setModelCapabilityTtlHours( await upsertSetting(db, MODEL_CAPABILITY_TTL_KEY, value); } +export async function getModelFailureCooldownMinutes( + db: D1Database, +): Promise { + const value = await readSetting(db, MODEL_FAILURE_COOLDOWN_KEY); + return parsePositiveNumber(value, DEFAULT_MODEL_FAILURE_COOLDOWN_MINUTES); +} + +export async function setModelFailureCooldownMinutes( + db: D1Database, + minutes: number, +): Promise { + const value = Math.max(1, Math.floor(minutes)).toString(); + await upsertSetting(db, MODEL_FAILURE_COOLDOWN_KEY, value); +} + /** * Loads generic settings as a key/value map. */ diff --git a/tests/ui/usage.test.ts b/tests/ui/usage.test.ts new file mode 100644 index 0000000..656ef11 --- /dev/null +++ b/tests/ui/usage.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { summarizeUsageLogs } from "../../apps/ui/src/core/utils"; + +const buildLog = (partial: Partial<{ status: string; total_tokens: number | null; prompt_tokens: number | null; completion_tokens: number | null; latency_ms: number | null }>) => ({ + id: "1", + model: "gpt-4", + channel_id: "c1", + token_id: "t1", + total_tokens: partial.total_tokens ?? null, + prompt_tokens: partial.prompt_tokens ?? null, + completion_tokens: partial.completion_tokens ?? null, + latency_ms: partial.latency_ms ?? null, + status: partial.status ?? "ok", + created_at: "2026-03-14T00:00:00Z", +}); + +describe("usage summary", () => { + it("summarizes success, failure, tokens, and latency", () => { + const logs = [ + buildLog({ status: "ok", total_tokens: 20, latency_ms: 1000 }), + buildLog({ status: "error", total_tokens: 10, latency_ms: 3000 }), + buildLog({ status: "ok", total_tokens: null, prompt_tokens: 5, completion_tokens: 5, latency_ms: null }), + ]; + const summary = summarizeUsageLogs(logs as any); + expect(summary.total).toBe(3); + expect(summary.success).toBe(2); + expect(summary.failed).toBe(1); + expect(summary.totalTokens).toBe(40); + expect(Math.round(summary.errorRate)).toBe(33); + expect(summary.avgLatencyMs).toBe(2000); + }); +});