From be9e707bfd9c078ac5cc11e2ee91d7f76dd589ac Mon Sep 17 00:00:00 2001 From: Qian <39391600+qianfree@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:53:18 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BC=82=E6=AD=A5=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E5=AF=BC=E8=87=B4=E7=9A=84=E5=A4=9A=E4=B8=AA=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20(#12)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix:修复响应污染问题 * fix:修改存储相关文件为中文注释 * fix:修复二次退款问题 * fix:修复多个考虑不全面问题 1.误删用户数据 2.R2存储默认区域 3.删除失败导致孤儿记录 4.网络抖动事变时重试 5.前端无限次请求获取图片 6.文件管理权限 * fix:修复字符串截断异常问题和文件下载超时问题 * fix:修复多图问题 * fix:异步生成图片时未配置oss提示错误信息 * fix:修复关闭异步后同步生成出错问题 * feat:新增文件存储测试功能，防止设置后不生效 * fix:修复同步图片转异步后计费错误 * fix:修复无下限扣款问题 * fix:修复结算重复扣款问题 * fix:修复计费出错导致的0费用结算 * fix:修复回调双重履约 * fix:修复预扣事务 * fix:修复费用字段类型问题 * fix:修复缓存问题 * fix:修复安全问题 * fix:修复数据库操作事务问题 * refactor:重构事务使用方式，改为官方推荐的dao形式 * fix:修复余额更新隐患 * fix:修复N+1查询问题 * fix:修复密钥安全问题 * fix:结算失败时打印错误日志 * improve:优化扣款 * fix:修复用户接口越权 --------- Co-authored-by: chengqian --- CLAUDE.md | 1 + api/admin/admin.go | 1 + api/admin/v1/settings.go | 23 + api/admin/v1/task_management.go | 1 + api/tenant/v1/models.go | 7 +- api/tenant/v1/task_tenant.go | 1 + docs/reference/goframe-conventions.md | 23 +- go.mod | 1 + go.sum | 2 + internal/cmd/cmd.go | 14 +- .../controller/admin/admin_v1_settings.go | 3 + internal/dao/internal/aud_request_logs.go | 2 - internal/handler/relay/sync_image.go | 63 ++- internal/handler/relay/sync_image_test.go | 24 + internal/handler/relay/task.go | 6 +- internal/logic/admin/channel.go | 12 +- internal/logic/admin/dashboard.go | 6 +- internal/logic/admin/data_governance.go | 24 +- internal/logic/admin/member.go | 25 +- internal/logic/admin/model.go | 81 ++-- internal/logic/admin/model_import_export.go | 14 +- internal/logic/admin/order.go | 7 +- internal/logic/admin/permission.go | 15 +- internal/logic/admin/redemption.go | 5 +- internal/logic/admin/settings.go | 131 +++++- internal/logic/admin/task_management.go | 9 + internal/logic/admin/tenant.go | 170 ++++--- internal/logic/billing/currency.go | 12 +- internal/logic/billing/money.go | 54 +++ internal/logic/billing/money_test.go | 89 ++++ internal/logic/billing/pricing.go | 29 +- internal/logic/billing/provider.go | 7 + internal/logic/billing/reconciliation.go | 11 +- internal/logic/billing/settlement.go | 434 +++++++++++------- internal/logic/billing/task_billing.go | 263 ++++++----- internal/logic/billing/task_billing_test.go | 112 +++++ internal/logic/billing/wallet.go | 135 +++--- internal/logic/billing/wallet_test.go | 4 +- internal/logic/common/cache.go | 48 ++ internal/logic/common/cron.go | 47 ++ internal/logic/common/security.go | 8 + internal/logic/common/session.go | 9 +- internal/logic/common/settings_registry.go | 4 +- internal/logic/common/storage.go | 84 ++-- internal/logic/common/storage_cos.go | 25 +- internal/logic/common/storage_factory.go | 29 +- internal/logic/common/storage_oss.go | 23 +- internal/logic/common/storage_s3.go | 33 +- internal/logic/common/storage_task_result.go | 55 +++ internal/logic/common/storage_test.go | 23 +- internal/logic/common/tenant_model.go | 40 +- internal/logic/open/open_api.go | 6 +- internal/logic/payment/callback.go | 20 +- internal/logic/payment/fulfill.go | 55 ++- internal/logic/task/async_provider.go | 2 + internal/logic/task/sync_image_worker.go | 192 ++++++-- internal/logic/task/sync_image_worker_test.go | 170 ++++++- internal/logic/tenant/api_key.go | 4 +- internal/logic/tenant/auth.go | 14 +- internal/logic/tenant/member.go | 163 ++++--- internal/logic/tenant/member_model_scope.go | 12 +- internal/logic/tenant/model.go | 18 +- internal/logic/tenant/open_platform.go | 6 +- internal/logic/tenant/organization.go | 8 +- internal/logic/tenant/project.go | 6 +- internal/logic/tenant/redemption.go | 38 +- internal/logic/tenant/security.go | 10 + internal/logic/tenant/task.go | 9 + internal/middleware/admin_auth.go | 19 +- internal/middleware/idempotency.go | 22 +- internal/middleware/open_auth.go | 48 ++ internal/middleware/operation_log.go | 3 + internal/middleware/security_fixes_test.go | 92 ++++ internal/model/do/aud_request_logs.go | 1 - internal/model/entity/aud_request_logs.go | 1 - internal/model/entity/sys_admin_users.go | 32 +- internal/service/admin.go | 7 + internal/utility/crypto/crypto.go | 21 +- internal/utility/export/csv.go | 3 +- internal/utility/export/helper.go | 21 + migrations/000012_v0_1_14_version.sql | 18 + relay/constant/errors.go | 57 +++ relay/constant/errors_test.go | 51 ++ relay/handler/realtime_handler.go | 39 +- relay/handler/relay_handler.go | 21 +- relay/handler/task_handler.go | 26 ++ relay/scheduler/affinity_test.go | 27 -- relay/scheduler/retry.go | 30 -- web/admin/src/views/ai/ChannelsPage.vue | 27 +- web/admin/src/views/ai/TaskLogsPage.vue | 18 +- web/admin/src/views/system/FilesPage.vue | 15 +- web/admin/src/views/system/SettingsPage.vue | 23 + web/admin/src/views/tenant/UsageLogsPage.vue | 14 +- .../src/views/service/PlaygroundPage.vue | 4 +- web/tenant/src/views/service/TaskLogsPage.vue | 19 +- .../src/views/service/UsageLogsPage.vue | 14 +- .../src/views/service/playground/ImageTab.vue | 172 +++++-- 97 files changed, 2810 insertions(+), 1017 deletions(-) create mode 100644 internal/logic/billing/money.go create mode 100644 internal/logic/billing/money_test.go create mode 100644 internal/logic/common/storage_task_result.go create mode 100644 internal/middleware/security_fixes_test.go create mode 100644 migrations/000012_v0_1_14_version.sql create mode 100644 relay/constant/errors_test.go delete mode 100644 relay/scheduler/retry.go diff --git a/CLAUDE.md b/CLAUDE.md index 2b9ca4c..06a4252 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -370,6 +370,7 @@ wallets 表包含 `frozen_balance` 字段，用于支付中/退款中的金额 - 每个 API 请求生成唯一 Request ID，贯穿全链路 - 敏感字段（密码、API Key 原值）AES-256 加密存储 - 缓存策略：API Key 60s、用户信息 300s、模型定价 600s、渠道信息 300s +- 代码注释统一使用中文编写 ### 前端 diff --git a/api/admin/admin.go b/api/admin/admin.go index 66f82e1..1a01448 100644 --- a/api/admin/admin.go +++ b/api/admin/admin.go @@ -224,6 +224,7 @@ type IAdminV1 interface { AdminSettingsCategories(ctx context.Context, req *v1.AdminSettingsCategoriesReq) (res *v1.AdminSettingsCategoriesRes, err error) AdminSettingsGet(ctx context.Context, req *v1.AdminSettingsGetReq) (res *v1.AdminSettingsGetRes, err error) AdminSettingsUpdate(ctx context.Context, req *v1.AdminSettingsUpdateReq) (res *v1.AdminSettingsUpdateRes, err error) + AdminStorageTest(ctx context.Context, req *v1.AdminStorageTestReq) (res *v1.AdminStorageTestRes, err error) TaskList(ctx context.Context, req *v1.TaskListReq) (res *v1.TaskListRes, err error) TaskDetail(ctx context.Context, req *v1.TaskDetailReq) (res *v1.TaskDetailRes, err error) TaskCancel(ctx context.Context, req *v1.TaskCancelReq) (res *v1.TaskCancelRes, err error) diff --git a/api/admin/v1/settings.go b/api/admin/v1/settings.go index 962798c..cfde1b9 100644 --- a/api/admin/v1/settings.go +++ b/api/admin/v1/settings.go @@ -47,3 +47,26 @@ type AdminSettingsUpdateReq struct { } type AdminSettingsUpdateRes struct{} + +// AdminStorageTestReq 测试对象存储（OSS/S3/COS）配置连通性。 +// 携带表单中的存储配置（可含未保存的改动）；access_key_id / access_key_secret 为空或掩码 +// "******" 时后端回落到已保存的值，从而支持「未保存直接测试」。 +type AdminStorageTestReq struct { + g.Meta `path:"/settings/storage/test" method:"post" mime:"json" tags:"管理后台-系统设置" summary:"测试对象存储配置"` + Provider string `json:"storage_provider" dc:"存储供应商"` + Endpoint string `json:"storage_endpoint" dc:"存储端点"` + Region string `json:"storage_region" dc:"存储区域"` + Bucket string `json:"storage_bucket" dc:"存储桶名称"` + AccessKeyID string `json:"storage_access_key_id" dc:"Access Key ID，空/掩码回落已保存值"` + SecretKey string `json:"storage_access_key_secret" dc:"Access Key Secret，空/掩码回落已保存值"` + UseSSL bool `json:"storage_use_ssl" dc:"是否启用 SSL"` + PathPrefix string `json:"storage_path_prefix" dc:"路径前缀"` +} + +type AdminStorageTestRes struct { + Uploaded bool `json:"uploaded"` // 测试图片上传成功 + Downloaded bool `json:"downloaded"` // 下载并校验内容一致 + Deleted bool `json:"deleted"` // 测试对象清理成功 + ElapsedMs int64 `json:"elapsed_ms"` // 往返耗时(毫秒) + Message string `json:"message"` // 结果描述 +} diff --git a/api/admin/v1/task_management.go b/api/admin/v1/task_management.go index 005729f..c2c2299 100644 --- a/api/admin/v1/task_management.go +++ b/api/admin/v1/task_management.go @@ -26,6 +26,7 @@ type ModelTaskItem struct { ActualCost float64 `json:"actual_cost"` BillingSettled bool `json:"billing_settled"` ResultURL string `json:"result_url,omitempty"` + ResultThumbURL string `json:"result_thumb_url,omitempty"` // 结果图预览缩略图 URL（仅任务详情、re-host 图片任务返回） TenantID int64 `json:"tenant_id"` UserID int64 `json:"user_id"` SubmitTime string `json:"submit_time,omitempty"` diff --git a/api/tenant/v1/models.go b/api/tenant/v1/models.go index 40b900e..004e46e 100644 --- a/api/tenant/v1/models.go +++ b/api/tenant/v1/models.go @@ -41,7 +41,10 @@ type TenantAvailableModelItem struct { CacheReadPrice *float64 `json:"cache_read_price"` CacheCreationPrice *float64 `json:"cache_creation_price"` PricingTiers []PricingTierItem `json:"pricing_tiers"` - // AsyncImage 图片模型是否必须走异步端点（提交 /v1/images/generations/async 后轮询取图）。 - // 仅图片分类模型有意义；DashScope 等异步图片 provider 为 true。 + // AsyncImage 图片模型异步端点是否可用（提交 /v1/images/generations/async 后轮询取图）。 + // 仅图片分类模型有意义；真异步厂商，或同步厂商且「同步图片异步化」开关开启时为 true。 AsyncImage bool `json:"async_image"` + // ImageSyncSupported 图片模型同步端点（/v1/images/generations，阻塞一次性返回）是否可用。 + // 仅图片分类模型有意义；「仅异步」厂商（阿里 image-synthesis 等）为 false，其余为 true。 + ImageSyncSupported bool `json:"image_sync_supported"` } diff --git a/api/tenant/v1/task_tenant.go b/api/tenant/v1/task_tenant.go index a614aee..2d16891 100644 --- a/api/tenant/v1/task_tenant.go +++ b/api/tenant/v1/task_tenant.go @@ -26,6 +26,7 @@ type TenantTaskItem struct { ActualCost float64 `json:"actual_cost"` BillingSettled bool `json:"billing_settled"` ResultURL string `json:"result_url,omitempty"` + ResultThumbURL string `json:"result_thumb_url,omitempty"` // 结果图预览缩略图 URL（仅任务详情、re-host 图片任务返回） Username string `json:"username,omitempty"` SubmitTime string `json:"submit_time,omitempty"` FinishTime string `json:"finish_time,omitempty"` diff --git a/docs/reference/goframe-conventions.md b/docs/reference/goframe-conventions.md index 4b542b3..896be81 100644 --- a/docs/reference/goframe-conventions.md +++ b/docs/reference/goframe-conventions.md @@ -244,18 +244,33 @@ dao.TntUsers.Ctx(ctx).Where("id", id).Data(do.TntUsers{ ### 事务 +**统一写法：`g.DB().Transaction` 入口 + 闭包内 `dao.Xxx.Ctx(ctx)` 传播式**（GoFrame 官方推荐）。 + +GoFrame 的 `Transaction` 会把事务对象**注入闭包的 `ctx`**，闭包内任何 `dao.Xxx.Ctx(ctx)` / `g.DB().Ctx(ctx)` 只要用的是这个 `ctx`，就自动挂到当前事务，无需手动持有 `tx` 句柄。 + ```go +// 正确 — ctx 传播式：闭包内统一用 dao.Xxx.Ctx(ctx)，不碰 tx 句柄 err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - // tx.Model() 替代 dao.Xxx.Ctx()，确保操作在同一事务内 - _, err := tx.Model("tnt_tenants").Ctx(ctx).Data(do.TntTenants{...}).Insert() + _, err := dao.TntTenants.Ctx(ctx).Data(do.TntTenants{...}).Insert() if err != nil { - return err // 自动回滚 + return err // 返回 error 自动回滚；返回 nil 自动提交；panic 也会回滚 } - _, err = tx.Model("tnt_users").Ctx(ctx).Data(do.TntUsers{...}).Insert() + _, err = dao.TntUsers.Ctx(ctx).Data(do.TntUsers{...}).Insert() return err }) ``` +**强约束（写事务必须遵守）：** + +1. **入口统一用 `g.DB().Transaction(ctx, ...)`**，不要用 `dao.Xxx.Transaction(...)`。后者会把事务"挂在某张表"上产生误导（闭包内往往操作的是别的表），且不利于统一。 +2. **闭包内一律用 `dao.Xxx.Ctx(ctx)`**（或原生 `g.DB().Ctx(ctx).Exec(...)`），**不使用 `tx` 句柄**。闭包签名 `func(ctx, tx gdb.TX)` 是框架要求，保留不动，但函数体不引用 `tx`（Go 不会因未使用的参数报错）。 +3. **`ctx` 必须逐层传递**——这是本写法的**承重纪律**。闭包内调用其它函数时，必须把闭包的 `ctx` 传下去，被调函数内部也必须用 `dao.Xxx.Ctx(ctx)`，事务才会贯穿。 +4. **原生 SQL 例外**：钱包自减/`GREATEST` 等 DO 表达不了的算术更新，保留原生 SQL，入口从 `tx.Ctx(ctx).Exec(...)` 换成 `g.DB().Ctx(ctx).Exec(...)`，SQL 与 PostgreSQL 的 `$1/$2` 占位符原样保留。 + +**⚠️ 头号陷阱——静默脱离事务**：闭包内若漏写 `.Ctx(ctx)`（如 `g.DB().Model("t").Insert()`）或误传了不带事务的 ctx，该语句会**脱离事务被真正提交、无法回滚**，且**没有任何编译/运行时报错**。验证手段：开 SQL 调试日志，事务内每条语句都应带 `[txid:N]`；不带 `txid` 的就是漏挂了。 + +**嵌套事务/传播行为**：默认传播类型为 `PropagationNested`（用 SavePoint），闭包内再调一个自己开 `Transaction` 的函数会自动成为父事务的嵌套保存点，跟随父事务回滚。若某操作需要"即使主流程回滚也要独立提交"（如审计留痕），必须显式用 `TransactionWithOptions(ctx, gdb.TxOptions{Propagation: gdb.PropagationRequiresNew}, ...)`。 + ### g.Map vs DO 结构体 - **禁止使用 `g.Map` 做数据库操作**：所有 `Data()` 调用必须使用 `do.XxxTable{}` 结构体 diff --git a/go.mod b/go.mod index 56f6f4f..9670cca 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/redis/go-redis/v9 v9.19.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil/v3 v3.24.5 + github.com/shopspring/decimal v1.4.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.73 github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 diff --git a/go.sum b/go.sum index 3438882..b92283a 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,8 @@ github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index e081600..5d18650 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -28,6 +28,7 @@ import ( "github.com/qianfree/team-api/internal/logic/update" "github.com/qianfree/team-api/internal/middleware" "github.com/qianfree/team-api/internal/response" + "github.com/qianfree/team-api/internal/utility/crypto" "github.com/qianfree/team-api/internal/handler/public" "github.com/qianfree/team-api/internal/handler/relay" @@ -61,6 +62,14 @@ var ( // Initialize JWT secret common.InitJWTSecret(ctx) + // 启动阶段校验加密密钥，避免运行时请求路径 panic（#3） + if hexKey := g.Cfg().MustGet(ctx, "crypto.encryptionKey").String(); hexKey == "" { + g.Log().Fatal(ctx, "crypto.encryptionKey 未配置，程序退出（运行 openssl rand -hex 32 生成）") + } else if _, keyErr := crypto.GetEncryptionKey(hexKey); keyErr != nil { + g.Log().Fatalf(ctx, "crypto.encryptionKey 配置无效，程序退出: %v", keyErr) + } + g.Log().Info(ctx, "加密密钥校验通过") + // System initialization: env auto-init or detect setup mode autoInit, initErr := admin.AutoInitAdmin(ctx) if initErr != nil { @@ -80,6 +89,9 @@ var ( common.Config().Warmup(ctx) common.Config().StartSubscriber(ctx) + // C1: 通用 Cache 的跨实例 L1 失效订阅（cache:invalidate） + common.StartCacheInvalidationSubscriber(ctx) + // Initialize content filter engine common.InitContentFilter(ctx) @@ -264,7 +276,7 @@ var ( // Tenant — public endpoints use g.Meta middleware:"-" to skip auth group.Group("/tenant", func(g *ghttp.RouterGroup) { - g.Middleware(middleware.DemoMode, middleware.MaintenanceMode, middleware.TenantAuth) + g.Middleware(middleware.DemoMode, middleware.MaintenanceMode, middleware.TenantAuth, middleware.Idempotency) g.Bind(tenantController.NewV1()) }) diff --git a/internal/controller/admin/admin_v1_settings.go b/internal/controller/admin/admin_v1_settings.go index aaffa6d..efbdf1b 100644 --- a/internal/controller/admin/admin_v1_settings.go +++ b/internal/controller/admin/admin_v1_settings.go @@ -16,3 +16,6 @@ func (c *ControllerV1) AdminSettingsGet(ctx context.Context, req *v1.AdminSettin func (c *ControllerV1) AdminSettingsUpdate(ctx context.Context, req *v1.AdminSettingsUpdateReq) (res *v1.AdminSettingsUpdateRes, err error) { return service.Admin().UpdateSettings(ctx, req) } +func (c *ControllerV1) AdminStorageTest(ctx context.Context, req *v1.AdminStorageTestReq) (res *v1.AdminStorageTestRes, err error) { + return service.Admin().TestStorageConfig(ctx, req) +} diff --git a/internal/dao/internal/aud_request_logs.go b/internal/dao/internal/aud_request_logs.go index 8cb94c2..f62e8d1 100644 --- a/internal/dao/internal/aud_request_logs.go +++ b/internal/dao/internal/aud_request_logs.go @@ -51,7 +51,6 @@ type AudRequestLogsColumns struct { TaskResult string // 异步任务完成时上游返回的原始响应体 TaskUpstreamHeaders string // 异步任务完成时上游返回的响应头（仅审计级别为 full 时记录） TaskCompletedAt string // 异步任务达到终态的时间 - AttachmentIds string // 提取的媒体附件 fil_files ID 数组（JSONB: [1,2,3]） } // audRequestLogsColumns holds the columns for the table aud_request_logs. @@ -86,7 +85,6 @@ var audRequestLogsColumns = AudRequestLogsColumns{ TaskResult: "task_result", TaskUpstreamHeaders: "task_upstream_headers", TaskCompletedAt: "task_completed_at", - AttachmentIds: "attachment_ids", } // NewAudRequestLogsDao creates and returns a new DAO object for table data access. diff --git a/internal/handler/relay/sync_image.go b/internal/handler/relay/sync_image.go index 4bf5c96..bde66eb 100644 --- a/internal/handler/relay/sync_image.go +++ b/internal/handler/relay/sync_image.go @@ -13,6 +13,7 @@ import ( "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" + lcommon "github.com/qianfree/team-api/internal/logic/common" "github.com/qianfree/team-api/internal/logic/task" relay_common "github.com/qianfree/team-api/relay/common" relay_constant "github.com/qianfree/team-api/relay/constant" @@ -57,6 +58,19 @@ func HandleSyncImageSubmit(r *ghttp.Request, body []byte, rc *relay_handler.Task // 提交很快即返回，并发槽在提交结束即释放（上游长连接由 worker 持有，不占此槽）。 defer taskBillingProvider.ReleaseApiKeyConcurrent(ctx, rc.ApiKeyID) + // Fast-fail：对象存储未配置、且本请求**必然**需要存储保存结果（b64 结果 / 强制 re-host）时， + // 提交阶段即返回友好提示——避免白打一次上游生成（生成成功却因无处保存而失败退款，且用户 + // 要空等 10–60s 才知道是配置问题）。url 透传类请求（未开启 re-host 且非 b64）不需要存储， + // 不在此拦截，仍可正常完成。 + if !lcommon.IsStorageConfigured(ctx) { + rehostOn := lcommon.Config().GetBool(ctx, "sync_image_rehost_url") + if rehostOn || requestForcesB64(body, modelName) { + writeSyncImageError(w, http.StatusServiceUnavailable, + "平台尚未配置对象存储（OSS/S3/COS），无法保存生成的图片，请联系管理员在系统设置中配置对象存储后重试") + return + } + } + // 2. 强制非流式：剥离 stream 字段，防 DoResponse 走 SSE 分支 cleanBody := stripStreamField(body) @@ -82,12 +96,15 @@ func HandleSyncImageSubmit(r *ghttp.Request, body []byte, rc *relay_handler.Task publicTaskID := generateSyncImagePublicID() now := time.Now() - // 从渠道类型推导实际的供应商平台（ali / gemini / volcengine / openai 等） + // 从渠道类型推导实际的供应商平台（ali / gemini / volcengine / openai 等），用于任务日志展示。 providerType := relay_constant.ProviderType(channelMeta.ChannelType) actualPlatform, ok := relay_constant.ProviderTypeToTaskPlatform(providerType) if !ok { - // 如果无法映射（如 openai、claude 等），直接使用供应商类型作为平台名 - actualPlatform = relay_constant.TaskPlatform(providerType) + // 未命中映射（如 openai、claude 等）：用供应商类型的可读名作平台名。 + // 注意不能写 TaskPlatform(providerType)——那是 int→string 的 rune 转换，会得到 + // 不可见控制字符（如 providerType=1 → "\x01"）而非 "openai"。取 String() 并转小写， + // 与已映射平台（"ali"/"gemini"）的小写风格保持一致。 + actualPlatform = relay_constant.TaskPlatform(strings.ToLower(providerType.String())) } privateData, _ := json.Marshal(map[string]any{ @@ -120,11 +137,15 @@ func HandleSyncImageSubmit(r *ghttp.Request, body []byte, rc *relay_handler.Task return } - // CreateTask 不回填自增 ID，回查取任务行主键供 worker CAS 使用。 + // CreateTask 不回填自增 ID（Postgres 无 LastInsertId），回查取任务行主键供 worker CAS 使用。 created, err := taskDataProvider.GetTaskByPublicID(ctx, publicTaskID) if err != nil || created == nil { - _ = taskBillingProvider.SettleTaskFailed(ctx, rc.TenantID, rc.RequestID, preDeduct) - g.Log().Errorf(ctx, "sync_image: load created task %s failed: %v", publicTaskID, err) + // 此时任务行已 CreateTask 成功落库（QUEUED、billing_settled=false、active 计数已 +1）， + // 但缺主键无法在这里安全地 CAS 收尾。**不能**在此直接退款：UnfreezePreDeduct 非幂等， + // 与超时兜底网 handleTimedOutTasks 会形成二次退款（侵蚀他人冻结额、可用余额虚高）。 + // 故此处只返回错误，把「CAS→FAILURE + 退款 + DecrActiveTask」交给超时兜底网做恰好一次结算 + // （handleTimedOutTasks 无平台过滤，会捕获这类 QUEUED 孤儿行）。 + g.Log().Errorf(ctx, "sync_image: load created task %s failed, deferring settlement to timeout sweeper: %v", publicTaskID, err) writeSyncImageError(w, http.StatusInternalServerError, "create task record failed") return } @@ -169,6 +190,24 @@ func HandleSyncImageSubmit(r *ghttp.Request, body []byte, rc *relay_handler.Task }) } +// requestForcesB64 判断该图片请求是否**必然**产生需要 re-host 的 base64 结果，即使 +// 未开启 re-host 开关也一定用到对象存储： +// - response_format=b64_json：显式要求 base64； +// - gpt-image 系列：上游只返回 b64_json（不支持 url）。 +// +// 其余情况（默认 / response_format=url）在未开启 re-host 时可 url 透传，不强制需要存储， +// 因此不在提交阶段 fast-fail，避免误伤纯 url 透传的模型。 +func requestForcesB64(body []byte, model string) bool { + var req struct { + ResponseFormat string `json:"response_format"` + } + _ = json.Unmarshal(body, &req) + if strings.EqualFold(strings.TrimSpace(req.ResponseFormat), "b64_json") { + return true + } + return strings.HasPrefix(strings.ToLower(model), "gpt-image") +} + // stripStreamField 移除请求体中的 stream 字段（强制非流式）。 func stripStreamField(body []byte) []byte { var m map[string]json.RawMessage @@ -234,6 +273,18 @@ func writeSyncImageError(w http.ResponseWriter, statusCode int, message string) }) } +// writeSyncImageErrorWithCode 与 writeSyncImageError 相同，但在错误体中附带机器可读的 code， +// 供前端做条件分支（如在线体验「异步被禁用 → 优雅降级到同步端点」）。 +func writeSyncImageErrorWithCode(w http.ResponseWriter, statusCode int, code, message string) { + writeSyncImageJSON(w, statusCode, map[string]any{ + "error": map[string]any{ + "type": "invalid_request_error", + "code": code, + "message": message, + }, + }) +} + func writeSyncImageJSON(w http.ResponseWriter, statusCode int, data any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) diff --git a/internal/handler/relay/sync_image_test.go b/internal/handler/relay/sync_image_test.go index 3714880..ef981f9 100644 --- a/internal/handler/relay/sync_image_test.go +++ b/internal/handler/relay/sync_image_test.go @@ -44,6 +44,30 @@ func TestStripStreamField_InvalidJSON(t *testing.T) { } } +func TestRequestForcesB64(t *testing.T) { + cases := []struct { + name string + body string + model string + want bool + }{ + {"explicit b64_json", `{"response_format":"b64_json"}`, "dall-e-3", true}, + {"b64_json case-insensitive", `{"response_format":"B64_JSON"}`, "dall-e-3", true}, + {"gpt-image always b64", `{}`, "gpt-image-1", true}, + {"gpt-image case-insensitive", `{}`, "GPT-Image-1", true}, + {"response_format url passthrough", `{"response_format":"url"}`, "dall-e-3", false}, + {"default dall-e url", `{"prompt":"a cat"}`, "dall-e-3", false}, + {"invalid json non-b64 model", `not json`, "dall-e-2", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := requestForcesB64([]byte(c.body), c.model); got != c.want { + t.Errorf("requestForcesB64(%q, %q) = %v, want %v", c.body, c.model, got, c.want) + } + }) + } +} + func TestCheckSyncImageIPWhitelist(t *testing.T) { cases := []struct { name string diff --git a/internal/handler/relay/task.go b/internal/handler/relay/task.go index 5e43fb0..dacd2c3 100644 --- a/internal/handler/relay/task.go +++ b/internal/handler/relay/task.go @@ -322,8 +322,10 @@ func HandleAliImageSubmit(r *ghttp.Request) { // 同步阻塞图片：worker 池异步化（阿里 multimodal 恒走；其余同步厂商受开关约束） HandleSyncImageSubmit(r, body, rc, channelMeta) default: - // 同步厂商 + 「同步图片异步化」关闭：异步端点不支持，提示改用同步端点 - writeSyncImageError(rc.Writer, 400, "async image generation is disabled for this model; call POST /v1/images/generations instead") + // 同步厂商 + 「同步图片异步化」关闭：异步端点不支持。附带 code=image_async_disabled， + // 供在线体验前端识别并优雅降级到同步端点 /v1/images/generations。 + writeSyncImageErrorWithCode(rc.Writer, 400, "image_async_disabled", + "async image generation is disabled for this model; call POST /v1/images/generations instead") } if rc.TaskID != "" { diff --git a/internal/logic/admin/channel.go b/internal/logic/admin/channel.go index bb1771a..fd2135d 100644 --- a/internal/logic/admin/channel.go +++ b/internal/logic/admin/channel.go @@ -360,16 +360,16 @@ func (s *sAdmin) UpdateChannel(ctx context.Context, req *v1.ChannelUpdateReq) (* // DeleteChannel 删除渠道 func (s *sAdmin) DeleteChannel(ctx context.Context, req *v1.ChannelDeleteReq) (*v1.ChannelDeleteRes, error) { err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - if _, err := tx.Model("chn_abilities").Ctx(ctx).Where("channel_id", req.ID).Delete(); err != nil { + if _, err := dao.ChnAbilities.Ctx(ctx).Where("channel_id", req.ID).Delete(); err != nil { return err } - if _, err := tx.Model("chn_channel_keys").Ctx(ctx).Where("channel_id", req.ID).Delete(); err != nil { + if _, err := dao.ChnChannelKeys.Ctx(ctx).Where("channel_id", req.ID).Delete(); err != nil { return err } - if _, err := tx.Model("chn_health_scores").Ctx(ctx).Where("channel_id", req.ID).Delete(); err != nil { + if _, err := dao.ChnHealthScores.Ctx(ctx).Where("channel_id", req.ID).Delete(); err != nil { return err } - if _, err := tx.Model("chn_channels").Ctx(ctx).Where("id", req.ID).Delete(); err != nil { + if _, err := dao.ChnChannels.Ctx(ctx).Where("id", req.ID).Delete(); err != nil { return err } return nil @@ -493,11 +493,11 @@ func (s *sAdmin) DeleteChannelKey(ctx context.Context, req *v1.ChannelKeyDeleteR // SetChannelAbilities 设置渠道模型能力 func (s *sAdmin) SetChannelAbilities(ctx context.Context, req *v1.ChannelAbilityBatchReq) (*v1.ChannelAbilityBatchRes, error) { err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - if _, err := tx.Model("chn_abilities").Ctx(ctx).Where("channel_id", req.ChannelID).Delete(); err != nil { + if _, err := dao.ChnAbilities.Ctx(ctx).Where("channel_id", req.ChannelID).Delete(); err != nil { return err } for _, ab := range req.Abilities { - if _, err := tx.Model("chn_abilities").Ctx(ctx).Insert(do.ChnAbilities{ + if _, err := dao.ChnAbilities.Ctx(ctx).Insert(do.ChnAbilities{ ChannelId: req.ChannelID, ModelName: ab.ModelName, UpstreamModel: ab.UpstreamModel, diff --git a/internal/logic/admin/dashboard.go b/internal/logic/admin/dashboard.go index 0634405..2f99713 100644 --- a/internal/logic/admin/dashboard.go +++ b/internal/logic/admin/dashboard.go @@ -404,7 +404,7 @@ func (s *sAdmin) AdjustBalance(ctx context.Context, req *v1.AdminWalletAdjustReq updateQuery += " AND balance >= ?" args = append(args, -amount) } - result, err := tx.Ctx(ctx).Exec(updateQuery, args...) + result, err := g.DB().Ctx(ctx).Exec(ctx, updateQuery, args...) if err != nil { return err } @@ -419,7 +419,7 @@ func (s *sAdmin) AdjustBalance(ctx context.Context, req *v1.AdminWalletAdjustReq Balance float64 `json:"balance"` FrozenBalance float64 `json:"frozen_balance"` } - if err = tx.Model("bil_wallets").Ctx(ctx). + if err = dao.BilWallets.Ctx(ctx). Where("tenant_id", tenantID). Fields("id, balance, frozen_balance"). Scan(&wallet); err != nil { @@ -430,7 +430,7 @@ func (s *sAdmin) AdjustBalance(ctx context.Context, req *v1.AdminWalletAdjustReq } // 记录流水 - if _, err = tx.Model("bil_transactions").Ctx(ctx).Insert(do.BilTransactions{ + if _, err = dao.BilTransactions.Ctx(ctx).Insert(do.BilTransactions{ TenantId: tenantID, WalletId: wallet.ID, Type: "adjust", diff --git a/internal/logic/admin/data_governance.go b/internal/logic/admin/data_governance.go index b3b5de0..851b2e5 100644 --- a/internal/logic/admin/data_governance.go +++ b/internal/logic/admin/data_governance.go @@ -152,10 +152,12 @@ type FileCleanupResult struct { } // deleteExpiredFiles 删除 created_at 早于 cutoff 的 fil_files 行及其存储对象。 -// exports=true 匹配 storage_path LIKE 'exports/%'（导出文件）；exports=false 匹配 -// 其余（AI re-host 图片等 provider 上传的文件）。删除经 FileService.Delete 完成， -// 会一并删除桶中对象（而非仅删库行，避免存储泄漏）。fileSvc 为 nil 时（对象存储未 -// 配置）退化为仅删库行。分批处理以限制内存与单次事务规模。 +// exports=true 匹配导出文件（storage_path LIKE 'exports/%'）；exports=false 只匹配 +// **图片**文件（mime_type LIKE 'image/%' 且非导出）——file_image_retention_days 的 +// 语义即「图片保留期」，绝不能连带删除非图片的 provider 上传文件（附件/头像等），否则 +// 会误删用户数据。删除经 FileService.Delete 完成，会一并删除桶中对象（而非仅删库行， +// 避免存储泄漏）。fileSvc 为 nil 时（对象存储未配置）退化为仅删库行。分批处理以限制 +// 内存与单次事务规模。 func deleteExpiredFiles(ctx context.Context, fileSvc *common.FileService, cutoff time.Time, exports bool) (int, error) { const batchSize = 500 total := 0 @@ -164,7 +166,9 @@ func deleteExpiredFiles(ctx context.Context, fileSvc *common.FileService, cutoff if exports { m = m.Where("storage_path LIKE ?", "exports/%") } else { - m = m.Where("storage_path NOT LIKE ?", "exports/%") + // 只清理图片：按 mime_type 精确限定，不再用「非导出」笼统匹配所有上传文件。 + m = m.Where("storage_path NOT LIKE ?", "exports/%"). + Where("mime_type LIKE ?", "image/%") } var batch []struct { @@ -229,8 +233,8 @@ func CleanupExpiredExportFiles(ctx context.Context) error { return err } -// CleanupExpiredImages 清理过期的 AI re-host 图片及其它 provider 上传文件（含存储对象）。 -// file_image_retention_days<=0 表示关闭（默认），直接跳过。 +// CleanupExpiredImages 清理过期的图片文件（含存储对象）。仅删除 mime_type 为 image/* 的文件， +// 不影响导出文件与其它非图片上传。file_image_retention_days<=0 表示关闭（默认），直接跳过。 func CleanupExpiredImages(ctx context.Context) (int, error) { retentionDays := common.Config().GetInt(ctx, "file_image_retention_days") if retentionDays <= 0 { @@ -345,14 +349,14 @@ func cleanupDeactivatedTenants(ctx context.Context) error { // remains consistent (no partial anonymization/deletion). err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { tenantID := t.ID - if _, err := tx.Model("tnt_users").Ctx(ctx).Where("tenant_id", tenantID). + if _, err := dao.TntUsers.Ctx(ctx).Where("tenant_id", tenantID). Data(do.TntUsers{DisplayName: "[deleted]", Email: fmt.Sprintf("deleted_%d@deleted.local", tenantID)}).Update(); err != nil { return gerror.Wrapf(err, "anonymize users for tenant %d", tenantID) } - if _, err := tx.Model("api_keys").Ctx(ctx).Where("tenant_id", tenantID).Delete(); err != nil { + if _, err := dao.ApiKeys.Ctx(ctx).Where("tenant_id", tenantID).Delete(); err != nil { return gerror.Wrapf(err, "delete api keys for tenant %d", tenantID) } - if _, err := tx.Model("tnt_tenants").Ctx(ctx).Where("id", tenantID). + if _, err := dao.TntTenants.Ctx(ctx).Where("id", tenantID). Data(do.TntTenants{DataRemovalAt: gtime.Now()}).Update(); err != nil { return gerror.Wrapf(err, "update data removal for tenant %d", tenantID) } diff --git a/internal/logic/admin/member.go b/internal/logic/admin/member.go index 1cde71c..b670bb3 100644 --- a/internal/logic/admin/member.go +++ b/internal/logic/admin/member.go @@ -8,6 +8,7 @@ import ( "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" v1 "github.com/qianfree/team-api/api/admin/v1" @@ -41,13 +42,13 @@ func (s *sAdmin) CreateMember(ctx context.Context, req *v1.AdminMemberCreateReq) var userID int64 - err = dao.TntTenants.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Check tenant exists and status var tenant *struct { Id int64 `json:"id"` Status string `json:"status"` } - err := tx.Model("tnt_tenants").Ctx(ctx). + err := dao.TntTenants.Ctx(ctx). Where("id", req.TenantID).Scan(&tenant) if err = common.IgnoreScanNoRows(err); err != nil { return err @@ -64,7 +65,7 @@ func (s *sAdmin) CreateMember(ctx context.Context, req *v1.AdminMemberCreateReq) if err != nil { return err } - memberCount, err := tx.Model("tnt_users").Ctx(ctx). + memberCount, err := dao.TntUsers.Ctx(ctx). Where("tenant_id", req.TenantID). Where("status", "active"). Count() @@ -77,7 +78,7 @@ func (s *sAdmin) CreateMember(ctx context.Context, req *v1.AdminMemberCreateReq) } // Check username uniqueness within tenant - count, err := tx.Model("tnt_users").Ctx(ctx). + count, err := dao.TntUsers.Ctx(ctx). Where("tenant_id", req.TenantID). Where("username", username).Count() if err != nil { @@ -88,7 +89,7 @@ func (s *sAdmin) CreateMember(ctx context.Context, req *v1.AdminMemberCreateReq) } // Check email uniqueness within tenant - count, err = tx.Model("tnt_users").Ctx(ctx). + count, err = dao.TntUsers.Ctx(ctx). Where("tenant_id", req.TenantID). Where("email", email).Count() if err != nil { @@ -99,7 +100,7 @@ func (s *sAdmin) CreateMember(ctx context.Context, req *v1.AdminMemberCreateReq) } // Create user - userResult, err := tx.Model("tnt_users").Ctx(ctx).Data(do.TntUsers{ + userResult, err := dao.TntUsers.Ctx(ctx).Data(do.TntUsers{ TenantId: req.TenantID, Username: username, Email: email, @@ -213,8 +214,8 @@ func (s *sAdmin) DisableMember(ctx context.Context, req *v1.AdminMemberDisableRe } // Revoke all active API keys and disable user in transaction - err = dao.TntUsers.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - _, err := tx.Model("api_keys").Ctx(ctx). + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + _, err := dao.ApiKeys.Ctx(ctx). Where("user_id", req.Id). Where("status", "active"). Data(do.ApiKeys{ @@ -224,7 +225,7 @@ func (s *sAdmin) DisableMember(ctx context.Context, req *v1.AdminMemberDisableRe return err } - _, err = tx.Model("tnt_users").Ctx(ctx). + _, err = dao.TntUsers.Ctx(ctx). Where("id", req.Id). Data(do.TntUsers{ Status: "disabled", @@ -257,8 +258,8 @@ func (s *sAdmin) EnableMember(ctx context.Context, req *v1.AdminMemberEnableReq) } // Restore disabled API keys and enable user in transaction - err = dao.TntUsers.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - _, err := tx.Model("api_keys").Ctx(ctx). + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + _, err := dao.ApiKeys.Ctx(ctx). Where("user_id", req.Id). Where("status", "disabled"). Data(do.ApiKeys{ @@ -268,7 +269,7 @@ func (s *sAdmin) EnableMember(ctx context.Context, req *v1.AdminMemberEnableReq) return err } - _, err = tx.Model("tnt_users").Ctx(ctx). + _, err = dao.TntUsers.Ctx(ctx). Where("id", req.Id). Data(do.TntUsers{ Status: "active", diff --git a/internal/logic/admin/model.go b/internal/logic/admin/model.go index 0463c8e..b101769 100644 --- a/internal/logic/admin/model.go +++ b/internal/logic/admin/model.go @@ -6,6 +6,7 @@ import ( "math" "strings" + "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/errors/gcode" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" @@ -339,29 +340,38 @@ func (s *sAdmin) UpdateModel(ctx context.Context, req *v1.ModelUpdateReq) (*v1.M // DeleteModel 删除模型（同时删除定价记录、租户分配记录和分组关联） func (s *sAdmin) DeleteModel(ctx context.Context, req *v1.ModelDeleteReq) (*v1.ModelDeleteRes, error) { - if _, err := dao.MdlPricing.Ctx(ctx).Where("model_id", req.ID).Delete(); err != nil { - return nil, gerror.Wrapf(err, "delete pricing tiers for model %d", req.ID) - } - if _, err := dao.MdlTenantModels.Ctx(ctx).Where("model_id", req.ID).Delete(); err != nil { - return nil, gerror.Wrapf(err, "delete tenant models for model %d", req.ID) - } - - // 级联删除：从所有分组中移除该模型，并清除受影响租户的缓存 + // 先查出受影响的分组，供事务提交后清理缓存（缓存失效不可回滚，故放在事务外） var affectedGroups []struct { GroupId int64 `json:"group_id"` } dao.MdlGroupModels.Ctx(ctx).Where("model_id", req.ID).Fields("group_id").Scan(&affectedGroups) - if _, err := dao.MdlGroupModels.Ctx(ctx).Where("model_id", req.ID).Delete(); err != nil { - return nil, gerror.Wrapf(err, "delete group models for model %d", req.ID) + + // 四表删除（定价 / 租户分配 / 分组关联 / 模型本体）放入同一事务， + // 避免中途失败留下「模型已删但定价/分配记录残留」的孤儿数据。 + err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + if _, err := dao.MdlPricing.Ctx(ctx).Where("model_id", req.ID).Delete(); err != nil { + return gerror.Wrapf(err, "delete pricing tiers for model %d", req.ID) + } + if _, err := dao.MdlTenantModels.Ctx(ctx).Where("model_id", req.ID).Delete(); err != nil { + return gerror.Wrapf(err, "delete tenant models for model %d", req.ID) + } + if _, err := dao.MdlGroupModels.Ctx(ctx).Where("model_id", req.ID).Delete(); err != nil { + return gerror.Wrapf(err, "delete group models for model %d", req.ID) + } + if _, err := dao.MdlModels.Ctx(ctx).Where("id", req.ID).Delete(); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err } + + // 事务提交后再清除受影响租户的缓存 for _, ag := range affectedGroups { invalidateTenantsInGroup(ctx, ag.GroupId) } - _, err := dao.MdlModels.Ctx(ctx).Where("id", req.ID).Delete() - if err != nil { - return nil, err - } return nil, nil } @@ -415,28 +425,35 @@ func (s *sAdmin) GetModelPricing(ctx context.Context, req *v1.PricingGetReq) (*v // SetModelPricing 设置模型定价（全量替换） func (s *sAdmin) SetModelPricing(ctx context.Context, req *v1.PricingSetReq) (*v1.PricingSetRes, error) { - _, err := dao.MdlPricing.Ctx(ctx).Where("model_id", req.ModelID).Delete() + // 全量替换：先删旧定价再插新定价，放入同一事务。否则若删除后插入中途失败， + // 该模型会残留「无定价」状态，导致计费失败或回退到默认价。 + err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + if _, err := dao.MdlPricing.Ctx(ctx).Where("model_id", req.ModelID).Delete(); err != nil { + return err + } + + for _, item := range req.Items { + if _, err := dao.MdlPricing.Ctx(ctx).Insert(do.MdlPricing{ + ModelId: req.ModelID, + BillingMode: item.BillingMode, + MinTokens: item.MinTokens, + MaxTokens: item.MaxTokens, + InputPrice: item.InputPrice, + OutputPrice: item.OutputPrice, + PerRequestPrice: item.PerRequestPrice, + CacheReadPrice: item.CacheReadPrice, + CacheCreationPrice: item.CacheCreationPrice, + }); err != nil { + return err + } + } + + return nil + }) if err != nil { return nil, err } - for _, item := range req.Items { - _, err := dao.MdlPricing.Ctx(ctx).Insert(do.MdlPricing{ - ModelId: req.ModelID, - BillingMode: item.BillingMode, - MinTokens: item.MinTokens, - MaxTokens: item.MaxTokens, - InputPrice: item.InputPrice, - OutputPrice: item.OutputPrice, - PerRequestPrice: item.PerRequestPrice, - CacheReadPrice: item.CacheReadPrice, - CacheCreationPrice: item.CacheCreationPrice, - }) - if err != nil { - return nil, err - } - } - return nil, nil } diff --git a/internal/logic/admin/model_import_export.go b/internal/logic/admin/model_import_export.go index 452d74a..716f030 100644 --- a/internal/logic/admin/model_import_export.go +++ b/internal/logic/admin/model_import_export.go @@ -212,12 +212,12 @@ func (s *sAdmin) ImportModels(ctx context.Context, req *v1.ModelImportReq) (*v1. } res := &v1.ModelImportRes{} - err := dao.MdlModels.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { for _, item := range req.Models { var existing *struct { ID int64 `orm:"id"` } - err := tx.Model("mdl_models").Ctx(ctx).Where("model_id", item.ModelId).Scan(&existing) + err := dao.MdlModels.Ctx(ctx).Where("model_id", item.ModelId).Scan(&existing) if err != nil { return err } @@ -252,17 +252,17 @@ func (s *sAdmin) ImportModels(ctx context.Context, req *v1.ModelImportReq) (*v1. updateData.Status = item.Status } - _, err = tx.Model("mdl_models").Ctx(ctx).Where("id", existing.ID).Data(updateData).Update() + _, err = dao.MdlModels.Ctx(ctx).Where("id", existing.ID).Data(updateData).Update() if err != nil { return err } - _, err = tx.Model("mdl_pricing").Ctx(ctx).Where("model_id", existing.ID).Delete() + _, err = dao.MdlPricing.Ctx(ctx).Where("model_id", existing.ID).Delete() if err != nil { return err } for _, p := range item.Pricing { - _, err = tx.Model("mdl_pricing").Ctx(ctx).Insert(do.MdlPricing{ + _, err = dao.MdlPricing.Ctx(ctx).Insert(do.MdlPricing{ ModelId: existing.ID, BillingMode: p.BillingMode, MinTokens: p.MinTokens, @@ -301,13 +301,13 @@ func (s *sAdmin) ImportModels(ctx context.Context, req *v1.ModelImportReq) (*v1. insertData.Capabilities = string(capJson) } - id, err := tx.Model("mdl_models").Ctx(ctx).InsertAndGetId(insertData) + id, err := dao.MdlModels.Ctx(ctx).InsertAndGetId(insertData) if err != nil { return err } for _, p := range item.Pricing { - _, err = tx.Model("mdl_pricing").Ctx(ctx).Insert(do.MdlPricing{ + _, err = dao.MdlPricing.Ctx(ctx).Insert(do.MdlPricing{ ModelId: id, BillingMode: p.BillingMode, MinTokens: p.MinTokens, diff --git a/internal/logic/admin/order.go b/internal/logic/admin/order.go index af01b1c..392defe 100644 --- a/internal/logic/admin/order.go +++ b/internal/logic/admin/order.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" v1 "github.com/qianfree/team-api/api/admin/v1" "github.com/qianfree/team-api/internal/dao" @@ -82,8 +83,8 @@ func (s *sAdmin) RefundOrder(ctx context.Context, req *v1.OrderRefundReq) (*v1.O return nil, common.NewBadRequestError("订单状态不支持退款") } - err = dao.OrdOrders.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - _, err := tx.Model("ord_orders").Ctx(ctx). + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + _, err := dao.OrdOrders.Ctx(ctx). Where("id", req.Id). Data(do.OrdOrders{ Status: "refunding", @@ -92,7 +93,7 @@ func (s *sAdmin) RefundOrder(ctx context.Context, req *v1.OrderRefundReq) (*v1.O return err } - _, err = tx.Model("ord_refunds").Ctx(ctx).Insert(do.OrdRefunds{ + _, err = dao.OrdRefunds.Ctx(ctx).Insert(do.OrdRefunds{ OrderId: req.Id, TenantId: order.TenantID, Amount: order.FinalAmount, diff --git a/internal/logic/admin/permission.go b/internal/logic/admin/permission.go index 1a55b15..2634ee0 100644 --- a/internal/logic/admin/permission.go +++ b/internal/logic/admin/permission.go @@ -7,6 +7,7 @@ import ( do "github.com/qianfree/team-api/internal/model/do" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" v1 "github.com/qianfree/team-api/api/admin/v1" "github.com/qianfree/team-api/internal/dao" "github.com/qianfree/team-api/internal/logic/common" @@ -215,9 +216,10 @@ func (s *sAdmin) UpdateUserPermissions(ctx context.Context, req *v1.AdminPermiss return nil, common.NewBadRequestError("超级管理员无需配置权限") } - err = dao.SysAdminUsers.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // 事务采用 ctx 传播式写法：闭包内统一使用 dao.Xxx.Ctx(ctx)，事务由 ctx 自动挂载。 + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Delete existing permissions - _, err := tx.Model("sys_admin_role_perms").Ctx(ctx). + _, err := dao.SysAdminRolePerms.Ctx(ctx). Where("admin_user_id", req.Id). Delete() if err != nil { @@ -241,7 +243,7 @@ func (s *sAdmin) UpdateUserPermissions(ctx context.Context, req *v1.AdminPermiss PermissionPoint: p, } } - _, err = tx.Model("sys_admin_role_perms").Ctx(ctx).Data(data).Insert() + _, err = dao.SysAdminRolePerms.Ctx(ctx).Data(data).Insert() if err != nil { return err } @@ -274,9 +276,10 @@ func (s *sAdmin) UpdateUserDataScopes(ctx context.Context, req *v1.AdminDataScop return nil, common.NewBadRequestError("超级管理员无需配置数据范围") } - err = dao.SysAdminUsers.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // 事务采用 ctx 传播式写法：闭包内统一使用 dao.Xxx.Ctx(ctx)，事务由 ctx 自动挂载。 + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Delete existing data scopes - _, err := tx.Model("sys_admin_data_scopes").Ctx(ctx). + _, err := dao.SysAdminDataScopes.Ctx(ctx). Where("admin_user_id", req.Id). Delete() if err != nil { @@ -293,7 +296,7 @@ func (s *sAdmin) UpdateUserDataScopes(ctx context.Context, req *v1.AdminDataScop ScopeValue: sc.ScopeValue, } } - _, err = tx.Model("sys_admin_data_scopes").Ctx(ctx).Data(data).Insert() + _, err = dao.SysAdminDataScopes.Ctx(ctx).Data(data).Insert() if err != nil { return err } diff --git a/internal/logic/admin/redemption.go b/internal/logic/admin/redemption.go index a0cb1f6..c7a7c3c 100644 --- a/internal/logic/admin/redemption.go +++ b/internal/logic/admin/redemption.go @@ -8,6 +8,7 @@ import ( "time" "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" v1 "github.com/qianfree/team-api/api/admin/v1" "github.com/qianfree/team-api/internal/dao" @@ -57,14 +58,14 @@ func (s *sAdmin) BatchCreateRedemptions(ctx context.Context, req *v1.RedemptionC batchNo := fmt.Sprintf("BATCH%s%04d", now.Format("YmdHis"), now.UnixNano()%10000) created := 0 - err := dao.OrdRedemptions.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { for i := 0; i < count; i++ { code, err := generateCode(12) if err != nil { return err } - _, err = tx.Model("ord_redemptions").Ctx(ctx).Insert(do.OrdRedemptions{ + _, err = dao.OrdRedemptions.Ctx(ctx).Insert(do.OrdRedemptions{ Code: code, Type: codeType, Value: req.Value, diff --git a/internal/logic/admin/settings.go b/internal/logic/admin/settings.go index 0ddcc3d..28856ff 100644 --- a/internal/logic/admin/settings.go +++ b/internal/logic/admin/settings.go @@ -1,8 +1,18 @@ package admin import ( + "bytes" "context" + "fmt" + "image" + "image/color" + "image/png" + "io" "strings" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/google/uuid" v1 "github.com/qianfree/team-api/api/admin/v1" "github.com/qianfree/team-api/internal/logic/common" @@ -70,10 +80,18 @@ func (s *sAdmin) UpdateSettings(ctx context.Context, req *v1.AdminSettingsUpdate // validateCrossFieldSettings validates interdependent settings within a category. func (s *sAdmin) validateCrossFieldSettings(ctx context.Context, category string, values map[string]string) error { - if category != "security" { + switch category { + case "security": + return s.validateSecuritySettings(ctx, values) + case "channel": + return s.validateChannelSettings(ctx, values) + default: return nil } +} +// validateSecuritySettings 校验安全配置项之间的依赖：启用 Turnstile 前必须已配置密钥。 +func (s *sAdmin) validateSecuritySettings(ctx context.Context, values map[string]string) error { enabled, ok := values["turnstile_enabled"] if !ok || (enabled != "true" && enabled != "1") { return nil @@ -97,6 +115,22 @@ func (s *sAdmin) validateCrossFieldSettings(ctx context.Context, category string return nil } +// validateChannelSettings 校验渠道配置项之间的依赖：同步图片厂商异步化依赖对象存储保存生成 +// 结果，开启前必须已在「存储配置」中配置对象存储（OSS/S3/COS），否则生成的图片无处保存。 +func (s *sAdmin) validateChannelSettings(ctx context.Context, values map[string]string) error { + enabled, ok := values["sync_image_async_enabled"] + if !ok || (enabled != "true" && enabled != "1") { + return nil + } + + // 存储配置属于 storage 分类、独立持久化，此处直接读已保存的存储配置判断是否可用。 + if !common.IsStorageConfigured(ctx) { + return common.NewBadRequestError("开启同步图片厂商异步化前，必须先在「存储配置」中配置对象存储（OSS/S3/COS）") + } + + return nil +} + func isValidCategory(category string) bool { for _, c := range common.Categories { if c.Key == category { @@ -105,3 +139,98 @@ func isValidCategory(category string) bool { } return false } + +// TestStorageConfig 测试对象存储配置连通性：用生成的测试图片走一次「上传 → 下载校验 → 删除」 +// 完整往返，验证凭证 / 桶 / 端点 / 读写删权限是否正常。 +// +// access_key_id / secret 为空或掩码（"******"）时回落到已保存的配置值——与 GET 接口对敏感字段 +// 的掩码逻辑（config.GetCategoryWithValues）以及 Turnstile 校验的 "******" 处理保持一致， +// 从而支持管理员在「未点保存」的状态下直接测试表单中新填/改动的配置。 +func (s *sAdmin) TestStorageConfig(ctx context.Context, req *v1.AdminStorageTestReq) (*v1.AdminStorageTestRes, error) { + cfg := &common.StorageConfig{ + Provider: strings.TrimSpace(req.Provider), + Endpoint: strings.TrimSpace(req.Endpoint), + Region: strings.TrimSpace(req.Region), + Bucket: strings.TrimSpace(req.Bucket), + AccessKeyID: strings.TrimSpace(req.AccessKeyID), + SecretKey: req.SecretKey, + UseSSL: req.UseSSL, + PathPrefix: strings.TrimSpace(req.PathPrefix), + } + // 敏感字段掩码 / 留空 → 回落已保存值 + if cfg.AccessKeyID == "" || cfg.AccessKeyID == "******" { + cfg.AccessKeyID = common.Config().GetString(ctx, "storage_access_key_id") + } + if cfg.SecretKey == "" || cfg.SecretKey == "******" { + cfg.SecretKey = common.Config().GetString(ctx, "storage_access_key_secret") + } + if cfg.PathPrefix == "" { + cfg.PathPrefix = "team-api" // 与 GetStorageConfig 的默认前缀保持一致 + } + + if cfg.Provider == "" || cfg.Bucket == "" { + return nil, common.NewBadRequestError("请先填写存储供应商和存储桶名称") + } + + provider, err := common.NewStorageProvider(cfg) + if err != nil { + return nil, common.NewBadRequestError("创建存储客户端失败：" + err.Error()) + } + + // 生成一张 1x1 PNG 作为测试图片（真实图片字节，验证二进制内容往返无损） + img := image.NewRGBA(image.Rect(0, 0, 1, 1)) + img.Set(0, 0, color.RGBA{R: 0x33, G: 0x99, B: 0xff, A: 0xff}) + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + return nil, common.NewBadRequestError("生成测试图片失败：" + err.Error()) + } + payload := buf.Bytes() + + // 独立 _healthcheck 目录 + uuid，避免与业务对象冲突；provider 内部会自动加路径前缀 + key := fmt.Sprintf("_healthcheck/%s.png", uuid.New().String()) + + // 整个往返设 20s 超时，端点不可达时不至于长时间挂起 + testCtx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + res := &v1.AdminStorageTestRes{} + start := time.Now() + + // 1. 上传 + if _, err := provider.Upload(testCtx, bytes.NewReader(payload), key, "image/png"); err != nil { + return nil, common.NewBadRequestError("上传测试图片失败：" + err.Error()) + } + res.Uploaded = true + // 兜底清理：覆盖下载失败等提前返回路径，避免遗留测试对象（Delete 幂等，成功路径重复删除无副作用） + defer func() { + if delErr := provider.Delete(context.WithoutCancel(testCtx), key); delErr != nil { + g.Log().Warningf(ctx, "storage test: cleanup test object %s failed: %v", key, delErr) + } + }() + + // 2. 下载并校验内容一致 + rc, err := provider.Download(testCtx, key) + if err != nil { + return nil, common.NewBadRequestError("下载测试图片失败：" + err.Error()) + } + got, readErr := io.ReadAll(rc) + _ = rc.Close() + if readErr != nil { + return nil, common.NewBadRequestError("读取测试图片失败：" + readErr.Error()) + } + if !bytes.Equal(got, payload) { + return nil, common.NewBadRequestError("下载内容与上传不一致，存储未正确保存对象") + } + res.Downloaded = true + + // 3. 删除（校验删除权限；失败不致命，仅告警，兜底 defer 会再尝试一次） + if err := provider.Delete(testCtx, key); err != nil { + g.Log().Warningf(ctx, "storage test: delete test object %s failed: %v", key, err) + } else { + res.Deleted = true + } + + res.ElapsedMs = time.Since(start).Milliseconds() + res.Message = "对象存储配置正常：测试图片上传、下载校验通过" + return res, nil +} diff --git a/internal/logic/admin/task_management.go b/internal/logic/admin/task_management.go index f411c1a..9d9e991 100644 --- a/internal/logic/admin/task_management.go +++ b/internal/logic/admin/task_management.go @@ -147,6 +147,15 @@ func (s *sAdmin) TaskDetail(ctx context.Context, req *v1.TaskDetailReq) (*v1.Tas item.FinishTime = task.FinishTime.Format("Y-m-d H:i:s") } + // 对 re-host 到对象存储的结果图，生成新鲜的缩略图 URL 供详情弹窗内联预览，并刷新原图 URL + // （规避 result_url 24h 预签名过期）。非图片/上游直链任务查不到文件记录，保持原 result_url。 + if thumb, orig := common.TaskResultImageURLs(ctx, task.TenantId, task.PublicTaskId, 600); thumb != "" { + item.ResultThumbURL = thumb + if orig != "" { + item.ResultURL = orig + } + } + return &v1.TaskDetailRes{ Task: item, }, nil diff --git a/internal/logic/admin/tenant.go b/internal/logic/admin/tenant.go index b3d8de8..8d09662 100644 --- a/internal/logic/admin/tenant.go +++ b/internal/logic/admin/tenant.go @@ -125,8 +125,8 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 var tenantID int64 - err = dao.TntTenants.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - tenantResult, err := tx.Model("tnt_tenants").Ctx(ctx).Data(do.TntTenants{ + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + tenantResult, err := dao.TntTenants.Ctx(ctx).Data(do.TntTenants{ Name: tenantName, Code: tenantCode, MaxMembers: maxMembersVal, @@ -142,7 +142,7 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 return gerror.Wrapf(err, "get tenant id") } - userResult, err := tx.Model("tnt_users").Ctx(ctx).Data(do.TntUsers{ + userResult, err := dao.TntUsers.Ctx(ctx).Data(do.TntUsers{ TenantId: tenantID, Username: username, Email: email, @@ -158,7 +158,7 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 return gerror.Wrapf(err, "get owner user id") } - _, err = tx.Model("tnt_tenants").Ctx(ctx). + _, err = dao.TntTenants.Ctx(ctx). Where("id", tenantID). Data(do.TntTenants{ OwnerUserId: ownerUserID, @@ -167,7 +167,7 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 return gerror.Wrapf(err, "set tenant owner") } - _, err = tx.Model("bil_wallets").Ctx(ctx).Data(do.BilWallets{ + _, err = dao.BilWallets.Ctx(ctx).Data(do.BilWallets{ TenantId: tenantID, Balance: 0, FrozenBalance: 0, @@ -181,7 +181,7 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 var defaultGroups []struct { Id int64 } - if err := tx.Model("mdl_model_groups").Ctx(ctx). + if err := dao.MdlModelGroups.Ctx(ctx). Where("is_default", true).Where("status", "active"). Fields("id").Scan(&defaultGroups); err == nil && len(defaultGroups) > 0 { insertData := make([]do.MdlTenantGroups, 0, len(defaultGroups)) @@ -191,7 +191,7 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 GroupId: dg.Id, }) } - if _, err := tx.Model("mdl_tenant_groups").Ctx(ctx). + if _, err := dao.MdlTenantGroups.Ctx(ctx). Batch(len(insertData)).Insert(insertData); err != nil { g.Log().Warningf(ctx, "assign default model groups failed: %v", err) } @@ -206,6 +206,57 @@ func (s *sAdmin) CreateTenant(ctx context.Context, req *v1.TenantCreateReq) (*v1 return &v1.TenantCreateRes{Id: tenantID}, nil } +// batchTenantAggregates 批量查询一批租户的 owner 名称、成员数、钱包余额， +// 用三条聚合查询替代逐行 N+1 查询。 +// 返回：ownerNames(ownerUserID→名称)、memberCounts(tenantID→成员数)、walletBalances(tenantID→余额)。 +func batchTenantAggregates(ctx context.Context, tenantIDs, ownerIDs []int64) ( + ownerNames map[int64]string, + memberCounts map[int64]int, + walletBalances map[int64]string, +) { + ownerNames = make(map[int64]string, len(ownerIDs)) + memberCounts = make(map[int64]int, len(tenantIDs)) + walletBalances = make(map[int64]string, len(tenantIDs)) + + // owner 名称：按 owner_user_id 批量取 + if len(ownerIDs) > 0 { + var owners []struct { + Id int64 `json:"id"` + DisplayName string `json:"display_name"` + } + _ = dao.TntUsers.Ctx(ctx).Fields("id, display_name").WhereIn("id", ownerIDs).Scan(&owners) + for _, o := range owners { + ownerNames[o.Id] = o.DisplayName + } + } + + if len(tenantIDs) > 0 { + // 成员数：按 tenant_id 分组统计 + var counts []struct { + TenantID int64 `json:"tenant_id"` + Count int `json:"count"` + } + _ = dao.TntUsers.Ctx(ctx).Fields("tenant_id, COUNT(*) AS count"). + WhereIn("tenant_id", tenantIDs).Group("tenant_id").Scan(&counts) + for _, c := range counts { + memberCounts[c.TenantID] = c.Count + } + + // 钱包余额：按 tenant_id 批量取 + var wallets []struct { + TenantID int64 `json:"tenant_id"` + Balance string `json:"balance"` + } + _ = dao.BilWallets.Ctx(ctx).Fields("tenant_id, balance"). + WhereIn("tenant_id", tenantIDs).Scan(&wallets) + for _, w := range wallets { + walletBalances[w.TenantID] = w.Balance + } + } + + return +} + // ListTenants returns a paginated list of tenants. func (s *sAdmin) ListTenants(ctx context.Context, req *v1.TenantListReq) (*v1.TenantListRes, error) { page, pageSize := common.NormalizePagination(req.Page, req.PageSize) @@ -270,6 +321,16 @@ func (s *sAdmin) ListTenants(ctx context.Context, req *v1.TenantListReq) (*v1.Te } items := make([]v1.TenantItem, len(tenants)) + + // 批量查询 owner 名称 / 成员数 / 钱包余额（避免逐行 N+1） + tenantIDs := make([]int64, len(tenants)) + ownerIDs := make([]int64, len(tenants)) + for i, t := range tenants { + tenantIDs[i] = t.Id + ownerIDs[i] = t.OwnerUserID + } + ownerNameMap, memberCountMap, walletBalanceMap := batchTenantAggregates(ctx, tenantIDs, ownerIDs) + for i, t := range tenants { item := v1.TenantItem{ ID: t.Id, @@ -303,29 +364,11 @@ func (s *sAdmin) ListTenants(ctx context.Context, req *v1.TenantListReq) (*v1.Te item.EffectiveMaxConcurrency = 0 } - // Get owner name - var owner *struct { - DisplayName string `json:"display_name"` - } - _ = dao.TntUsers.Ctx(ctx). - Where("id", t.OwnerUserID).Scan(&owner) - if owner != nil { - item.OwnerName = owner.DisplayName - } - - // Get member count - memberCount, _ := dao.TntUsers.Ctx(ctx). - Where("tenant_id", t.Id).Count() - item.MemberCount = memberCount - - // Get wallet balance - var wallet *struct { - Balance string `json:"balance"` - } - _ = dao.BilWallets.Ctx(ctx). - Where("tenant_id", t.Id).Scan(&wallet) - if wallet != nil && wallet.Balance != "" { - item.WalletBalance = wallet.Balance + // 从批量结果中取 owner 名称 / 成员数 / 钱包余额 + item.OwnerName = ownerNameMap[t.OwnerUserID] + item.MemberCount = memberCountMap[t.Id] + if bal, ok := walletBalanceMap[t.Id]; ok && bal != "" { + item.WalletBalance = bal } else { item.WalletBalance = "0" } @@ -550,47 +593,6 @@ func (s *sAdmin) ExportTenants(ctx context.Context, req *v1.TenantExportReq) (*v Columns: columns, } - fetchTenantRow := func(t struct { - Id int64 `json:"id"` - Name string `json:"name"` - Code string `json:"code"` - OwnerUserID int64 `json:"owner_user_id"` - Status string `json:"status"` - CreatedAt *gtime.Time `json:"created_at"` - }) map[string]any { - var owner *struct { - DisplayName string `json:"display_name"` - } - _ = dao.TntUsers.Ctx(ctx).Where("id", t.OwnerUserID).Scan(&owner) - - memberCount, _ := dao.TntUsers.Ctx(ctx).Where("tenant_id", t.Id).Count() - - var wallet *struct { - Balance string `json:"balance"` - } - _ = dao.BilWallets.Ctx(ctx).Where("tenant_id", t.Id).Scan(&wallet) - walletBalance := "0" - if wallet != nil && wallet.Balance != "" { - walletBalance = wallet.Balance - } - - ownerName := "" - if owner != nil { - ownerName = owner.DisplayName - } - - return map[string]any{ - "id": t.Id, - "name": t.Name, - "code": t.Code, - "owner_name": ownerName, - "status": t.Status, - "member_count": memberCount, - "wallet_balance": walletBalance, - "created_at": t.CreatedAt.String(), - } - } - return nil, export.GenericExport(ctx, config, func(yield func(map[string]any) bool) { offset := 0 for { @@ -613,8 +615,32 @@ func (s *sAdmin) ExportTenants(ctx context.Context, req *v1.TenantExportReq) (*v if err := m.Fields("id, name, code, owner_user_id, status, created_at").OrderDesc("id").Limit(1000).Offset(offset).Scan(&batch); err != nil { return } + + // 批量查询本页的 owner 名称 / 成员数 / 钱包余额（避免逐行 N+1） + tenantIDs := make([]int64, len(batch)) + ownerIDs := make([]int64, len(batch)) + for i, t := range batch { + tenantIDs[i] = t.Id + ownerIDs[i] = t.OwnerUserID + } + ownerNameMap, memberCountMap, walletBalanceMap := batchTenantAggregates(ctx, tenantIDs, ownerIDs) + for _, t := range batch { - if !yield(fetchTenantRow(t)) { + walletBalance := "0" + if bal, ok := walletBalanceMap[t.Id]; ok && bal != "" { + walletBalance = bal + } + row := map[string]any{ + "id": t.Id, + "name": t.Name, + "code": t.Code, + "owner_name": ownerNameMap[t.OwnerUserID], + "status": t.Status, + "member_count": memberCountMap[t.Id], + "wallet_balance": walletBalance, + "created_at": t.CreatedAt.String(), + } + if !yield(row) { return } } diff --git a/internal/logic/billing/currency.go b/internal/logic/billing/currency.go index dc09ca1..d0537ed 100644 --- a/internal/logic/billing/currency.go +++ b/internal/logic/billing/currency.go @@ -2,7 +2,6 @@ package billing import ( "context" - "math" lcommon "github.com/qianfree/team-api/internal/logic/common" ) @@ -10,19 +9,18 @@ import ( const defaultCNYToUSD = 0.14 const defaultUSDToCNY = 7.25 -// usdPrecision 钱包余额精度：小数点后 6 位。 -const usdPrecision = 1e6 - // ConvertCNYToUSD 将人民币金额转换为美元，向上取整到小数点后 6 位。 +// A8：用 decimal 精确乘法替代 float64 链式运算，避免 cnyAmount×rate 的浮点误差。 func ConvertCNYToUSD(ctx context.Context, cnyAmount float64) float64 { rate := GetExchangeRateCNYToUSD(ctx) - return math.Ceil(cnyAmount*rate*usdPrecision) / usdPrecision + return ceilUSD(dec(cnyAmount).Mul(dec(rate))) } -// ConvertUSDToCNY 将美元金额转换为人民币 +// ConvertUSDToCNY 将美元金额转换为人民币。 +// A8：decimal 精确乘法 + 四舍五入到存储精度（10 位），消除 float64 累计误差。 func ConvertUSDToCNY(ctx context.Context, usdAmount float64) float64 { rate := GetExchangeRateUSDToCNY(ctx) - return usdAmount * rate + return roundMoney(dec(usdAmount).Mul(dec(rate))) } // GetExchangeRateCNYToUSD 获取 CNY→USD 兑换比例 diff --git a/internal/logic/billing/money.go b/internal/logic/billing/money.go new file mode 100644 index 0000000..e1857b3 --- /dev/null +++ b/internal/logic/billing/money.go @@ -0,0 +1,54 @@ +package billing + +import "github.com/shopspring/decimal" + +// 金额精度工具（A8 Phase 1）。 +// +// 背景：系统所有资金列均为 PostgreSQL NUMERIC(20,10)，DB 端加减为精确定点运算； +// 但 Go 层历史上用 float64 承载金额，链式乘除（如 tokens/1e6 × 单价 × 租户倍率、CNY×汇率） +// 会累积二进制浮点误差，违反「金额不用 FLOAT」规范。本文件提供 decimal 精确计算 + 统一 +// 四舍五入的边界工具：在纯计算处用 decimal 运算，最终在边界收敛到固定精度再落库/返回， +// 消除 float64 累计漂移。Phase 1 仅覆盖换汇与成本汇总，不改动生成的 entity 与 Redis Lua。 + +const ( + // moneyScale 金额存储精度：与 DB 列 NUMERIC(20,10) 对齐。 + moneyScale = 10 + // usdDisplayScale 钱包/展示精度：小数点后 6 位（见 CLAUDE.md 资金显示精度）。 + usdDisplayScale = 6 +) + +// dec 把 float64 安全转成 decimal。decimal.NewFromFloat 取能往返的最短十进制表示， +// 因此 0.1 → "0.1" 而非 0.1000000000000000055…，不会把浮点噪声带进精确运算。 +func dec(f float64) decimal.Decimal { + return decimal.NewFromFloat(f) +} + +// roundMoney 将 decimal 金额四舍五入到 10 位小数（与 NUMERIC(20,10) 对齐）后返回 float64。 +// 用于把 decimal 精确计算的结果收敛到存储精度，作为 Go 层金额的统一出口。 +func roundMoney(d decimal.Decimal) float64 { + f, _ := d.Round(moneyScale).Float64() + return f +} + +// ceilUSD 将 decimal 金额向上取整到 6 位小数（钱包 USD 精度）后返回 float64。 +// 用于 CNY→USD 充值入账：宁可多给用户零头也不少给，且与展示精度一致。 +func ceilUSD(d decimal.Decimal) float64 { + f, _ := d.RoundCeil(usdDisplayScale).Float64() + return f +} + +// microScale 整数微单位换算比例：1 USD = 1_000_000 micro（6 位小数，钱包/展示精度）。 +// Redis 钱包金额以整数 micro 存储（Phase 3），用整数加减（HINCRBY）替代 HINCRBYFLOAT， +// 彻底消除 IEEE double 累计漂移。int64 上限 ~9.2e18，按 micro 计可表示约 $9.2 万亿/租户，余量充足。 +const microScale = 1_000_000 + +// toMicro 将 USD 金额（float64）四舍五入换算为整数微单位（用 decimal 精确取整）。 +func toMicro(usd float64) int64 { + return dec(usd).Mul(decimal.NewFromInt(microScale)).Round(0).IntPart() +} + +// fromMicro 将整数微单位换算回 USD（float64）。 +func fromMicro(micro int64) float64 { + f, _ := decimal.New(micro, -6).Float64() + return f +} diff --git a/internal/logic/billing/money_test.go b/internal/logic/billing/money_test.go new file mode 100644 index 0000000..4cf6147 --- /dev/null +++ b/internal/logic/billing/money_test.go @@ -0,0 +1,89 @@ +package billing + +import ( + "testing" + + "github.com/shopspring/decimal" +) + +// TestRoundMoney 验证四舍五入到 10 位小数（对齐 NUMERIC(20,10)）。 +func TestRoundMoney(t *testing.T) { + cases := []struct { + in string + want float64 + }{ + {"0.30000000000000004", 0.3}, + {"0.12345678905", 0.1234567891}, // 第 11 位进位 + {"0.12345678904", 0.123456789}, // 第 11 位舍去 + {"1", 1}, + } + for _, c := range cases { + d, _ := decimal.NewFromString(c.in) + if got := roundMoney(d); got != c.want { + t.Fatalf("roundMoney(%s) = %v, want %v", c.in, got, c.want) + } + } +} + +// TestComputeCostDecimalPrecision 验证 token 成本用 decimal 精确计算： +// 100000 token × $3/1M = $0.3。float64 链式运算会得到 0.30000000000000004， +// decimal 精确运算 + 四舍五入到 10 位得到精确的 0.3。 +func TestComputeCostDecimalPrecision(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + InputPrice: 3.0, + OutputPrice: 0, + TenantMultiplier: 1.0, + Currency: "USD", + } + bd := computeCost(pricing, 100000, 0, nil) + if bd.TotalCost != 0.3 { + t.Fatalf("TotalCost = %v (%.20f), want exactly 0.3", bd.TotalCost, bd.TotalCost) + } + if bd.InputCost != 0.3 { + t.Fatalf("InputCost = %v, want 0.3", bd.InputCost) + } +} + +// TestMicroRoundTrip 验证 USD ↔ 整数微单位换算的正确性与精确取整。 +func TestMicroRoundTrip(t *testing.T) { + cases := []struct { + usd float64 + wantMicro int64 + }{ + {0, 0}, + {1, 1_000_000}, + {0.1, 100_000}, + {0.015, 15_000}, + {0.000001, 1}, + {0.0000004, 0}, // 四舍五入到 0 + {0.0000006, 1}, // 四舍五入到 1 micro + {12345.678901, 12_345_678_901}, + } + for _, c := range cases { + if got := toMicro(c.usd); got != c.wantMicro { + t.Fatalf("toMicro(%v) = %d, want %d", c.usd, got, c.wantMicro) + } + } + // fromMicro 反向 + if got := fromMicro(15_000); got != 0.015 { + t.Fatalf("fromMicro(15000) = %v, want 0.015", got) + } + if got := fromMicro(1_000_000); got != 1.0 { + t.Fatalf("fromMicro(1000000) = %v, want 1.0", got) + } +} + +// (100000/1M × $3) × 1.1 = 0.33。 +func TestComputeCostWithMultiplier(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + InputPrice: 3.0, + TenantMultiplier: 1.1, + Currency: "USD", + } + bd := computeCost(pricing, 100000, 0, nil) + if bd.TotalCost != 0.33 { + t.Fatalf("TotalCost = %v (%.20f), want exactly 0.33", bd.TotalCost, bd.TotalCost) + } +} diff --git a/internal/logic/billing/pricing.go b/internal/logic/billing/pricing.go index 49bb8f9..66cc7fc 100644 --- a/internal/logic/billing/pricing.go +++ b/internal/logic/billing/pricing.go @@ -8,6 +8,7 @@ import ( "time" "github.com/gogf/gf/v2/errors/gerror" + "github.com/shopspring/decimal" "github.com/qianfree/team-api/internal/dao" lcommon "github.com/qianfree/team-api/internal/logic/common" @@ -287,19 +288,25 @@ func computeCost(pricing *PricingResult, inputTokens, outputTokens int, usage *r // 输出费用 outputCost := computeOutputCost(pricing, outputTokens) - // Cache 费用 - cacheReadCost := float64(cacheReadTokens) / 1_000_000.0 * pricing.CacheReadPrice - cacheCreationCost := float64(cacheCreationTokens) / 1_000_000.0 * pricing.CacheCreationPrice + // A8：token 成本链式计算（÷1e6 × 单价 × 租户倍率 + 各项求和）改用 decimal 精确运算， + // 最终四舍五入到 10 位（NUMERIC(20,10)）再返回 float64，消除 float64 累计误差。 + million := decimal.NewFromInt(1_000_000) + mul := dec(pricing.TenantMultiplier) + + baseInputCostD := dec(baseInputCost) + outputCostD := dec(outputCost) + cacheReadCostD := decimal.NewFromInt(int64(cacheReadTokens)).Div(million).Mul(dec(pricing.CacheReadPrice)) + cacheCreationCostD := decimal.NewFromInt(int64(cacheCreationTokens)).Div(million).Mul(dec(pricing.CacheCreationPrice)) // 总费用 = (基础输入 + 输出 + cache各项) × 租户倍率 - subtotal := baseInputCost + outputCost + cacheReadCost + cacheCreationCost - totalCost := subtotal * pricing.TenantMultiplier + subtotalD := baseInputCostD.Add(outputCostD).Add(cacheReadCostD).Add(cacheCreationCostD) + totalCostD := subtotalD.Mul(mul) return &CostBreakdown{ - BaseCost: subtotal, - InputCost: baseInputCost * pricing.TenantMultiplier, - OutputCost: outputCost * pricing.TenantMultiplier, - TotalCost: totalCost, + BaseCost: roundMoney(subtotalD), + InputCost: roundMoney(baseInputCostD.Mul(mul)), + OutputCost: roundMoney(outputCostD.Mul(mul)), + TotalCost: roundMoney(totalCostD), InputTokens: inputTokens, OutputTokens: outputTokens, BillingMode: pricing.BillingMode, @@ -309,8 +316,8 @@ func computeCost(pricing *PricingResult, inputTokens, outputTokens int, usage *r Currency: pricing.Currency, CacheCreationTokens: cacheCreationTokens, CacheReadTokens: cacheReadTokens, - CacheCreationCost: cacheCreationCost * pricing.TenantMultiplier, - CacheReadCost: cacheReadCost * pricing.TenantMultiplier, + CacheCreationCost: roundMoney(cacheCreationCostD.Mul(mul)), + CacheReadCost: roundMoney(cacheReadCostD.Mul(mul)), } } diff --git a/internal/logic/billing/provider.go b/internal/logic/billing/provider.go index f1eb753..8104ba2 100644 --- a/internal/logic/billing/provider.go +++ b/internal/logic/billing/provider.go @@ -3,6 +3,8 @@ package billing import ( "context" + "github.com/gogf/gf/v2/frame/g" + "github.com/qianfree/team-api/relay/common" ) @@ -44,6 +46,11 @@ func (b *BillingProviderImpl) SettleWithUsage(ctx context.Context, tenantID, use result, err := SettleWithUsage(ctx, tenantID, userID, apiKeyID, channelID, modelName, requestID, relayMode, usage, preDeductAmount, relayInfo) if err != nil { + // 接口签名无 error 返回值，无法上抛；此处结算失败意味着预扣冻结金可能仍被冻结、 + // 未完成扣款/退款，必须记 Error 级日志告警，带上对账所需上下文以便人工/定时对账追回。 + g.Log().Errorf(ctx, + "settle_with_usage failed, pre-deduct may be stuck frozen (needs reconciliation): request_id=%s tenant_id=%d model=%s pre_deduct=%.6f: %v", + requestID, tenantID, modelName, preDeductAmount, err) return nil } return toCommonSettlementResult(result) diff --git a/internal/logic/billing/reconciliation.go b/internal/logic/billing/reconciliation.go index 7a48cf5..8d68fcb 100644 --- a/internal/logic/billing/reconciliation.go +++ b/internal/logic/billing/reconciliation.go @@ -7,6 +7,7 @@ import ( "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" + "github.com/shopspring/decimal" "github.com/qianfree/team-api/internal/dao" ) @@ -189,17 +190,19 @@ func CleanExpiredPreDeducts(ctx context.Context) { g.Log().Warningf(ctx, "[PRE-DEDUCT] found %d orphaned pre-deducts to clean", len(tracks)) - // 2. 按 tenant_id 分组聚合 - tenantAmounts := make(map[int64]float64) + // 2. 按 tenant_id 分组聚合（A8：金额用 decimal 累加，避免 float64 逐笔求和漂移； + // 该总额会喂给 frozen_balance 释放，属金额变更路径，须精确） + tenantAmounts := make(map[int64]decimal.Decimal) tenantRequests := make(map[int64][]string) for _, t := range tracks { - tenantAmounts[t.TenantID] += t.Amount + tenantAmounts[t.TenantID] = tenantAmounts[t.TenantID].Add(dec(t.Amount)) tenantRequests[t.TenantID] = append(tenantRequests[t.TenantID], t.RequestID) } // 3. 逐租户释放冻结金额 - for tenantID, totalAmount := range tenantAmounts { + for tenantID, totalAmountD := range tenantAmounts { requestIDs := tenantRequests[tenantID] + totalAmount := roundMoney(totalAmountD) // decimal 累加结果收敛到 10 位再落库 // 释放 DB frozen_balance _, err := g.DB().Ctx(ctx).Exec(ctx, diff --git a/internal/logic/billing/settlement.go b/internal/logic/billing/settlement.go index 147dabc..6c8493b 100644 --- a/internal/logic/billing/settlement.go +++ b/internal/logic/billing/settlement.go @@ -2,10 +2,13 @@ package billing import ( "context" + "errors" "fmt" - do "github.com/qianfree/team-api/internal/model/do" + "strings" "time" + do "github.com/qianfree/team-api/internal/model/do" + "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" @@ -15,6 +18,24 @@ import ( rcommon "github.com/qianfree/team-api/relay/common" ) +// errAlreadySettled 结算幂等哨兵：当同一 request_id 的计费记录已存在（bil_records 唯一约束冲突）时， +// 从结算事务闭包返回该错误使整个事务回滚（钱包扣款一并撤销），调用方据此识别为幂等空操作， +// 不重复扣款、不重复写账单。必须原样返回（不可 gerror.Wrap），以保证 errors.Is 能识别。 +var errAlreadySettled = errors.New("billing: request already settled (idempotent skip)") + +// isDuplicateKeyErr 判断 error 是否为 PostgreSQL 唯一约束冲突（SQLSTATE 23505）。 +// 结算时 bil_records.request_id 唯一索引会拒绝同一请求的第二次插入，据此把重复结算识别为 +// 幂等冲突。跨驱动（lib/pq、pgx）统一走错误文案匹配，避免耦合具体驱动的错误类型。 +func isDuplicateKeyErr(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "duplicate key value") || + strings.Contains(msg, "23505") || + strings.Contains(msg, "uk_bil_records_request") +} + // SettlementResult 结算结果 type SettlementResult struct { PreDeductAmount float64 // 预扣金额 @@ -31,6 +52,105 @@ type SettlementResult struct { RateMultiplier float64 // 费率倍率 } +// settlementTxParams 结算事务参数：三处结算共用的事务骨架的可变部分。 +type settlementTxParams struct { + walletID int64 // 钱包 ID + preDeductAmount float64 // 预扣冻结金额（事务内从 frozen_balance 释放） + actualCost float64 // 实际扣款金额（事务内从 balance 扣除） + logPrefix string // 错误信息前缀（settle / settle_with_usage / settle task） + // createBillingRecord 在事务内创建计费记录并返回其 ID； + // 唯一冲突（同一 request_id 已结算）时返回的 error 由骨架识别为 errAlreadySettled 并回滚。 + createBillingRecord func(ctx context.Context) (int64, error) + // buildTransaction 根据事务内读到的准确余额与计费记录 ID 构造消费流水。 + buildTransaction func(billingID int64, balanceAfter, frozenAfter float64) do.BilTransactions + // predeductRequestIDs 本次结算需置为 settled 的预扣追踪 request_id（task 结算含 "_adjust"）。 + predeductRequestIDs []string +} + +// executeSettlementTx 结算事务公共骨架：钱包扣款 → 事务内读准确余额 → 创建计费记录 +// → 记录消费流水 → 标记预扣追踪已结算，五步在同一事务内原子完成。 +// Settle / SettleWithUsage / SettleTaskSuccess 三处共用，差异部分（计费记录构造、 +// 流水构造、预扣追踪 request_id 集合）由 params 注入。 +// +// 幂等：计费记录命中 request_id 唯一约束时整个事务回滚（钱包扣款一并撤销）， +// 返回 errAlreadySettled，由调用方按幂等空操作处理。 +func executeSettlementTx(ctx context.Context, p settlementTxParams) (int64, error) { + var billingID int64 + err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // a. 更新钱包：释放预扣冻结 + 扣除实际成本 + now := time.Now() + _, err := g.DB().Ctx(ctx).Exec(ctx, + "UPDATE bil_wallets SET frozen_balance = GREATEST(frozen_balance - ?, 0), balance = balance - ?, updated_at = ? WHERE id = ?", + p.preDeductAmount, p.actualCost, now, p.walletID) + if err != nil { + return gerror.Wrapf(err, "%s: update wallet", p.logPrefix) + } + + // b. 事务内读取准确余额（best-effort，失败则保持 0） + balanceAfter, frozenAfter := readWalletBalanceTx(ctx, p.walletID) + + // c. 创建计费记录 + billingID, err = p.createBillingRecord(ctx) + if err != nil { + if isDuplicateKeyErr(err) { + // 同一 request_id 已结算：整个事务回滚（步骤 a 钱包扣款一并撤销），避免重复扣款/重复账单 + return errAlreadySettled + } + return gerror.Wrapf(err, "%s: create billing record", p.logPrefix) + } + + // d. 记录消费流水（事务内） + if _, err = dao.BilTransactions.Ctx(ctx).Data(p.buildTransaction(billingID, balanceAfter, frozenAfter)).Insert(); err != nil { + return gerror.Wrapf(err, "%s: record transaction", p.logPrefix) + } + + // e. 标记预扣追踪记录为已结算（事务内） + if err = markPredeductSettledTx(ctx, p.predeductRequestIDs); err != nil { + return gerror.Wrapf(err, "%s: mark prededuct settled", p.logPrefix) + } + + return nil + }) + return billingID, err +} + +// readWalletBalanceTx 在结算事务内读取钱包的准确余额与冻结额，供消费流水快照使用。 +// best-effort：读取失败返回 0/0，不影响主结算流程（与重构前逐处内联的行为一致）。 +func readWalletBalanceTx(ctx context.Context, walletID int64) (balanceAfter, frozenAfter float64) { + type balRow struct { + Balance float64 `json:"balance"` + FrozenBalance float64 `json:"frozen_balance"` + } + var br *balRow + err := dao.BilWallets.Ctx(ctx). + Where("id", walletID). + Fields("balance, frozen_balance"). + Scan(&br) + if err == nil && br != nil { + return br.Balance, br.FrozenBalance + } + return 0, 0 +} + +// markPredeductSettledTx 在结算事务内将指定 request_id 的预扣追踪记录从 frozen 置为 settled。 +// task 结算会传入 requestID 与 requestID+"_adjust" 两条（补扣调整产生），一并置为 settled， +// 避免残留 _adjust frozen 追踪被日对账判为不一致或被孤儿清理二次释放。 +func markPredeductSettledTx(ctx context.Context, requestIDs []string) error { + if len(requestIDs) == 0 { + return nil + } + placeholders := make([]string, len(requestIDs)) + args := make([]any, len(requestIDs)) + for i, rid := range requestIDs { + placeholders[i] = fmt.Sprintf("$%d", i+1) + args[i] = rid + } + _, err := g.DB().Ctx(ctx).Exec(ctx, + fmt.Sprintf("UPDATE bil_prededuct_tracks SET status = 'settled' WHERE request_id IN (%s) AND status = 'frozen'", strings.Join(placeholders, ", ")), + args...) + return err +} + // Settle 结算请求费用 // 预扣→调用→结算→退差额/补扣 func Settle(ctx context.Context, tenantID, userID, apiKeyID, channelID int64, @@ -41,8 +161,19 @@ func Settle(ctx context.Context, tenantID, userID, apiKeyID, channelID int64, // 1. 计算实际费用 breakdown, err := CalculateCost(ctx, tenantID, modelName, inputTokens, outputTokens) if err != nil { - g.Log().Errorf(ctx, "settle: calculate cost failed for %s: %v", requestID, err) - breakdown = &CostBreakdown{TotalCost: 0, Currency: "USD"} + // A4 修复：计价失败【不得】按零费用结算——那会把定价异常/模型未配价/短暂 DB 故障 + // 都变成免费请求。改为 fail-closed 兜底：按已冻结的预扣额计费（预扣是请求受理时的估价， + // 当前可得的最佳估值），与 task 结算路径（async_polling / sync_image_worker 默认 + // actualCost = PreDeductAmount）保持一致。保留 token 数便于账单核对。 + g.Log().Errorf(ctx, "settle: calculate cost failed for %s (model=%s), fallback to pre-deduct estimate %.6f: %v", + requestID, modelName, preDeductAmount, err) + breakdown = &CostBreakdown{ + TotalCost: preDeductAmount, + BaseCost: preDeductAmount, + InputTokens: inputTokens, + OutputTokens: outputTokens, + Currency: "USD", + } } actualCost := breakdown.TotalCost @@ -64,86 +195,61 @@ func Settle(ctx context.Context, tenantID, userID, apiKeyID, channelID int64, pricingResult, _ := GetModelPrice(ctx, tenantID, modelName) // 5. 事务内执行结算（钱包扣款 + 计费记录 + 流水 + tracks 状态） - var balanceAfter, frozenAfter float64 - var billingID int64 - err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - // 5a. 更新钱包 - now := time.Now() - _, err := tx.Ctx(ctx).Exec( - "UPDATE bil_wallets SET frozen_balance = GREATEST(frozen_balance - ?, 0), balance = balance - ?, updated_at = ? WHERE id = ?", - preDeductAmount, actualCost, now, wallet.ID) - if err != nil { - return gerror.Wrapf(err, "settle: update wallet") - } - - // 5b. 事务内读取准确余额 - type balRow struct { - Balance float64 `json:"balance"` - FrozenBalance float64 `json:"frozen_balance"` - } - var br *balRow - err = tx.Model("bil_wallets").Ctx(ctx). - Where("id", wallet.ID). - Fields("balance, frozen_balance"). - Scan(&br) - if err == nil && br != nil { - balanceAfter = br.Balance - frozenAfter = br.FrozenBalance - } - - // 5c. 创建计费记录 - var inputSnapPrice, outputSnapPrice float64 - var billingMode string - var discountRatio, billingInputMult, billingOutputMult float64 - if pricingResult != nil { - inputSnapPrice = pricingResult.InputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier - outputSnapPrice = pricingResult.OutputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier - billingMode = pricingResult.BillingMode - discountRatio = pricingResult.DiscountRatio - billingInputMult = breakdown.InputMultiplier - billingOutputMult = breakdown.OutputMultiplier - } - - billingID, err = createBillingRecord(ctx, tx, tenantID, userID, apiKeyID, channelID, - modelName, requestID, relayMode, inputTokens, outputTokens, - inputSnapPrice, outputSnapPrice, actualCost, - billingMode, discountRatio, billingInputMult, billingOutputMult) - if err != nil { - return gerror.Wrapf(err, "settle: create billing record") - } - - // 5d. 记录消费流水（事务内） - _, err = tx.Model("bil_transactions").Ctx(ctx).Data(do.BilTransactions{ - TenantId: tenantID, - WalletId: wallet.ID, - Type: "consume", - Amount: -actualCost, - BalanceAfter: balanceAfter, - FrozenAfter: frozenAfter, - RelatedId: billingID, - RelatedType: "billing_record", - Description: fmt.Sprintf("consume: %s model=%s input=%d output=%d pre_deduct=%.6f actual=%.6f", requestID, modelName, inputTokens, outputTokens, preDeductAmount, actualCost), - UserId: userID, - RequestId: requestID, - ModelName: modelName, - ProjectId: projectID, - ApiKeyId: apiKeyID, - }).Insert() - if err != nil { - return gerror.Wrapf(err, "settle: record transaction") - } - - // 5e. 标记预扣追踪记录为已结算（事务内） - _, err = tx.Ctx(ctx).Exec( - "UPDATE bil_prededuct_tracks SET status = 'settled' WHERE request_id = $1 AND status = 'frozen'", - requestID) - if err != nil { - return gerror.Wrapf(err, "settle: mark prededuct settled") - } - - return nil + billingID, err := executeSettlementTx(ctx, settlementTxParams{ + walletID: wallet.ID, + preDeductAmount: preDeductAmount, + actualCost: actualCost, + logPrefix: "settle", + createBillingRecord: func(ctx context.Context) (int64, error) { + var inputSnapPrice, outputSnapPrice float64 + var billingMode string + var discountRatio, billingInputMult, billingOutputMult float64 + if pricingResult != nil { + inputSnapPrice = pricingResult.InputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier + outputSnapPrice = pricingResult.OutputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier + billingMode = pricingResult.BillingMode + discountRatio = pricingResult.DiscountRatio + billingInputMult = breakdown.InputMultiplier + billingOutputMult = breakdown.OutputMultiplier + } + return createBillingRecord(ctx, tenantID, userID, apiKeyID, channelID, + modelName, requestID, relayMode, inputTokens, outputTokens, + inputSnapPrice, outputSnapPrice, actualCost, + billingMode, discountRatio, billingInputMult, billingOutputMult) + }, + buildTransaction: func(billingID int64, balanceAfter, frozenAfter float64) do.BilTransactions { + return do.BilTransactions{ + TenantId: tenantID, + WalletId: wallet.ID, + Type: "consume", + Amount: -actualCost, + BalanceAfter: balanceAfter, + FrozenAfter: frozenAfter, + RelatedId: billingID, + RelatedType: "billing_record", + Description: fmt.Sprintf("consume: %s model=%s input=%d output=%d pre_deduct=%.6f actual=%.6f", requestID, modelName, inputTokens, outputTokens, preDeductAmount, actualCost), + UserId: userID, + RequestId: requestID, + ModelName: modelName, + ProjectId: projectID, + ApiKeyId: apiKeyID, + } + }, + predeductRequestIDs: []string{requestID}, }) if err != nil { + if errors.Is(err, errAlreadySettled) { + // 幂等跳过：该请求此前已结算完成，本次为重复调用，不再扣款/写账单 + g.Log().Warningf(ctx, "settle: duplicate settlement skipped for request=%s (idempotent)", requestID) + return &SettlementResult{ + PreDeductAmount: preDeductAmount, + ActualCost: actualCost, + BaseCost: breakdown.BaseCost, + RefundAmount: refundAmt, + SupplementAmount: supplementAmt, + CostBreakdown: breakdown, + }, nil + } return nil, err } @@ -173,8 +279,19 @@ func SettleWithUsage(ctx context.Context, tenantID, userID, apiKeyID, channelID // 1. 使用完整 Usage 计算实际费用（含 cache token） breakdown, err := CalculateCostWithUsage(ctx, tenantID, modelName, usage) if err != nil { - g.Log().Errorf(ctx, "settle_with_usage: calculate cost failed for %s: %v", requestID, err) - breakdown = &CostBreakdown{TotalCost: 0, Currency: "USD"} + // A4 修复：计价失败 fail-closed 兜底按预扣额计费，而非零费用（免费请求）。见 Settle 同段说明。 + g.Log().Errorf(ctx, "settle_with_usage: calculate cost failed for %s (model=%s), fallback to pre-deduct estimate %.6f: %v", + requestID, modelName, preDeductAmount, err) + fb := &CostBreakdown{ + TotalCost: preDeductAmount, + BaseCost: preDeductAmount, + Currency: "USD", + } + if usage != nil { + fb.InputTokens = usage.PromptTokens + fb.OutputTokens = usage.CompletionTokens + } + breakdown = fb } actualCost := breakdown.TotalCost @@ -196,90 +313,65 @@ func SettleWithUsage(ctx context.Context, tenantID, userID, apiKeyID, channelID pricingResult, _ := GetModelPrice(ctx, tenantID, modelName) // 5. 事务内执行结算（钱包扣款 + 计费记录 + 流水 + tracks 状态） - var balanceAfter, frozenAfter float64 - var billingID int64 - err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - // 5a. 更新钱包 - now := time.Now() - _, err := tx.Ctx(ctx).Exec( - "UPDATE bil_wallets SET frozen_balance = GREATEST(frozen_balance - ?, 0), balance = balance - ?, updated_at = ? WHERE id = ?", - preDeductAmount, actualCost, now, wallet.ID) - if err != nil { - return gerror.Wrapf(err, "settle_with_usage: update wallet") - } - - // 5b. 事务内读取准确余额 - type balRow struct { - Balance float64 `json:"balance"` - FrozenBalance float64 `json:"frozen_balance"` - } - var br *balRow - err = tx.Model("bil_wallets").Ctx(ctx). - Where("id", wallet.ID). - Fields("balance, frozen_balance"). - Scan(&br) - if err == nil && br != nil { - balanceAfter = br.Balance - frozenAfter = br.FrozenBalance - } - - // 5c. 创建计费记录（含快照） - var inputSnapPrice, outputSnapPrice float64 - var billingMode string - var discountRatio, billingInputMult, billingOutputMult float64 - if pricingResult != nil { - inputSnapPrice = pricingResult.InputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier - outputSnapPrice = pricingResult.OutputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier - billingMode = pricingResult.BillingMode - discountRatio = pricingResult.DiscountRatio - billingInputMult = breakdown.InputMultiplier - billingOutputMult = breakdown.OutputMultiplier - } - - billingID, err = createBillingRecordWithSnapshot(ctx, tx, tenantID, userID, apiKeyID, channelID, - modelName, requestID, relayMode, breakdown, - inputSnapPrice, outputSnapPrice, actualCost, - billingMode, discountRatio, billingInputMult, billingOutputMult, pricingResult) - if err != nil { - return gerror.Wrapf(err, "settle_with_usage: create billing record") - } - - // 5d. 记录消费流水（事务内） - var txProjectID int64 - if relayInfo != nil { - txProjectID = relayInfo.ProjectID - } - _, err = tx.Model("bil_transactions").Ctx(ctx).Data(do.BilTransactions{ - TenantId: tenantID, - WalletId: wallet.ID, - Type: "consume", - Amount: -actualCost, - BalanceAfter: balanceAfter, - FrozenAfter: frozenAfter, - RelatedId: billingID, - RelatedType: "billing_record", - Description: fmt.Sprintf("consume: %s model=%s input=%d output=%d pre_deduct=%.6f actual=%.6f", requestID, modelName, breakdown.InputTokens, breakdown.OutputTokens, preDeductAmount, actualCost), - UserId: userID, - RequestId: requestID, - ModelName: modelName, - ProjectId: txProjectID, - ApiKeyId: apiKeyID, - }).Insert() - if err != nil { - return gerror.Wrapf(err, "settle_with_usage: record transaction") - } - - // 5e. 标记预扣追踪记录为已结算（事务内） - _, err = tx.Ctx(ctx).Exec( - "UPDATE bil_prededuct_tracks SET status = 'settled' WHERE request_id = $1 AND status = 'frozen'", - requestID) - if err != nil { - return gerror.Wrapf(err, "settle_with_usage: mark prededuct settled") - } - - return nil + billingID, err := executeSettlementTx(ctx, settlementTxParams{ + walletID: wallet.ID, + preDeductAmount: preDeductAmount, + actualCost: actualCost, + logPrefix: "settle_with_usage", + createBillingRecord: func(ctx context.Context) (int64, error) { + var inputSnapPrice, outputSnapPrice float64 + var billingMode string + var discountRatio, billingInputMult, billingOutputMult float64 + if pricingResult != nil { + inputSnapPrice = pricingResult.InputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier + outputSnapPrice = pricingResult.OutputPrice * pricingResult.ModelMultiplier * pricingResult.TenantMultiplier + billingMode = pricingResult.BillingMode + discountRatio = pricingResult.DiscountRatio + billingInputMult = breakdown.InputMultiplier + billingOutputMult = breakdown.OutputMultiplier + } + return createBillingRecordWithSnapshot(ctx, tenantID, userID, apiKeyID, channelID, + modelName, requestID, relayMode, breakdown, + inputSnapPrice, outputSnapPrice, actualCost, + billingMode, discountRatio, billingInputMult, billingOutputMult, pricingResult) + }, + buildTransaction: func(billingID int64, balanceAfter, frozenAfter float64) do.BilTransactions { + var txProjectID int64 + if relayInfo != nil { + txProjectID = relayInfo.ProjectID + } + return do.BilTransactions{ + TenantId: tenantID, + WalletId: wallet.ID, + Type: "consume", + Amount: -actualCost, + BalanceAfter: balanceAfter, + FrozenAfter: frozenAfter, + RelatedId: billingID, + RelatedType: "billing_record", + Description: fmt.Sprintf("consume: %s model=%s input=%d output=%d pre_deduct=%.6f actual=%.6f", requestID, modelName, breakdown.InputTokens, breakdown.OutputTokens, preDeductAmount, actualCost), + UserId: userID, + RequestId: requestID, + ModelName: modelName, + ProjectId: txProjectID, + ApiKeyId: apiKeyID, + } + }, + predeductRequestIDs: []string{requestID}, }) if err != nil { + if errors.Is(err, errAlreadySettled) { + // 幂等跳过：该请求此前已结算完成，本次为重复调用，不再扣款/写账单 + g.Log().Warningf(ctx, "settle_with_usage: duplicate settlement skipped for request=%s (idempotent)", requestID) + return &SettlementResult{ + PreDeductAmount: preDeductAmount, + ActualCost: actualCost, + BaseCost: breakdown.BaseCost, + RefundAmount: refundAmt, + SupplementAmount: supplementAmt, + CostBreakdown: breakdown, + }, nil + } return nil, err } @@ -346,8 +438,8 @@ func SettleStreamInterrupted(ctx context.Context, tenantID, userID, apiKeyID, ch preDeductAmount, projectID) } -// createBillingRecord 创建计费记录（含快照字段） -func createBillingRecord(ctx context.Context, tx gdb.TX, tenantID, userID, apiKeyID, channelID int64, +// createBillingRecord 创建计费记录（含快照字段）。依赖调用方传入携带事务的 ctx，内部用 dao.Xxx.Ctx(ctx) 传播事务 +func createBillingRecord(ctx context.Context, tenantID, userID, apiKeyID, channelID int64, modelName, requestID, relayMode string, inputTokens, outputTokens int, inputPrice, outputPrice, totalCost float64, @@ -385,7 +477,7 @@ func createBillingRecord(ctx context.Context, tx gdb.TX, tenantID, userID, apiKe data.BillingInputMultiplier = billingInputMult data.BillingOutputMultiplier = billingOutputMult - result, err := tx.Model("bil_records").Ctx(ctx).Data(data).Insert() + result, err := dao.BilRecords.Ctx(ctx).Data(data).Insert() if err != nil { return 0, err } @@ -431,8 +523,8 @@ func UpdateUsageLogCostWithSnapshot(ctx context.Context, requestID string, break Data(data).Update() } -// createBillingRecordWithSnapshot 创建计费记录（含 cache token 和完整快照） -func createBillingRecordWithSnapshot(ctx context.Context, tx gdb.TX, tenantID, userID, apiKeyID, channelID int64, +// createBillingRecordWithSnapshot 创建计费记录（含 cache token 和完整快照）。依赖调用方传入携带事务的 ctx +func createBillingRecordWithSnapshot(ctx context.Context, tenantID, userID, apiKeyID, channelID int64, modelName, requestID, relayMode string, breakdown *CostBreakdown, inputPrice, outputPrice, totalCost float64, @@ -484,7 +576,7 @@ func createBillingRecordWithSnapshot(ctx context.Context, tx gdb.TX, tenantID, u data.BaseOutputPrice = pricing.BaseOutputPrice } - result, err := tx.Model("bil_records").Ctx(ctx).Data(data).Insert() + result, err := dao.BilRecords.Ctx(ctx).Data(data).Insert() if err != nil { return 0, err } diff --git a/internal/logic/billing/task_billing.go b/internal/logic/billing/task_billing.go index 251ab8a..1bb50cf 100644 --- a/internal/logic/billing/task_billing.go +++ b/internal/logic/billing/task_billing.go @@ -2,14 +2,14 @@ package billing import ( "context" - "database/sql" + "errors" "fmt" "time" - "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" + "github.com/qianfree/team-api/internal/dao" do "github.com/qianfree/team-api/internal/model/do" "github.com/qianfree/team-api/relay/common" ) @@ -29,13 +29,34 @@ func (b *TaskBillingProviderImpl) EstimateTaskCost(ctx context.Context, tenantID if err != nil { return 0.01, nil } + return estimateTaskCost(pricing, ratios), nil +} + +// imagePlaceholderPreDeduct 图片等按次任务**未配置按次单价**时的占位预扣（USD）。 +// 图片提交时拿不到 token 用量、无法精确估价，故先按此小额占位冻结，结算阶段再按上游 +// 返回的真实 token 用量多退少补。仅用于「无有效定价」的兜底，不覆盖已配置的真实按次单价。 +const imagePlaceholderPreDeduct = 0.1 + +// estimateTaskCost 纯函数：根据定价与计费比率估算任务预扣费用，不依赖数据库/缓存，便于单测。 +// +// 计费口径按任务类型分流： +// - per_request（按次计费，图片/音乐等）：直接取按次单价； +// - 时长类任务（视频生成，ratios 携带 duration/resolution 信号）：按 +// 10000 tokens/s × duration × resolution 预估 token 再乘输出单价； +// - 其余无时长信号的任务（如未显式配成 per_request 的图片模型）：退回按次单价， +// **不再**套用视频 token 估算——图片没有时长/分辨率，套 10000×5×2.25 会凭空估出 +// 11.25 万 token 的天价预扣（$30/1M 输出价即得 $3.375），且与结算的「0 token」自相矛盾。 +func estimateTaskCost(pricing *PricingResult, ratios map[string]float64) float64 { + if pricing == nil { + return 0.01 + } var cost float64 - - // 按次计费：直接用单价 - if pricing.BillingMode == "per_request" { + switch { + case pricing.BillingMode == "per_request": + // 按次计费：直接用单价 cost = pricing.PerRequestPrice - } else if pricing.OutputPrice > 0 { + case pricing.OutputPrice > 0 && hasDurationSignal(ratios): duration := 5.0 // 默认 5 秒 if d, ok := ratios["duration"]; ok && d > 0 { duration = d @@ -49,8 +70,15 @@ func (b *TaskBillingProviderImpl) EstimateTaskCost(ctx context.Context, tenantID // 火山方舟视频生成约 10000 tokens/s (480p 基准)，用于预扣估算 estimatedTokens := 10000.0 * duration * resolutionMul cost = estimatedTokens / 1_000_000.0 * pricing.OutputPrice * pricing.TenantMultiplier - } else { - cost = pricing.PerRequestPrice + default: + // 无时长信号（图片等扁平计费任务）：优先按次单价；未配按次价时用占位预扣， + // 绝不走视频 token 估算。结算阶段再按上游真实 token 用量多退少补 + // （见 sync_image_worker.settleSyncImageSuccess）。 + if pricing.PerRequestPrice > 0 { + cost = pricing.PerRequestPrice + } else { + cost = imagePlaceholderPreDeduct + } } // 应用附加比率（video_input 折扣等） @@ -64,7 +92,21 @@ func (b *TaskBillingProviderImpl) EstimateTaskCost(ctx context.Context, tenantID if cost < 0.01 { cost = 0.01 } - return cost, nil + return cost +} + +// hasDurationSignal 判断计费比率里是否携带时长类任务（视频）的 duration/resolution 信号。 +// 视频提交管线（VolcengineVideoAdaptor.EstimateBilling）必然写入这两个键；图片等同步/异步 +// 任务传 nil ratios，据此区分「该走视频 token 估算」还是「按次计费」。 +func hasDurationSignal(ratios map[string]float64) bool { + if ratios == nil { + return false + } + if _, ok := ratios["duration"]; ok { + return true + } + _, ok := ratios["resolution"] + return ok } // PreDeductTask 预扣任务费用 @@ -121,107 +163,89 @@ func (b *TaskBillingProviderImpl) SettleTaskSuccess(ctx context.Context, tenantI } // 3. 事务内执行结算（钱包扣款 + 计费记录 + 流水 + tracks 状态） - var balanceAfter, frozenAfter float64 - var taskBillingID int64 - err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - // 3a. 更新钱包 - now := time.Now() - _, err := tx.Ctx(ctx).Exec( - "UPDATE bil_wallets SET frozen_balance = GREATEST(frozen_balance - ?, 0), balance = balance - ?, updated_at = ? WHERE id = ?", - preDeductAmount, actualCost, now, wallet.ID) - if err != nil { - return fmt.Errorf("settle task: update wallet: %w", err) - } - - // 3b. 事务内读取准确余额 - type balRow struct { - Balance float64 `json:"balance"` - FrozenBalance float64 `json:"frozen_balance"` - } - var br *balRow - err = tx.Model("bil_wallets").Ctx(ctx). - Where("id", wallet.ID). - Fields("balance, frozen_balance"). - Scan(&br) - if err == nil && br != nil { - balanceAfter = br.Balance - frozenAfter = br.FrozenBalance - } - - // 3c. 创建计费记录 - var billingResult sql.Result - billingResult, err = tx.Model("bil_records").Ctx(ctx).Data(do.BilRecords{ - TenantId: tenantID, - UserId: userID, - ApiKeyId: apiKeyID, - ChannelId: channelID, - ModelName: modelName, - RequestId: requestID, - RelayMode: "task", - InputTokens: 0, - OutputTokens: totalTokens, - InputPrice: 0, - OutputPrice: effectiveOutputPrice, - TotalCost: actualCost, - Currency: "USD", - Status: "settled", - SettledAt: gtime.NewFromTime(time.Now()), - BillingMode: billingMode, - DiscountRatio: func() float64 { - if discountRatio > 0 { - return discountRatio - } - return 0 - }(), - EffectiveInputPrice: 0, - EffectiveOutputPrice: effectiveOutputPrice, - BillingInputMultiplier: 0, - BillingOutputMultiplier: 0, - CacheCreationTokens: 0, - CacheReadTokens: 0, - CacheCreationCost: 0, - CacheReadCost: 0, - }).Insert() - if err != nil { - return fmt.Errorf("settle task: create billing record: %w", err) - } - if billingResult != nil { - taskBillingID, _ = billingResult.LastInsertId() - } - - // 3d. 记录消费流水（事务内） - _, err = tx.Model("bil_transactions").Ctx(ctx).Data(do.BilTransactions{ - TenantId: tenantID, - WalletId: wallet.ID, - Type: "consume", - Amount: -actualCost, - BalanceAfter: balanceAfter, - FrozenAfter: frozenAfter, - RelatedId: taskBillingID, - RelatedType: "billing_record", - Description: fmt.Sprintf("consume: %s model=%s pre_deduct=%.6f actual=%.6f", requestID, modelName, preDeductAmount, actualCost), - UserId: userID, - RequestId: requestID, - ModelName: modelName, - ProjectId: 0, - ApiKeyId: apiKeyID, - TaskId: taskID, - }).Insert() - if err != nil { - return fmt.Errorf("settle task: record transaction: %w", err) - } - - // 3e. 标记预扣追踪记录为已结算（事务内） - _, err = tx.Ctx(ctx).Exec( - "UPDATE bil_prededuct_tracks SET status = 'settled' WHERE request_id = $1 AND status = 'frozen'", - requestID) - if err != nil { - return fmt.Errorf("settle task: mark prededuct settled: %w", err) - } - - return nil + // 预扣追踪同时覆盖 requestID 与 requestID+"_adjust"：步骤 3a 已按总预扣额（含 AdjustTaskBilling + // 补扣产生的 _adjust 冻结）一次性释放，两条追踪记录都应随之置为 settled；否则残留的 _adjust + // frozen 追踪会被日对账判为不一致，并被孤儿清理二次释放。 + _, err = executeSettlementTx(ctx, settlementTxParams{ + walletID: wallet.ID, + preDeductAmount: preDeductAmount, + actualCost: actualCost, + logPrefix: "settle task", + createBillingRecord: func(ctx context.Context) (int64, error) { + billingResult, err := dao.BilRecords.Ctx(ctx).Data(do.BilRecords{ + TenantId: tenantID, + UserId: userID, + ApiKeyId: apiKeyID, + ChannelId: channelID, + ModelName: modelName, + RequestId: requestID, + RelayMode: "task", + InputTokens: 0, + OutputTokens: totalTokens, + InputPrice: 0, + OutputPrice: effectiveOutputPrice, + TotalCost: actualCost, + Currency: "USD", + Status: "settled", + SettledAt: gtime.NewFromTime(time.Now()), + BillingMode: billingMode, + DiscountRatio: func() float64 { + if discountRatio > 0 { + return discountRatio + } + return 0 + }(), + EffectiveInputPrice: 0, + EffectiveOutputPrice: effectiveOutputPrice, + BillingInputMultiplier: 0, + BillingOutputMultiplier: 0, + CacheCreationTokens: 0, + CacheReadTokens: 0, + CacheCreationCost: 0, + CacheReadCost: 0, + }).Insert() + if err != nil { + return 0, err + } + var id int64 + if billingResult != nil { + id, _ = billingResult.LastInsertId() + } + return id, nil + }, + buildTransaction: func(billingID int64, balanceAfter, frozenAfter float64) do.BilTransactions { + return do.BilTransactions{ + TenantId: tenantID, + WalletId: wallet.ID, + Type: "consume", + Amount: -actualCost, + BalanceAfter: balanceAfter, + FrozenAfter: frozenAfter, + RelatedId: billingID, + RelatedType: "billing_record", + Description: fmt.Sprintf("consume: %s model=%s pre_deduct=%.6f actual=%.6f", requestID, modelName, preDeductAmount, actualCost), + UserId: userID, + RequestId: requestID, + ModelName: modelName, + ProjectId: 0, + ApiKeyId: apiKeyID, + TaskId: taskID, + } + }, + predeductRequestIDs: []string{requestID, requestID + "_adjust"}, }) if err != nil { + if errors.Is(err, errAlreadySettled) { + // 幂等跳过：该任务此前已结算完成，本次为重复调用（轮询/重放），不再扣款/写账单 + g.Log().Warningf(ctx, "settle task: duplicate settlement skipped for request=%s (idempotent)", requestID) + return &common.SettlementResult{ + PreDeductAmount: preDeductAmount, + ActualCost: actualCost, + BaseCost: breakdown.BaseCost, + TotalCost: actualCost, + OutputCost: breakdown.OutputCost, + }, nil + } return nil, err } @@ -234,14 +258,14 @@ func (b *TaskBillingProviderImpl) SettleTaskSuccess(ctx context.Context, tenantI // 5. 异步检查余额预警 go CheckBalanceWarning(context.Background(), tenantID) - // 6. 差额处理：实际 < 预扣时退还差额（实际 > 预扣时已在步骤 3 扣完，无需额外操作） - if diff < -0.001 { - if err := SettleFailed(ctx, tenantID, requestID+"_adjust", -diff); err != nil { - g.Log().Warningf(ctx, "settle task adjust refund failed for %s: %v", requestID, err) - } - } + // 6. 差额已在步骤 3a 一次性结清，此处不得再做任何解冻/退款： + // 步骤 3a 已 frozen_balance -= preDeductAmount（释放全部预扣冻结）、balance -= actualCost + // （只扣真实成本），无论 actualCost 大于还是小于预扣，可用余额都已精确调整到位 + // （available 变化量恰为 preDeductAmount - actualCost）。 + // 切勿再对 requestID+"_adjust" 调 UnfreezePreDeduct/SettleFailed——那会在步骤 3a 之外 + // 二次释放从未单独冻结过的金额，导致 frozen_balance 被过度释放（Redis 侧无下限时甚至为负）。 - // 6. 生成计费快照 + 摘要 + // 7. 生成计费快照 + 摘要 result := &common.SettlementResult{ PreDeductAmount: preDeductAmount, ActualCost: actualCost, @@ -300,6 +324,15 @@ func buildTaskCostBreakdown(pricing *PricingResult, actualCost float64, totalTok return bd } + // token 模式但没有真实 token 用量（图片等扁平计费任务）：把费用整体记为 BaseCost， + // 不摊进 OutputCost。否则快照会生成「0 token 却有 output 费用」的自相矛盾行 + // （如 0 tokens × $30/1M = $3.375）。真实费用仍由结算的 actual_cost 体现。 + if totalTokens <= 0 { + bd.BaseCost = actualCost + bd.TotalCost = actualCost + return bd + } + // token 模式 bd.OutputTokens = totalTokens bd.OutputCost = actualCost diff --git a/internal/logic/billing/task_billing_test.go b/internal/logic/billing/task_billing_test.go index f936c47..7ebed4a 100644 --- a/internal/logic/billing/task_billing_test.go +++ b/internal/logic/billing/task_billing_test.go @@ -97,3 +97,115 @@ func TestBuildTaskCostBreakdown_CarriesPricingFields(t *testing.T) { t.Errorf("Currency = %q, want USD", bd.Currency) } } + +// TestBuildTaskCostBreakdown_TokenMode_ZeroTokens 覆盖图片等「token 模式但无真实 token 用量」 +// 的场景：费用必须整体记为 BaseCost，OutputCost/OutputTokens 保持 0，避免快照生成 +// 「0 token 却有 output 费用」的自相矛盾行（如 0 tokens × $30/1M = $3.375）。 +func TestBuildTaskCostBreakdown_TokenMode_ZeroTokens(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + OutputPrice: 30.0, + TenantMultiplier: 1.0, + Currency: "USD", + } + + bd := buildTaskCostBreakdown(pricing, 3.375, 0, 0) + if bd.OutputTokens != 0 { + t.Errorf("OutputTokens = %d, want 0", bd.OutputTokens) + } + assertFloat(t, bd.OutputCost, 0, "OutputCost") + assertFloat(t, bd.BaseCost, 3.375, "BaseCost") + assertFloat(t, bd.TotalCost, 3.375, "TotalCost") +} + +// TestEstimateTaskCost_PerRequest 按次计费直接取按次单价。 +func TestEstimateTaskCost_PerRequest(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "per_request", + PerRequestPrice: 0.04, + OutputPrice: 30.0, // 即使配了 output_price 也不应走 token 估算 + TenantMultiplier: 1.0, + } + assertFloat(t, estimateTaskCost(pricing, nil), 0.04, "per_request cost") +} + +// TestEstimateTaskCost_VideoWithDurationSignal 视频任务（ratios 携带 duration/resolution） +// 才走 10000×duration×resolution 的 token 估算：默认 5s×720p(2.25)×$30/1M = $3.375。 +func TestEstimateTaskCost_VideoWithDurationSignal(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + OutputPrice: 30.0, + TenantMultiplier: 1.0, + } + ratios := map[string]float64{"duration": 5, "resolution": 2.25} + assertFloat(t, estimateTaskCost(pricing, ratios), 3.375, "video cost") +} + +// TestEstimateTaskCost_ImageNoDurationSignal 图片任务（token 模式、有 output_price、但 ratios 无 +// 时长信号）必须退回按次口径，**不得**凭空估出 11.25 万 token 的 $3.375 预扣。 +// 未配 per_request_price 时用占位预扣 $0.1（结算再按真实 token 多退少补）。 +func TestEstimateTaskCost_ImageNoDurationSignal(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + OutputPrice: 30.0, + PerRequestPrice: 0, // 未配按次价 + TenantMultiplier: 1.0, + } + assertFloat(t, estimateTaskCost(pricing, nil), 0.1, "image placeholder pre-deduct") +} + +// TestEstimateTaskCost_PerRequestBelowPlaceholder 已配置的真实按次单价（低于占位值）必须被尊重， +// 不得被占位预扣 $0.1 抬价——否则 per_request 图片（预扣==实收）会被超扣。 +func TestEstimateTaskCost_PerRequestBelowPlaceholder(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "per_request", + PerRequestPrice: 0.04, // DALL·E 类真实单价，低于 0.1 占位 + TenantMultiplier: 1.0, + } + assertFloat(t, estimateTaskCost(pricing, nil), 0.04, "configured per_request price respected") +} + +// TestEstimateTaskCost_ImageUsesPerRequestPrice 图片模型即便 billing_mode 仍是 token，只要配了 +// per_request_price，无时长信号时就按按次单价估价（而非 token 估算）。 +func TestEstimateTaskCost_ImageUsesPerRequestPrice(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + OutputPrice: 30.0, + PerRequestPrice: 0.05, + TenantMultiplier: 1.0, + } + assertFloat(t, estimateTaskCost(pricing, nil), 0.05, "image per_request cost") +} + +// TestEstimateTaskCost_VideoInputDiscount 附加比率（video_input 折扣）在时长估算之上叠加。 +func TestEstimateTaskCost_VideoInputDiscount(t *testing.T) { + pricing := &PricingResult{ + BillingMode: "token", + OutputPrice: 30.0, + TenantMultiplier: 1.0, + } + ratios := map[string]float64{"duration": 5, "resolution": 2.25, "video_input": 0.5} + // 3.375 × 0.5 = 1.6875 + assertFloat(t, estimateTaskCost(pricing, ratios), 1.6875, "video discounted cost") +} + +// TestEstimateTaskCost_NilPricing 定价缺失时按最低消费兜底。 +func TestEstimateTaskCost_NilPricing(t *testing.T) { + assertFloat(t, estimateTaskCost(nil, nil), 0.01, "nil pricing floor") +} + +// TestHasDurationSignal 区分视频（有 duration/resolution）与图片（nil）。 +func TestHasDurationSignal(t *testing.T) { + if hasDurationSignal(nil) { + t.Error("nil ratios should not signal duration") + } + if hasDurationSignal(map[string]float64{"video_input": 0.5}) { + t.Error("video_input alone should not signal duration") + } + if !hasDurationSignal(map[string]float64{"duration": 5}) { + t.Error("duration key should signal") + } + if !hasDurationSignal(map[string]float64{"resolution": 2.25}) { + t.Error("resolution key should signal") + } +} diff --git a/internal/logic/billing/wallet.go b/internal/logic/billing/wallet.go index 58aa523..6a51292 100644 --- a/internal/logic/billing/wallet.go +++ b/internal/logic/billing/wallet.go @@ -16,12 +16,20 @@ import ( ) const ( - // PreDeductRedisKeyPrefix 预扣 Redis key 前缀 - PreDeductRedisKeyPrefix = "prededuct:" + // PreDeductRedisKeyPrefix 预扣 Redis key 前缀。 + // v2（Phase 3）：预扣 amount 改为整数微单位(micro-USD)存储，与旧版 float 值不兼容， + // 故 bump 版本号；旧 key 自然按 TTL 过期，杜绝新代码把旧 float 值误读成 micro。 + PreDeductRedisKeyPrefix = "prededuct:v2:" // PreDeductMaxAge 预扣记录最大存活时间（秒），防止异常未结算的预扣占用余额 PreDeductMaxAge = 1800 // 30 分钟 ) +// walletHashKey 钱包 Redis hash key。 +// v2（Phase 3）：balance / frozen_balance 以整数 micro-USD 存储；旧版 float key 随 TTL 过期。 +func walletHashKey(tenantID int64) string { + return fmt.Sprintf("wallet:v2:%d", tenantID) +} + // walletCache 钱包缓存（TTL 300s） var walletCache = lcommon.NewCache("wallet", 300*time.Second) @@ -130,10 +138,10 @@ func PreDeduct(ctx context.Context, tenantID int64, amount float64, requestID st return preDeductDB(ctx, tenantID, amount, requestID) } - // Redis Lua 脚本：原子检查+冻结 - // KEYS[1] = wallet:{tenant_id} (hash: balance, frozen_balance) - // KEYS[2] = prededuct:{request_id} - // ARGV[1] = amount + // Redis Lua 脚本：原子检查+冻结（v2：金额全部为整数 micro-USD，整数运算无浮点漂移） + // KEYS[1] = wallet:v2:{tenant_id} (hash: balance, frozen_balance —— 均为整数 micro) + // KEYS[2] = prededuct:v2:{request_id} + // ARGV[1] = amount_micro（整数微单位） // ARGV[2] = request_id // ARGV[3] = ttl (PreDeductMaxAge) // ARGV[4] = tenant_id @@ -152,7 +160,7 @@ if exists == 1 then return 1 end --- 获取钱包信息 +-- 获取钱包信息（整数 micro） local balance = tonumber(redis.call("HGET", wallet_key, "balance") or "0") local frozen = tonumber(redis.call("HGET", wallet_key, "frozen_balance") or "0") @@ -162,8 +170,8 @@ if available < amount then return 0 end --- 冻结金额 -redis.call("HINCRBYFLOAT", wallet_key, "frozen_balance", amount) +-- 冻结金额（整数自增，无浮点漂移） +redis.call("HINCRBY", wallet_key, "frozen_balance", amount) redis.call("HSET", prededuct_key, "amount", amount) redis.call("HSET", prededuct_key, "tenant_id", ARGV[4]) redis.call("HSET", prededuct_key, "model_name", ARGV[5]) @@ -175,12 +183,13 @@ redis.call("EXPIRE", prededuct_key, ttl) return 1 ` - walletRedisKey := fmt.Sprintf("wallet:%d", tenantID) + walletRedisKey := walletHashKey(tenantID) predeductRedisKey := fmt.Sprintf("%s%s", PreDeductRedisKeyPrefix, requestID) + amountMicro := toMicro(amount) result, err := g.Redis().Do(ctx, "EVAL", luaScript, 2, walletRedisKey, predeductRedisKey, - amount, requestID, PreDeductMaxAge, tenantID, modelName, time.Now().Unix()) + amountMicro, requestID, PreDeductMaxAge, tenantID, modelName, time.Now().Unix()) if err != nil { // Redis 不可用，降级到 DB 直接扣减 return preDeductDB(ctx, tenantID, amount, requestID) @@ -203,29 +212,24 @@ return 1 // preDeductDB DB 降级预扣（Redis 不可用时） func preDeductDB(ctx context.Context, tenantID int64, amount float64, requestID string) (bool, error) { - // 使用条件检查：WHERE balance - frozen_balance >= amount - type walletRow struct { - ID int64 `json:"id"` - Balance float64 `json:"balance"` - FrozenBalance float64 `json:"frozen_balance"` - } - var w *walletRow - err := dao.BilWallets.Ctx(ctx). - Where("tenant_id", tenantID). - Where("balance - frozen_balance >= ?", amount). - Fields("id, balance, frozen_balance"). - LockUpdate(). - Scan(&w) - if err != nil || w == nil { - return false, gerror.New("insufficient balance") - } - - _, err = g.DB().Exec(ctx, - "UPDATE bil_wallets SET frozen_balance = frozen_balance + ?, updated_at = ? WHERE id = ? ", - amount, time.Now(), w.ID) + // A9 修复：用单条原子条件更新替代「先 SELECT ... FOR UPDATE，再独立 UPDATE」。 + // 原实现两条语句在 autocommit 下各自成一个事务，FOR UPDATE 的行锁在 SELECT 语句提交后即释放， + // 两个并发降级预扣可都通过 available 检查、再各自 frozen += amount → 超额冻结（可用余额被冻成负）。 + // 单条 "WHERE tenant_id=? AND balance - frozen_balance >= amount" 的 UPDATE 在语句执行期间持有 + // 行锁并原子重算谓词：RowsAffected==1 表示冻结成功，==0 表示可用余额不足（或钱包不存在）。 + res, err := g.DB().Exec(ctx, + "UPDATE bil_wallets SET frozen_balance = frozen_balance + ?, updated_at = ? WHERE tenant_id = ? AND balance - frozen_balance >= ?", + amount, time.Now(), tenantID, amount) if err != nil { return false, gerror.Wrapf(err, "pre-deduct db update") } + affected, err := res.RowsAffected() + if err != nil { + return false, gerror.Wrapf(err, "pre-deduct db update result") + } + if affected == 0 { + return false, gerror.New("insufficient balance") + } // 清除缓存 walletCache.Delete(ctx, fmt.Sprintf("%d", tenantID)) @@ -246,6 +250,25 @@ func preDeductSyncDB(ctx context.Context, tenantID int64, amount float64) { } } +// unfreezeClampLua 解冻 frozen_balance 并保证不低于 0（对齐 DB 侧 GREATEST(frozen_balance - ?, 0) 下限）。 +// v2：金额为整数 micro，整数运算无浮点漂移。钱包 hash 不存在（TTL 过期）时直接返回， +// 不凭空创建只含 frozen_balance 的残缺 key——冻结状态以 DB 为准，下次 PreDeduct 的 doSyncWalletToRedis 会重建。 +// KEYS[1] = wallet:v2:{tenant_id}；ARGV[1] = 解冻金额（整数 micro） +const unfreezeClampLua = ` +local wallet_key = KEYS[1] +local amount = tonumber(ARGV[1]) +if redis.call("EXISTS", wallet_key) == 0 then + return 0 +end +local frozen = tonumber(redis.call("HGET", wallet_key, "frozen_balance") or "0") +local newFrozen = frozen - amount +if newFrozen < 0 then + newFrozen = 0 +end +redis.call("HSET", wallet_key, "frozen_balance", newFrozen) +return 1 +` + // UnfreezePreDeduct 解冻预扣金额（请求失败时调用） func UnfreezePreDeduct(ctx context.Context, tenantID int64, requestID string, amount float64) { if amount <= 0 { @@ -253,14 +276,16 @@ func UnfreezePreDeduct(ctx context.Context, tenantID int64, requestID string, am } predeductRedisKey := fmt.Sprintf("%s%s", PreDeductRedisKeyPrefix, requestID) - walletRedisKey := fmt.Sprintf("wallet:%d", tenantID) + walletRedisKey := walletHashKey(tenantID) activeSetKey := fmt.Sprintf("prededuct_active:%d", tenantID) // 先尝试 Redis 解冻 _, err := g.Redis().Do(ctx, "DEL", predeductRedisKey) if err == nil { - // 解冻 - g.Redis().Do(ctx, "HINCRBYFLOAT", walletRedisKey, "frozen_balance", -amount) + // 解冻（带 0 下限保护，整数运算）。HINCRBY 亦无下限：若因重复/多余调用扣减超过已冻结额， + // 会把 frozen_balance 打成负数。改用 Lua 读-clamp-写，与 DB 侧 + // GREATEST(frozen_balance - ?, 0) 保持一致；钱包 hash 不存在时不凭空创建（交由 DB 兜底）。 + g.Redis().Do(ctx, "EVAL", unfreezeClampLua, 1, walletRedisKey, toMicro(amount)) g.Redis().Do(ctx, "SREM", activeSetKey, requestID) go unfreezeSyncDB(tenantID, amount) return @@ -276,21 +301,14 @@ func unfreezeSyncDB(tenantID int64, amount float64) { } func unfreezeDB(ctx context.Context, tenantID int64, amount float64) { - type walletRow struct { - ID int64 `json:"id"` + // A9：单条原子更新即可，无需先 SELECT 再 UPDATE。 + // GREATEST(frozen_balance - ?, 0) 保证不会扣成负数；WHERE tenant_id 直接定位钱包行。 + _, err := g.DB().Exec(ctx, + "UPDATE bil_wallets SET frozen_balance = GREATEST(frozen_balance - ?, 0), updated_at = ? WHERE tenant_id = ?", + amount, time.Now(), tenantID) + if err != nil { + g.Log().Errorf(ctx, "unfreeze db: tenant=%d amount=%.6f: %v", tenantID, amount, err) } - var w *walletRow - err := dao.BilWallets.Ctx(ctx). - Where("tenant_id", tenantID). - Fields("id"). - Scan(&w) - if err != nil || w == nil { - return - } - - g.DB().Exec(ctx, - "UPDATE bil_wallets SET frozen_balance = GREATEST(frozen_balance - ?, 0), updated_at = ? WHERE id = ? ", - amount, time.Now(), w.ID) walletCache.Delete(ctx, fmt.Sprintf("%d", tenantID)) } @@ -300,7 +318,8 @@ func GetPreDeductAmount(ctx context.Context, requestID string) (float64, bool) { predeductRedisKey := fmt.Sprintf("%s%s", PreDeductRedisKeyPrefix, requestID) result, err := g.Redis().Do(ctx, "HGET", predeductRedisKey, "amount") if err == nil && !result.IsNil() { - return result.Float64(), true + // v2：amount 以整数 micro 存储，换算回 USD + return fromMicro(result.Int64()), true } return 0, false } @@ -318,7 +337,7 @@ func syncWalletToRedis(ctx context.Context, tenantID int64) error { // 每次预扣前调用，确保 Redis 中的 balance 与 DB 一致 // frozen_balance 由 Redis Lua 脚本管理，仅在 key 首次创建时从 DB 初始化 func doSyncWalletToRedis(ctx context.Context, tenantID int64) error { - walletRedisKey := fmt.Sprintf("wallet:%d", tenantID) + walletRedisKey := walletHashKey(tenantID) // 从 DB 读取钱包数据（跳过内存缓存，直接查库确保最新） type walletRow struct { @@ -341,10 +360,10 @@ func doSyncWalletToRedis(ctx context.Context, tenantID int64) error { exists, _ := g.Redis().Do(ctx, "EXISTS", walletRedisKey) if exists.Int64() == 0 { - // key 不存在：完整初始化（balance + frozen_balance） + // key 不存在：完整初始化（balance + frozen_balance，均为整数 micro） _, err = g.Redis().Do(ctx, "HMSET", walletRedisKey, - "balance", w.Balance, - "frozen_balance", w.FrozenBalance, + "balance", toMicro(w.Balance), + "frozen_balance", toMicro(w.FrozenBalance), ) if err != nil { return gerror.Wrapf(err, "sync wallet to redis") @@ -355,7 +374,7 @@ func doSyncWalletToRedis(ctx context.Context, tenantID int64) error { } else { // key 已存在：只更新 balance（frozen_balance 由 Lua 脚本管理，不覆盖） _, err = g.Redis().Do(ctx, "HSET", walletRedisKey, - "balance", w.Balance, + "balance", toMicro(w.Balance), ) } if err != nil { @@ -370,8 +389,7 @@ func doSyncWalletToRedis(ctx context.Context, tenantID int64) error { // InvalidateWalletRedis 清除 Redis 中的钱包缓存（余额变更后调用） func InvalidateWalletRedis(ctx context.Context, tenantID int64) { - walletRedisKey := fmt.Sprintf("wallet:%d", tenantID) - g.Redis().Do(ctx, "DEL", walletRedisKey) + g.Redis().Do(ctx, "DEL", walletHashKey(tenantID)) } // CleanupPreDeduct 清理预扣记录（结算成功后调用） @@ -429,7 +447,8 @@ func GetFrozenItems(ctx context.Context, tenantID int64) ([]FrozenItem, error) { var amount float64 if v, ok := m["amount"]; ok { - amount = gconv.Float64(v) + // v2：amount 以整数 micro 存储，换算回 USD + amount = fromMicro(gconv.Int64(v)) } var modelName string @@ -507,7 +526,7 @@ func rebuildPredeductFromDB(ctx context.Context, tenantID int64) { predeductKey := fmt.Sprintf("%s%s", PreDeductRedisKeyPrefix, t.RequestID) g.Redis().Do(ctx, "HMSET", predeductKey, - "amount", t.Amount, + "amount", toMicro(t.Amount), "tenant_id", tenantID, "model_name", t.ModelName, "created_at", t.CreatedAt, diff --git a/internal/logic/billing/wallet_test.go b/internal/logic/billing/wallet_test.go index 145883b..06fd75e 100644 --- a/internal/logic/billing/wallet_test.go +++ b/internal/logic/billing/wallet_test.go @@ -54,8 +54,8 @@ func TestWalletInfo_Fields(t *testing.T) { } func TestPreDeductConstants(t *testing.T) { - if PreDeductRedisKeyPrefix != "prededuct:" { - t.Errorf("PreDeductRedisKeyPrefix = %q, want %q", PreDeductRedisKeyPrefix, "prededuct:") + if PreDeductRedisKeyPrefix != "prededuct:v2:" { + t.Errorf("PreDeductRedisKeyPrefix = %q, want %q", PreDeductRedisKeyPrefix, "prededuct:v2:") } if PreDeductMaxAge != 1800 { t.Errorf("PreDeductMaxAge = %d, want 1800", PreDeductMaxAge) diff --git a/internal/logic/common/cache.go b/internal/logic/common/cache.go index 371ed81..df19188 100644 --- a/internal/logic/common/cache.go +++ b/internal/logic/common/cache.go @@ -5,6 +5,7 @@ import ( "encoding/json" "time" + "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gcache" "github.com/gogf/gf/v2/util/gconv" @@ -195,5 +196,52 @@ func PublishInvalidation(ctx context.Context, fullKey string) { _, _ = g.Redis().Do(ctx, "PUBLISH", "cache:invalidate", fullKey) } +// StartCacheInvalidationSubscriber 订阅 Redis "cache:invalidate" 频道，收到失效通知时移除本进程 L1(gcache)中对应 key。 +// C1 修复：此前 Cache.Delete / PublishInvalidation 只向 cache:invalidate 频道 PUBLISH，全仓库无人 SUBSCRIBE， +// 多实例部署下某实例删除 key 后，其他实例的 L1 gcache 会一直陈旧到自然 TTL。此订阅方补齐跨实例 L1 失效。 +// 在 cmd.go 启动时调用一次；断线自动重连，语义与 ConfigService.StartSubscriber 一致。 +func StartCacheInvalidationSubscriber(ctx context.Context) { + go func() { + var reconnectCount int + for { + conn, _, err := g.Redis().Subscribe(ctx, "cache:invalidate") + if err != nil { + reconnectCount++ + g.Log().Warningf(ctx, "[PubSub:cache] 连接失败 (第%d次): %v", reconnectCount, err) + time.Sleep(5 * time.Second) + continue + } + + if reconnectCount > 0 { + g.Log().Infof(ctx, "[PubSub:cache] 重连成功 (此前失败%d次)", reconnectCount) + reconnectCount = 0 + } else { + g.Log().Info(ctx, "[PubSub:cache] 订阅已启动") + } + + for { + v, err := conn.Receive(ctx) + if err != nil { + reconnectCount++ + g.Log().Warningf(ctx, "[PubSub:cache] 接收错误 (第%d次): %v", reconnectCount, err) + time.Sleep(5 * time.Second) + break // reconnect + } + + msg, ok := v.Val().(*gredis.Message) + if !ok { + continue // skip Subscription/Pong 等 + } + + // 仅移除本进程 L1；L2(Redis) 由发布方已 DEL，无需重复处理 + gcache.Remove(ctx, msg.Payload) + g.Log().Debugf(ctx, "[PubSub:cache] L1 已失效: %s", msg.Payload) + } + + conn.Close(ctx) + } + }() +} + // TenantGroupModelCache 缓存租户通过分组可访问的模型集合，TTL 300s var TenantGroupModelCache = NewCache("tenant_group_models", 300*time.Second) diff --git a/internal/logic/common/cron.go b/internal/logic/common/cron.go index e4b5959..ba87ece 100644 --- a/internal/logic/common/cron.go +++ b/internal/logic/common/cron.go @@ -8,6 +8,7 @@ import ( "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gctx" + "github.com/gogf/gf/v2/util/guid" "github.com/robfig/cron/v3" ) @@ -161,6 +162,16 @@ func (cs *CronScheduler) runJobInternal(ctx context.Context, name, triggeredBy s return gerror.Newf("job %s not registered", name) } + // 分布式锁（C2）：多实例部署下，同名 job 只允许一个实例执行，避免日结算/告警检测/任务领取等 + // 重复触发。进程内 running map 仅单实例互斥，跨实例无效。auto 与 manual 均加锁，防止手动触发与 + // 另一实例的定时执行重叠。未获取到锁说明别的实例正在执行，本次跳过。 + lockToken, locked := acquireCronLock(ctx, name) + if !locked { + g.Log().Infof(ctx, "cron job %s skipped: distributed lock held by another instance", name) + return nil + } + defer releaseCronLock(ctx, name, lockToken) + // Execute job startTime := time.Now() handlerErr := job.Handler(ctx) @@ -273,3 +284,39 @@ func (cs *CronScheduler) StartBackground(ctx context.Context) { g.Log().Info(ctx, "cron scheduler started in background") } + +// cronLockTTL 分布式锁过期时间：仅用于实例崩溃时的兜底自动释放（正常完成会主动释放）。 +// 取值需大于绝大多数 job 的执行时长；过长则崩溃后会阻塞该 job 至多这么久。 +const cronLockTTL = 10 * time.Minute + +// cronLockKey 返回某 job 的分布式锁 Redis key。 +func cronLockKey(name string) string { + return "cron:lock:" + name +} + +// acquireCronLock 通过 Redis SET NX EX 原子获取 job 分布式锁。 +// 返回 (token, true) 表示获取成功；("", false) 表示锁被其他实例持有。 +// Redis 不可用时降级为「获取成功但无 token」——退回进程内互斥语义（可能重复执行）， +// 优于因 Redis 抖动导致所有实例都不执行（关键 job 如结算被整体跳过）。 +func acquireCronLock(ctx context.Context, name string) (string, bool) { + token := guid.S() + res, err := g.Redis().Do(ctx, "SET", cronLockKey(name), token, "NX", "EX", int64(cronLockTTL.Seconds())) + if err != nil { + g.Log().Warningf(ctx, "cron lock acquire failed for %s, running without distributed lock: %v", name, err) + return "", true + } + if res.IsNil() { + return "", false + } + return token, true +} + +// releaseCronLock 释放分布式锁：仅当锁仍属于本次持有的 token 时才删除（Lua CAS）， +// 避免锁已过期被别的实例重新持有后被误删。token 为空（Redis 降级）时不操作。 +func releaseCronLock(ctx context.Context, name, token string) { + if token == "" { + return + } + const lua = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end` + _, _ = g.Redis().Do(ctx, "EVAL", lua, 1, cronLockKey(name), token) +} diff --git a/internal/logic/common/security.go b/internal/logic/common/security.go index 9da9a58..b3363c8 100644 --- a/internal/logic/common/security.go +++ b/internal/logic/common/security.go @@ -64,6 +64,10 @@ func GenerateProvisionalToken(ctx context.Context, userID int64, userType, role // ParseProvisionalToken parses and validates a provisional 2FA token. func ParseProvisionalToken(tokenStr string) (*ProvisionalClaims, error) { token, err := jwt.ParseWithClaims(tokenStr, &ProvisionalClaims{}, func(token *jwt.Token) (any, error) { + // 校验签名算法，防止 "alg":"none" 或非对称算法伪造令牌 + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, gerror.Newf("unexpected signing method: %v", token.Header["alg"]) + } return GetJWTSecret(), nil }) if err != nil { @@ -97,6 +101,10 @@ func GenerateConfirmToken(ctx context.Context, userID int64, userType string) (s // ParseConfirmToken parses and validates a high-risk confirm token. func ParseConfirmToken(tokenStr string) (*ConfirmClaims, error) { token, err := jwt.ParseWithClaims(tokenStr, &ConfirmClaims{}, func(token *jwt.Token) (any, error) { + // 校验签名算法，防止 "alg":"none" 或非对称算法伪造令牌 + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, gerror.Newf("unexpected signing method: %v", token.Header["alg"]) + } return GetJWTSecret(), nil }) if err != nil { diff --git a/internal/logic/common/session.go b/internal/logic/common/session.go index bb35e59..2ab7dff 100644 --- a/internal/logic/common/session.go +++ b/internal/logic/common/session.go @@ -34,9 +34,10 @@ func CreateSession(ctx context.Context, userType string, userID, tenantID int64, refreshExpire := getRefreshExpire(ctx) expiresAt := gtime.New(time.Now().Add(refreshExpire)) + // 事务采用 ctx 传播式写法：闭包内统一使用 dao.Xxx.Ctx(ctx)，事务由 ctx 自动挂载。 err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Count existing active sessions - count, err := tx.Model("sys_sessions").Ctx(ctx). + count, err := dao.SysSessions.Ctx(ctx). Where("user_type", userType). Where("user_id", userID). Where("expires_at > NOW()"). @@ -51,7 +52,7 @@ func CreateSession(ctx context.Context, userType string, userID, tenantID int64, // PostgreSQL does not support DELETE ... ORDER BY ... LIMIT, // so use a subquery to find the oldest session IDs first. var oldIDs []int64 - err = tx.Model("sys_sessions").Ctx(ctx). + err = dao.SysSessions.Ctx(ctx). Fields("id"). Where("user_type", userType). Where("user_id", userID). @@ -63,7 +64,7 @@ func CreateSession(ctx context.Context, userType string, userID, tenantID int64, return gerror.Wrapf(err, "find old sessions") } if len(oldIDs) > 0 { - _, err = tx.Model("sys_sessions").Ctx(ctx). + _, err = dao.SysSessions.Ctx(ctx). WhereIn("id", oldIDs). Delete() if err != nil { @@ -73,7 +74,7 @@ func CreateSession(ctx context.Context, userType string, userID, tenantID int64, } // Insert new session - result, err := tx.Model("sys_sessions").Ctx(ctx).Data(do.SysSessions{ + result, err := dao.SysSessions.Ctx(ctx).Data(do.SysSessions{ UserType: userType, UserId: userID, TenantId: tenantID, diff --git a/internal/logic/common/settings_registry.go b/internal/logic/common/settings_registry.go index 1ff60fb..1b93fcf 100644 --- a/internal/logic/common/settings_registry.go +++ b/internal/logic/common/settings_registry.go @@ -221,8 +221,8 @@ var Registry = []SettingDef{ Label: "健康快照保留天数", Validation: "min:1,max:90"}, {Key: "channel_proxy_url", Type: SettingTypeString, Default: "", Category: "channel", Label: "代理地址", Description: "全局代理 URL，支持 http:// 和 socks5://，如 http://127.0.0.1:7890。启用代理的渠道会通过此代理转发请求"}, - {Key: "sync_image_async_enabled", Type: SettingTypeBool, Default: "true", Category: "channel", - Label: "同步图片厂商异步化", Description: "开启后，同步阻塞返回的图片厂商（如 OpenAI/DALL·E）走 /v1/images/generations/async 时由后台 worker 池异步处理，客户端提交即拿 task_id 后轮询取图；关闭则该端点对同步厂商返回不支持"}, + {Key: "sync_image_async_enabled", Type: SettingTypeBool, Default: "false", Category: "channel", + Label: "同步图片厂商异步化", Description: "开启后，同步阻塞返回的图片厂商（如 OpenAI/DALL·E）走 /v1/images/generations/async 时由后台 worker 池异步处理，客户端提交即拿 task_id 后轮询取图；关闭则该端点对同步厂商返回不支持。依赖对象存储保存生成结果，开启前需先在「存储配置」中配置对象存储（OSS/S3/COS）"}, {Key: "sync_image_rehost_url", Type: SettingTypeBool, Default: "false", Category: "channel", Label: "同步图片 URL 转存对象存储", Description: "开启后，上游返回图片 URL 时下载并转存对象存储（返回 24h 稳定链接，需已配置存储）；关闭则直接透传上游 URL（部分厂商约 1h 过期）。b64_json 始终转存"}, diff --git a/internal/logic/common/storage.go b/internal/logic/common/storage.go index 993dcea..4561c44 100644 --- a/internal/logic/common/storage.go +++ b/internal/logic/common/storage.go @@ -10,34 +10,32 @@ import ( "time" "github.com/gogf/gf/v2/errors/gerror" - "github.com/gogf/gf/v2/frame/g" "github.com/google/uuid" ) -// StorageProvider defines the interface for file storage backends. +// StorageProvider 定义文件存储后端的统一接口。 type StorageProvider interface { - // Upload uploads a file and returns the storage path. + // Upload 上传文件并返回存储路径。 Upload(ctx context.Context, reader io.Reader, key string, contentType string) (string, error) - // Download returns a reader for the file at the given path. + // Download 返回指定路径文件的读取器。 Download(ctx context.Context, key string) (io.ReadCloser, error) - // Delete deletes a file. + // Delete 删除文件。 Delete(ctx context.Context, key string) error - // PresignedURL generates a temporary URL for downloading. + // PresignedURL 生成用于下载的临时 URL。 PresignedURL(ctx context.Context, key string, expires time.Duration) (string, error) - // PresignedThumbnailURL generates a temporary URL that returns a resized - // thumbnail (width in pixels, height auto). Providers with native - // server-side image processing (OSS/COS) apply it in the signed URL; - // providers without it (S3/MinIO/R2) return the original-object URL. + // PresignedThumbnailURL 生成返回缩略图的临时 URL（宽度按像素，高度自适应）。 + // 具备原生服务端图片处理能力的 provider（OSS/COS）在签名 URL 中直接应用缩放； + // 不具备的 provider（S3/MinIO/R2）回退返回原图对象的 URL。 PresignedThumbnailURL(ctx context.Context, key string, width int, expires time.Duration) (string, error) } -// FileService provides file storage operations with metadata tracking. +// FileService 提供带元数据追踪的文件存储操作。 type FileService struct { provider StorageProvider providerName string } -// NewFileService creates a new FileService with the given provider. +// NewFileService 用给定的 provider 创建一个 FileService。 func NewFileService(provider StorageProvider) *FileService { return &FileService{ provider: provider, @@ -45,7 +43,7 @@ func NewFileService(provider StorageProvider) *FileService { } } -// FileUpload represents a file upload request. +// FileUpload 表示一次文件上传请求。 type FileUpload struct { Reader io.Reader Filename string @@ -55,7 +53,7 @@ type FileUpload struct { UserID int64 } -// FileRecord represents a file metadata record. +// FileRecord 表示一条文件元数据记录。 type FileRecord struct { ID int64 `json:"id"` TenantID int64 `json:"tenant_id"` @@ -71,26 +69,40 @@ type FileRecord struct { CreatedAt string `json:"created_at"` } -// Upload uploads a file, stores it via the provider, and records metadata. +// applyStoragePrefix 幂等地为原始 key 拼接配置的路径前缀。 +// +// 兼容性关键：早期版本的 FileService.Upload 持久化的是 provider 返回的**已带前缀** key +// （如 team-api/42/..），现已改为持久化 raw key（42/..）并在读写时由 provider 重新加前缀。 +// 若对存量行再无条件加一次前缀，会得到 team-api/team-api/42/.. → 删除/下载/缩略图全部失效。 +// 因此这里对「已带前缀」的 key 原样返回，使 Delete/PresignedURL/Download 对旧行（带前缀）与 +// 新行（raw）都正确，无需数据迁移。raw key 以数字 tenant_id 开头，绝不会与前缀名冲突。 +func applyStoragePrefix(prefix, key string) string { + if prefix == "" { + return key + } + if key == prefix || strings.HasPrefix(key, prefix+"/") { + return key + } + return prefix + "/" + key +} + +// Upload 上传文件、通过 provider 存储，并写入元数据记录。 func (s *FileService) Upload(ctx context.Context, upload *FileUpload) (*FileRecord, error) { - // Generate storage key (relative to the provider's configured path prefix): - // {date}/{uuid}{ext} — date-based dirs for lifecycle policies, UUID for - // uniqueness and non-guessability, original extension retained. + // 生成存储 key（相对于 provider 配置的路径前缀）：{日期}/{uuid}{扩展名}—— + // 按日期分目录便于生命周期策略，UUID 保证唯一且不可猜测，保留原始扩展名。 ext := extFromFilename(upload.Filename) uuidStr := uuid.New().String() key := fmt.Sprintf("%s/%s%s", time.Now().Format("2006-01-02"), uuidStr, ext) fileName := uuidStr + ext - // Upload to storage provider. The provider applies the configured path - // prefix internally, so we persist the RAW key (not the returned full key): - // Download/Delete/PresignedURL pass this key back through the provider, - // which re-applies the prefix exactly once. Storing the prefixed key here - // would cause the prefix to be applied twice on retrieval. + // 上传到存储 provider。provider 内部会应用配置的路径前缀，因此这里持久化 raw key + // （而非返回的 full key）：Download/Delete/PresignedURL 会把该 key 再次传回 provider， + // 由其恰好再加一次前缀。若在此存储已带前缀的 key，读取时会被重复加前缀两次。 if _, err := s.provider.Upload(ctx, upload.Reader, key, upload.ContentType); err != nil { return nil, gerror.Wrapf(err, "upload to storage") } - // Record metadata + // 写入元数据记录 result, err := dao.FilFiles.Ctx(ctx).Data(do.FilFiles{ TenantId: upload.TenantID, UserId: upload.UserID, @@ -103,7 +115,7 @@ func (s *FileService) Upload(ctx context.Context, upload *FileUpload) (*FileReco VirusScanStatus: "pending", }).Insert() if err != nil { - // Try to clean up the uploaded file + // 元数据写入失败，尝试清理已上传的文件 _ = s.provider.Delete(ctx, key) return nil, gerror.Wrapf(err, "insert file record") } @@ -124,7 +136,7 @@ func (s *FileService) Upload(ctx context.Context, upload *FileUpload) (*FileReco }, nil } -// GetDownloadURL generates a presigned URL for downloading a file. +// GetDownloadURL 生成用于下载文件的预签名 URL。 func (s *FileService) GetDownloadURL(ctx context.Context, fileID int64) (string, error) { var record *FileRecord err := dao.FilFiles.Ctx(ctx). @@ -140,9 +152,8 @@ func (s *FileService) GetDownloadURL(ctx context.Context, fileID int64) (string, return s.provider.PresignedURL(ctx, record.StoragePath, 24*time.Hour) } -// GetThumbnailURL generates a presigned URL for previewing a downscaled -// thumbnail. Only images get server-side resizing; non-image files (and -// providers without native image processing) fall back to the original object. +// GetThumbnailURL 生成用于预览缩略图的预签名 URL。只有图片会做服务端缩放； +// 非图片文件（以及不支持原生图片处理的 provider）回退到原图对象。 func (s *FileService) GetThumbnailURL(ctx context.Context, fileID int64, width int) (string, error) { var record *FileRecord err := dao.FilFiles.Ctx(ctx). @@ -158,14 +169,14 @@ func (s *FileService) GetThumbnailURL(ctx context.Context, fileID int64, width i if width <= 0 { width = 400 } - // Non-image objects cannot be resized; return the original presigned URL. + // 非图片对象无法缩放；返回原图的预签名 URL。 if !strings.HasPrefix(record.MimeType, "image/") { return s.provider.PresignedURL(ctx, record.StoragePath, 24*time.Hour) } return s.provider.PresignedThumbnailURL(ctx, record.StoragePath, width, 24*time.Hour) } -// Delete deletes a file from storage and marks it as deleted. +// Delete 从存储中删除文件并删除其元数据记录。 func (s *FileService) Delete(ctx context.Context, fileID int64) error { var record *FileRecord err := dao.FilFiles.Ctx(ctx). @@ -178,12 +189,14 @@ func (s *FileService) Delete(ctx context.Context, fileID int64) error { return gerror.Newf("file not found: %d", fileID) } - // Delete from storage + // 从存储删除对象。删除失败时**不**继续删库行——否则对象会成为孤儿且失去 DB 索引， + // 之后再也无法定位清理。保留库行并返回错误，交由调用方（保留期清理循环）下一轮重试。 + // S3/OSS/COS 删除不存在的对象均返回成功，故此处的错误代表真实的瞬时/权限问题，值得重试。 if err := s.provider.Delete(ctx, record.StoragePath); err != nil { - g.Log().Warningf(ctx, "delete file from storage: %v", err) + return gerror.Wrapf(err, "delete file %d from storage", fileID) } - // Delete record + // 删除记录 _, err = dao.FilFiles.Ctx(ctx). Where("id", fileID). Delete() @@ -194,8 +207,7 @@ func (s *FileService) Delete(ctx context.Context, fileID int64) error { return nil } -// extFromFilename extracts the file extension (including the leading dot) -// from a filename. Returns empty string if no extension is found. +// extFromFilename 从文件名中提取扩展名（含前导点）。无扩展名时返回空字符串。 func extFromFilename(name string) string { if idx := strings.LastIndex(name, "."); idx > 0 { return name[idx:] diff --git a/internal/logic/common/storage_cos.go b/internal/logic/common/storage_cos.go index b66dc22..9b3663b 100644 --- a/internal/logic/common/storage_cos.go +++ b/internal/logic/common/storage_cos.go @@ -11,7 +11,7 @@ import ( "github.com/tencentyun/cos-go-sdk-v5" ) -// COSStorageProvider implements StorageProvider using Tencent COS. +// COSStorageProvider 基于腾讯云 COS 实现 StorageProvider。 type COSStorageProvider struct { client *cos.Client bucket string @@ -20,7 +20,7 @@ type COSStorageProvider struct { secretKey string } -// NewCOSProvider creates a new Tencent COS storage provider. +// NewCOSProvider 创建一个腾讯云 COS 存储 provider。 func NewCOSProvider(cfg *StorageConfig) (*COSStorageProvider, error) { if cfg.Endpoint == "" { return nil, fmt.Errorf("cos endpoint is required") @@ -49,15 +49,12 @@ func NewCOSProvider(cfg *StorageConfig) (*COSStorageProvider, error) { }, nil } -// fullKey returns the full object key with prefix. +// fullKey 返回带前缀的完整对象 key（对已带前缀的存量 key 幂等）。 func (s *COSStorageProvider) fullKey(key string) string { - if s.prefix != "" { - return s.prefix + "/" + key - } - return key + return applyStoragePrefix(s.prefix, key) } -// Upload uploads a file to Tencent COS and returns the storage key. +// Upload 上传文件到腾讯云 COS 并返回存储 key。 func (s *COSStorageProvider) Upload(ctx context.Context, reader io.Reader, key string, contentType string) (string, error) { fullKey := s.fullKey(key) @@ -74,7 +71,7 @@ func (s *COSStorageProvider) Upload(ctx context.Context, reader io.Reader, key s return fullKey, nil } -// Download returns a reader for the file at the given key. +// Download 返回指定 key 文件的读取器。 func (s *COSStorageProvider) Download(ctx context.Context, key string) (io.ReadCloser, error) { fullKey := s.fullKey(key) @@ -86,7 +83,7 @@ func (s *COSStorageProvider) Download(ctx context.Context, key string) (io.ReadC return resp.Body, nil } -// Delete deletes a file from Tencent COS. +// Delete 从腾讯云 COS 删除文件。 func (s *COSStorageProvider) Delete(ctx context.Context, key string) error { fullKey := s.fullKey(key) @@ -98,7 +95,7 @@ func (s *COSStorageProvider) Delete(ctx context.Context, key string) error { return nil } -// PresignedURL generates a temporary presigned URL for downloading. +// PresignedURL 生成用于下载的临时预签名 URL。 func (s *COSStorageProvider) PresignedURL(ctx context.Context, key string, expires time.Duration) (string, error) { fullKey := s.fullKey(key) @@ -110,9 +107,9 @@ func (s *COSStorageProvider) PresignedURL(ctx context.Context, key string, expir return presignedURL.String(), nil } -// PresignedThumbnailURL signs a URL with a COS CI imageMogr2/thumbnail op so COS -// returns a downscaled thumbnail (width px, height auto). The process param is -// injected into the signed query so the signature covers it. +// PresignedThumbnailURL 在签名 URL 中附加 COS CI 的 imageMogr2/thumbnail 处理参数， +// 让 COS 返回按比例缩小的缩略图（宽度按像素，高度自适应）。处理参数被注入到签名的 +// query 中，从而被签名覆盖。 func (s *COSStorageProvider) PresignedThumbnailURL(ctx context.Context, key string, width int, expires time.Duration) (string, error) { fullKey := s.fullKey(key) diff --git a/internal/logic/common/storage_factory.go b/internal/logic/common/storage_factory.go index 73d65df..9f89b16 100644 --- a/internal/logic/common/storage_factory.go +++ b/internal/logic/common/storage_factory.go @@ -2,10 +2,18 @@ package common import ( "context" + "errors" "fmt" ) -// StorageConfig holds the configuration for a storage provider. +// ErrStorageNotConfigured 表示对象存储尚未配置（缺少 provider / bucket）。 +// +// 用哨兵错误而非普通 error，便于上层（如 sync_image worker）用 errors.Is 精确识别「未配置」 +// 这一**可操作**场景，给出面向用户的友好提示（引导去系统设置配置 OSS），从而与上传/下载 +// 等技术性失败区分开。 +var ErrStorageNotConfigured = errors.New("object storage not configured") + +// StorageConfig 保存某个存储 provider 的配置。 type StorageConfig struct { Provider string Endpoint string @@ -17,7 +25,7 @@ type StorageConfig struct { PathPrefix string } -// GetStorageConfig reads storage configuration from the config service. +// GetStorageConfig 从配置服务读取存储配置。 func GetStorageConfig(ctx context.Context) *StorageConfig { cfg := &StorageConfig{ Provider: Config().GetString(ctx, "storage_provider"), @@ -35,7 +43,7 @@ func GetStorageConfig(ctx context.Context) *StorageConfig { return cfg } -// NewStorageProvider creates a StorageProvider based on the given configuration. +// NewStorageProvider 根据给定配置创建对应的 StorageProvider。 func NewStorageProvider(cfg *StorageConfig) (StorageProvider, error) { switch cfg.Provider { case "s3", "minio", "r2": @@ -49,12 +57,19 @@ func NewStorageProvider(cfg *StorageConfig) (StorageProvider, error) { } } -// NewFileServiceFromConfig creates a FileService using the storage configuration -// from the config service (database settings). +// IsStorageConfigured 轻量判断对象存储是否已配置（provider + bucket 齐全）。 +// 只读配置、不构造 provider、不产生网络调用，供提交阶段的 fast-fail 使用。 +func IsStorageConfigured(ctx context.Context) bool { + cfg := GetStorageConfig(ctx) + return cfg.Provider != "" && cfg.Bucket != "" +} + +// NewFileServiceFromConfig 使用配置服务（数据库设置）中的存储配置创建一个 FileService。 +// 未配置（缺 provider 或 bucket）时返回 ErrStorageNotConfigured 哨兵错误，供上层友好提示。 func NewFileServiceFromConfig(ctx context.Context) (*FileService, error) { cfg := GetStorageConfig(ctx) - if cfg.Bucket == "" { - return nil, fmt.Errorf("storage bucket not configured") + if cfg.Provider == "" || cfg.Bucket == "" { + return nil, ErrStorageNotConfigured } provider, err := NewStorageProvider(cfg) diff --git a/internal/logic/common/storage_oss.go b/internal/logic/common/storage_oss.go index 4272401..0c3d6e6 100644 --- a/internal/logic/common/storage_oss.go +++ b/internal/logic/common/storage_oss.go @@ -9,14 +9,14 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" ) -// OSSStorageProvider implements StorageProvider using Aliyun OSS. +// OSSStorageProvider 基于阿里云 OSS 实现 StorageProvider。 type OSSStorageProvider struct { client *oss.Client bucket *oss.Bucket prefix string } -// NewOSSProvider creates a new Aliyun OSS storage provider. +// NewOSSProvider 创建一个阿里云 OSS 存储 provider。 func NewOSSProvider(cfg *StorageConfig) (*OSSStorageProvider, error) { if cfg.Endpoint == "" { return nil, fmt.Errorf("oss endpoint is required") @@ -39,15 +39,12 @@ func NewOSSProvider(cfg *StorageConfig) (*OSSStorageProvider, error) { }, nil } -// fullKey returns the full object key with prefix. +// fullKey 返回带前缀的完整对象 key（对已带前缀的存量 key 幂等）。 func (s *OSSStorageProvider) fullKey(key string) string { - if s.prefix != "" { - return s.prefix + "/" + key - } - return key + return applyStoragePrefix(s.prefix, key) } -// Upload uploads a file to Aliyun OSS and returns the storage key. +// Upload 上传文件到阿里云 OSS 并返回存储 key。 func (s *OSSStorageProvider) Upload(ctx context.Context, reader io.Reader, key string, contentType string) (string, error) { fullKey := s.fullKey(key) @@ -62,7 +59,7 @@ func (s *OSSStorageProvider) Upload(ctx context.Context, reader io.Reader, key s return fullKey, nil } -// Download returns a reader for the file at the given key. +// Download 返回指定 key 文件的读取器。 func (s *OSSStorageProvider) Download(ctx context.Context, key string) (io.ReadCloser, error) { fullKey := s.fullKey(key) @@ -74,7 +71,7 @@ func (s *OSSStorageProvider) Download(ctx context.Context, key string) (io.ReadC return resp, nil } -// Delete deletes a file from Aliyun OSS. +// Delete 从阿里云 OSS 删除文件。 func (s *OSSStorageProvider) Delete(ctx context.Context, key string) error { fullKey := s.fullKey(key) @@ -86,7 +83,7 @@ func (s *OSSStorageProvider) Delete(ctx context.Context, key string) error { return nil } -// PresignedURL generates a temporary signed URL for downloading. +// PresignedURL 生成用于下载的临时签名 URL。 func (s *OSSStorageProvider) PresignedURL(ctx context.Context, key string, expires time.Duration) (string, error) { fullKey := s.fullKey(key) @@ -98,8 +95,8 @@ func (s *OSSStorageProvider) PresignedURL(ctx context.Context, key string, expir return signedURL, nil } -// PresignedThumbnailURL signs a URL with an x-oss-process image/resize op so OSS -// returns a downscaled thumbnail (width px, height auto, no upscaling). +// PresignedThumbnailURL 在签名 URL 中附加 x-oss-process 的 image/resize 处理参数， +// 让 OSS 返回按比例缩小的缩略图（宽度按像素，高度自适应，不放大）。 func (s *OSSStorageProvider) PresignedThumbnailURL(ctx context.Context, key string, width int, expires time.Duration) (string, error) { fullKey := s.fullKey(key) diff --git a/internal/logic/common/storage_s3.go b/internal/logic/common/storage_s3.go index 7742b78..5ae010c 100644 --- a/internal/logic/common/storage_s3.go +++ b/internal/logic/common/storage_s3.go @@ -12,14 +12,14 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -// S3StorageProvider implements StorageProvider using AWS S3 / MinIO. +// S3StorageProvider 基于 AWS S3 / MinIO 实现 StorageProvider。 type S3StorageProvider struct { client *s3.Client bucket string prefix string } -// NewS3Provider creates a new S3/MinIO storage provider. +// NewS3Provider 创建一个 S3/MinIO 存储 provider。 func NewS3Provider(cfg *StorageConfig) (*S3StorageProvider, error) { var opts []func(*awsconfig.LoadOptions) error @@ -44,10 +44,14 @@ func NewS3Provider(cfg *StorageConfig) (*S3StorageProvider, error) { client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { if cfg.Region != "" { o.Region = cfg.Region + } else if cfg.Provider == "r2" { + // Cloudflare R2 的 SigV4 签名要求非空 region，官方约定用 "auto"。 + // 后端在此兜底，避免仅依赖前端 UI 填值——经配置接口/DB 直写或 region 被清空时， + // 空 region 会导致签名失败。 + o.Region = "auto" } - // Cloudflare R2 is S3-compatible but requires an account-level - // BaseEndpoint. Unlike MinIO it uses virtual-hosted addressing, so - // path-style must NOT be forced. + // Cloudflare R2 兼容 S3，但需要账号级的 BaseEndpoint。与 MinIO 不同，它使用 + // 虚拟主机式寻址（virtual-hosted），因此**不能**强制 path-style。 if (cfg.Provider == "minio" || cfg.Provider == "r2") && cfg.Endpoint != "" { o.BaseEndpoint = aws.String(cfg.Endpoint) if cfg.Provider == "minio" { @@ -63,15 +67,12 @@ func NewS3Provider(cfg *StorageConfig) (*S3StorageProvider, error) { }, nil } -// fullKey returns the full object key with prefix. +// fullKey 返回带前缀的完整对象 key（对已带前缀的存量 key 幂等）。 func (s *S3StorageProvider) fullKey(key string) string { - if s.prefix != "" { - return s.prefix + "/" + key - } - return key + return applyStoragePrefix(s.prefix, key) } -// Upload uploads a file to S3/MinIO and returns the storage key. +// Upload 上传文件到 S3/MinIO 并返回存储 key。 func (s *S3StorageProvider) Upload(ctx context.Context, reader io.Reader, key string, contentType string) (string, error) { fullKey := s.fullKey(key) @@ -88,7 +89,7 @@ func (s *S3StorageProvider) Upload(ctx context.Context, reader io.Reader, key st return fullKey, nil } -// Download returns a reader for the file at the given key. +// Download 返回指定 key 文件的读取器。 func (s *S3StorageProvider) Download(ctx context.Context, key string) (io.ReadCloser, error) { fullKey := s.fullKey(key) @@ -103,7 +104,7 @@ func (s *S3StorageProvider) Download(ctx context.Context, key string) (io.ReadCl return resp.Body, nil } -// Delete deletes a file from S3/MinIO. +// Delete 从 S3/MinIO 删除文件。 func (s *S3StorageProvider) Delete(ctx context.Context, key string) error { fullKey := s.fullKey(key) @@ -118,7 +119,7 @@ func (s *S3StorageProvider) Delete(ctx context.Context, key string) error { return nil } -// PresignedURL generates a temporary presigned URL for downloading. +// PresignedURL 生成用于下载的临时预签名 URL。 func (s *S3StorageProvider) PresignedURL(ctx context.Context, key string, expires time.Duration) (string, error) { fullKey := s.fullKey(key) @@ -134,8 +135,8 @@ func (s *S3StorageProvider) PresignedURL(ctx context.Context, key string, expire return req.URL, nil } -// PresignedThumbnailURL falls back to the original object: S3/MinIO/R2 have no -// native server-side image processing, so a true thumbnail is not available. +// PresignedThumbnailURL 回退到原图对象：S3/MinIO/R2 没有原生的服务端图片处理能力， +// 无法生成真正的缩略图。 func (s *S3StorageProvider) PresignedThumbnailURL(ctx context.Context, key string, width int, expires time.Duration) (string, error) { return s.PresignedURL(ctx, key, expires) } diff --git a/internal/logic/common/storage_task_result.go b/internal/logic/common/storage_task_result.go new file mode 100644 index 0000000..07ba136 --- /dev/null +++ b/internal/logic/common/storage_task_result.go @@ -0,0 +1,55 @@ +package common + +import ( + "context" + "strings" + + "github.com/qianfree/team-api/internal/dao" +) + +// TaskResultImageURLs 为「同步图片异步化」任务的结果图生成**新鲜**的预览缩略图 URL 与原图 URL。 +// +// tsk_model_tasks 不直接存 file_id：这些 re-host 结果图以 original_name = "{public_task_id}_{index}{ext}" +// 落在 fil_files（见 task/sync_image_worker.go rehostImage），据此定位任务主图（index 0，按 id 升序取第一条）， +// 再基于文件记录重新签名，因此也顺带规避了任务表里 result_url 24h 预签名过期的问题。 +// +// 只对 re-host 到对象存储的图片任务命中；视频/音频任务或上游直链任务（无 fil_files 记录）返回空串， +// 调用方回退到任务表存储的 result_url，保持向后兼容。thumbWidth<=0 时用 600。 +// +// tenantID 必须传任务所属租户，确保只在该租户的文件里查找，不跨租户。 +func TaskResultImageURLs(ctx context.Context, tenantID int64, publicTaskID string, thumbWidth int) (thumbURL, originalURL string) { + if publicTaskID == "" || tenantID <= 0 { + return "", "" + } + + // 转义 public_task_id 中的下划线，避免 LIKE 的 "_" 通配符误匹配（task_ 前缀本身含下划线）。 + pattern := strings.ReplaceAll(publicTaskID, "_", `\_`) + `\_%` + + var rec *FileRecord + err := dao.FilFiles.Ctx(ctx). + Where("tenant_id", tenantID). + Where("original_name LIKE ?", pattern). + Where("mime_type LIKE ?", "image/%"). + OrderAsc("id"). + Limit(1). + Scan(&rec) + if err != nil || rec == nil { + return "", "" + } + + svc, err := NewFileServiceFromConfig(ctx) + if err != nil { + return "", "" // 对象存储未配置 → 无法生成，回退原 result_url + } + + if thumbWidth <= 0 { + thumbWidth = 600 + } + if u, e := svc.GetThumbnailURL(ctx, rec.ID, thumbWidth); e == nil { + thumbURL = u + } + if u, e := svc.GetDownloadURL(ctx, rec.ID); e == nil { + originalURL = u + } + return thumbURL, originalURL +} diff --git a/internal/logic/common/storage_test.go b/internal/logic/common/storage_test.go index 3d3f176..634383a 100644 --- a/internal/logic/common/storage_test.go +++ b/internal/logic/common/storage_test.go @@ -2,12 +2,10 @@ package common import "testing" -// TestProviderFullKeyAppliesPrefixOnce guards the invariant behind the -// storage_path fix: every provider prepends the configured path prefix exactly -// once to whatever key it is given. FileService therefore MUST persist the RAW -// key (relative to the prefix) — if it stored the already-prefixed key returned -// by Upload, Download/Delete/PresignedURL would prefix it a second time and -// target a non-existent object (e.g. "team-api/team-api/…"). +// TestProviderFullKeyAppliesPrefixOnce 守护 storage_path 前缀不变量：每个 provider 恰好 +// 为 key 加一次配置的路径前缀，且对已带前缀的 key 幂等。新行持久化 raw key（相对前缀）， +// 旧行持久化的是 Upload 返回的已带前缀 key。两者都必须解析到同一个「只加一次前缀」的对象， +// 从而 Download/Delete/PresignedURL 无需数据迁移即可正常工作。 func TestProviderFullKeyAppliesPrefixOnce(t *testing.T) { const prefix = "team-api" const rawKey = "42/1700000000/task_abc.png" @@ -24,21 +22,20 @@ func TestProviderFullKeyAppliesPrefixOnce(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - // Raw key -> single prefix (the correct, fixed behavior). + // 新行（raw key）-> 只加一次前缀。 if got := c.fullKey(rawKey); got != want { t.Fatalf("fullKey(raw) = %q, want %q", got, want) } - // An already-prefixed key would be double-prefixed — this is exactly - // the bug we avoid by storing the raw key in fil_files.storage_path. - if got := c.fullKey(want); got != prefix+"/"+want { - t.Fatalf("fullKey(prefixed) = %q, want double-prefixed %q (documents why raw key must be stored)", got, prefix+"/"+want) + // 旧行（已带前缀的 key）-> 原样返回（幂等），不再二次加前缀。 + // 这正是让旧 fil_files 行仍能正确解析的关键。 + if got := c.fullKey(want); got != want { + t.Fatalf("fullKey(prefixed) = %q, want %q (must be idempotent for legacy rows)", got, want) } }) } } -// TestProviderFullKeyNoPrefix verifies that with an empty prefix the key is -// returned unchanged, so keys round-trip identically when no prefix is set. +// TestProviderFullKeyNoPrefix 验证前缀为空时 key 原样返回，即未配置前缀时 key 往返一致。 func TestProviderFullKeyNoPrefix(t *testing.T) { const rawKey = "exports/tenant_7/20240101_000000.json" providers := map[string]func(string) string{ diff --git a/internal/logic/common/tenant_model.go b/internal/logic/common/tenant_model.go index a93dab1..a2f06b2 100644 --- a/internal/logic/common/tenant_model.go +++ b/internal/logic/common/tenant_model.go @@ -138,19 +138,30 @@ func GetTenantAvailableModels(ctx context.Context, tenantID int64, category, sea return result, nil } -// GetAsyncImageModelSet 判定给定图片模型（按 model_id 字符串）是否「走异步图片端点」。 +// ImageModelModes 描述某图片模型可用的调用模式，供在线体验决定同步/异步开关的可用性。 +type ImageModelModes struct { + // AsyncSupported 异步端点（/v1/images/generations/async，提交+轮询）是否可用： + // 真异步厂商，或同步厂商且「同步图片异步化」开关开启。 + AsyncSupported bool + // SyncSupported 同步端点（/v1/images/generations，阻塞一次性返回）是否可用： + // 该模型并非「仅异步」厂商（阿里 image-synthesis 等仅异步厂商为 false）。 + SyncSupported bool +} + +// GetImageModelModes 批量判定给定图片模型（按 model_id 字符串）可用的调用模式。 // // 真相源是服务该模型的渠道 provider 类型：桥接 mdl_models.model_id = chn_abilities.model_name // → chn_channels.type，用 constant.IsAsyncImageModel 分类（与同步端点拦截 gate 同源）。 // - 当某模型的全部在役服务渠道都是异步图片 provider（如 DashScope wanx/qwen-image）时，同步 -// /v1/images/generations 必失败，必须走 /v1/images/generations/async 提交+轮询 → true。 -// - 此外若「同步图片厂商异步化」总开关（sync_image_async_enabled，默认开启）打开，同步阻塞 -// 返回的图片厂商（OpenAI/DALL·E 等）也由 worker 池在异步端点处理，故所有有渠道的图片模型 -// 一并走异步端点 → true。开关关闭时，同步厂商回落同步端点 → false。 +// /v1/images/generations 必失败 → SyncSupported=false，只能走异步端点。 +// - 否则为同步阻塞返回的图片厂商（OpenAI/DALL·E 等）：同步端点始终可用 → SyncSupported=true； +// 此外若「同步图片厂商异步化」总开关（sync_image_async_enabled，默认开启）打开，还可由 worker +// 池在异步端点处理 → AsyncSupported=true；开关关闭时仅同步端点可用。 // -// 一次批量查询（非 N+1）；仅对图片分类模型调用。modelIDs 为空时返回空 map。 -func GetAsyncImageModelSet(ctx context.Context, modelIDs []string) (map[string]bool, error) { - result := make(map[string]bool) +// 一次批量查询（非 N+1）；仅对图片分类模型调用。无在役渠道的模型不出现在结果里（两模式皆不可用）。 +// modelIDs 为空时返回空 map。 +func GetImageModelModes(ctx context.Context, modelIDs []string) (map[string]ImageModelModes, error) { + result := make(map[string]ImageModelModes) if len(modelIDs) == 0 { return result, nil } @@ -190,12 +201,17 @@ func GetAsyncImageModelSet(ctx context.Context, modelIDs []string) (map[string]b } } // 「同步图片厂商异步化」总开关（默认开启）：开启时，同步阻塞返回的图片厂商（OpenAI/DALL·E - // 等）也能走 /v1/images/generations/async 由 worker 池异步处理，因此任何有在役渠道的图片 - // 模型都应走异步端点（在线体验用提交+轮询）。关闭时，仅「全部在役渠道均为异步图片 provider」 - // 的模型才异步（此时同步端点必失败）；同步厂商回落同步端点。 + // 等）也能走 /v1/images/generations/async 由 worker 池异步处理。SyncSupported 只看模型是否 + // 「仅异步」厂商，与开关无关——因此翻转开关不影响同步端点可用性。无在役渠道的模型不入结果。 syncImageEnabled := Config().GetBool(ctx, "sync_image_async_enabled") for name, st := range agg { - result[name] = st.hasChannel && (st.allAsync || syncImageEnabled) + if !st.hasChannel { + continue + } + result[name] = ImageModelModes{ + AsyncSupported: st.allAsync || syncImageEnabled, + SyncSupported: !st.allAsync, + } } return result, nil } diff --git a/internal/logic/open/open_api.go b/internal/logic/open/open_api.go index 61ddf9e..7c5887b 100644 --- a/internal/logic/open/open_api.go +++ b/internal/logic/open/open_api.go @@ -337,7 +337,7 @@ func (s *sOpen) OpenMemberModelsUpdate(ctx context.Context, req *v1.OpenMemberMo } err := g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - _, err := tx.Model("tnt_member_model_scopes").Ctx(ctx). + _, err := dao.TntMemberModelScopes.Ctx(ctx). Where("tenant_id", tenantID). Where("user_id", req.Id). Delete() @@ -350,7 +350,7 @@ func (s *sOpen) OpenMemberModelsUpdate(ctx context.Context, req *v1.OpenMemberMo if modelID == 0 { continue } - _, err = tx.Model("tnt_member_model_scopes").Ctx(ctx).Data(do.TntMemberModelScopes{ + _, err = dao.TntMemberModelScopes.Ctx(ctx).Data(do.TntMemberModelScopes{ TenantId: tenantID, UserId: req.Id, ModelId: modelID, @@ -360,7 +360,7 @@ func (s *sOpen) OpenMemberModelsUpdate(ctx context.Context, req *v1.OpenMemberMo } } } else { - _, err = tx.Model("tnt_member_model_scopes").Ctx(ctx).Data(do.TntMemberModelScopes{ + _, err = dao.TntMemberModelScopes.Ctx(ctx).Data(do.TntMemberModelScopes{ TenantId: tenantID, UserId: req.Id, ModelId: -1, diff --git a/internal/logic/payment/callback.go b/internal/logic/payment/callback.go index 6f46d0e..87395ce 100644 --- a/internal/logic/payment/callback.go +++ b/internal/logic/payment/callback.go @@ -119,8 +119,13 @@ func ProcessCallback(ctx context.Context, r *http.Request, channelType string) e } if result.Success { - // 7. 更新订单为已支付 - _, err = dao.OrdOrders.Ctx(ctx). + // 7. 原子领取订单：pending → paid。 + // 该条件更新的 RowsAffected 是【跨实例】幂等闸门：多实例并发回调时，两个回调都会 + // 通过上面第 5 步的 pending 检查，但 "WHERE status='pending'" 的原子更新只有一个能命中 + // （另一个在行锁释放后重读到 status 已是 paid → 0 行）。据此仅让真正把订单从 pending + // 翻成 paid 的那次回调去履约，杜绝重复入账/重复发套餐。进程内 orderLocks 仅单实例有效， + // 不能作为并发保护依赖。 + res, err := dao.OrdOrders.Ctx(ctx). Where("id", order.ID). Where("status", "pending"). Data(do.OrdOrders{ @@ -131,8 +136,17 @@ func ProcessCallback(ctx context.Context, r *http.Request, channelType string) e if err != nil { return gerror.Wrapf(err, "更新订单状态失败") } + affected, err := res.RowsAffected() + if err != nil { + return gerror.Wrapf(err, "确认订单状态更新结果失败") + } + if affected == 0 { + // 另一并发回调已抢先领取并负责履约，本次幂等返回，绝不重复履约 + g.Log().Infof(ctx, "[Payment] order=%s already claimed by a concurrent callback, skip duplicate fulfill", result.OrderNo) + return nil + } - // 8. 履约 + // 8. 履约（仅领取成功者执行） return FulfillOrder(ctx, order.ID) } diff --git a/internal/logic/payment/fulfill.go b/internal/logic/payment/fulfill.go index f160ceb..3b16406 100644 --- a/internal/logic/payment/fulfill.go +++ b/internal/logic/payment/fulfill.go @@ -3,12 +3,14 @@ package payment import ( "context" "fmt" + do "github.com/qianfree/team-api/internal/model/do" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" + "github.com/qianfree/team-api/internal/dao" "github.com/qianfree/team-api/internal/logic/billing" ) @@ -23,8 +25,11 @@ func FulfillOrder(ctx context.Context, orderID int64) error { FinalAmount float64 `json:"final_amount"` Status string `json:"status"` } - err := tx.Model("ord_orders").Ctx(ctx). + // 无行锁时两个并发事务（如支付回调与管理后台手动履约）可同时读到 paid 各自履约 → 重复入账/重复发套餐。 + // 加锁后后到的事务阻塞，待前者提交（状态已改为 fulfilled）后再读到最新状态，据此跳过。 + err := dao.OrdOrders.Ctx(ctx). Where("id", orderID). + LockUpdate(). Scan(&order) if err != nil { return err @@ -32,6 +37,10 @@ func FulfillOrder(ctx context.Context, orderID int64) error { if order == nil { return gerror.Newf("order %d not found", orderID) } + // 已履约：幂等空操作（并发后到者 / 回调重放 / 管理后台重复点击都会走到这里） + if order.Status == "fulfilled" { + return nil + } if order.Status != "paid" { return gerror.New("order status must be paid to fulfill") } @@ -42,13 +51,13 @@ func FulfillOrder(ctx context.Context, orderID int64) error { if order.OrderType == "renew" { months = 1 } - if err = subscribePlanTx(ctx, tx, order.TenantID, order.PlanID, months, true); err != nil { + if err = subscribePlanTx(ctx, order.TenantID, order.PlanID, months, true); err != nil { return gerror.Wrapf(err, "subscribe plan failed") } case "recharge": usdAmount := billing.ConvertCNYToUSD(ctx, order.FinalAmount) - if err = creditWalletTx(ctx, tx, order.TenantID, usdAmount, fmt.Sprintf("Recharge: order #%d (CNY %.2f → USD %.6f)", orderID, order.FinalAmount, usdAmount)); err != nil { + if err = creditWalletTx(ctx, order.TenantID, usdAmount, fmt.Sprintf("Recharge: order #%d (CNY %.2f → USD %.6f)", orderID, order.FinalAmount, usdAmount)); err != nil { return gerror.Wrapf(err, "credit wallet failed") } _ = billing.CheckAndUpgradeLevel(ctx, order.TenantID) @@ -59,24 +68,36 @@ func FulfillOrder(ctx context.Context, orderID int64) error { return gerror.Newf("unsupported order type for fulfillment: %s", order.OrderType) } - _, err = tx.Ctx(ctx).Model("ord_orders"). + // 持有行锁时理论上不会出现 0 行；一旦出现说明状态被并发改动，回滚整个履约事务以防重复入账。 + res, err := dao.OrdOrders.Ctx(ctx). Where("id", orderID). + Where("status", "paid"). Data(do.OrdOrders{ Status: "fulfilled", FulfilledAt: gtime.Now(), }).Update() - return err + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return gerror.Newf("order %d fulfill aborted: status changed concurrently", orderID) + } + return nil }) } -// subscribePlanTx 在事务内订阅套餐 -func subscribePlanTx(ctx context.Context, tx gdb.TX, tenantID int64, planID int64, months int, autoRenew bool) error { +// subscribePlanTx 在事务内订阅套餐（依赖调用方传入携带事务的 ctx，内部统一用 dao.Xxx.Ctx(ctx) 传播） +func subscribePlanTx(ctx context.Context, tenantID int64, planID int64, months int, autoRenew bool) error { var plan *struct { MonthlyPrice float64 `json:"monthly_price"` YearlyPrice float64 `json:"yearly_price"` MonthlyQuotaTokens int64 `json:"monthly_quota_tokens"` } - err := tx.Model("pln_plans").Ctx(ctx). + err := dao.PlnPlans.Ctx(ctx). Where("id", planID). Where("status", "active"). Scan(&plan) @@ -95,7 +116,7 @@ func subscribePlanTx(ctx context.Context, tx gdb.TX, tenantID int64, planID int6 endAt := now.AddDate(0, months, 0) // 先取消当前活跃订阅 - _, err = tx.Model("pln_tenant_plans").Ctx(ctx). + _, err = dao.PlnTenantPlans.Ctx(ctx). Where("tenant_id", tenantID). Where("status", "active"). Data(do.PlnTenantPlans{ @@ -106,7 +127,7 @@ func subscribePlanTx(ctx context.Context, tx gdb.TX, tenantID int64, planID int6 return gerror.Wrapf(err, "cancel old plan for tenant %d", tenantID) } - _, err = tx.Model("pln_tenant_plans").Ctx(ctx).Insert(do.PlnTenantPlans{ + _, err = dao.PlnTenantPlans.Ctx(ctx).Insert(do.PlnTenantPlans{ TenantId: tenantID, PlanId: planID, Status: "active", @@ -123,16 +144,16 @@ func subscribePlanTx(ctx context.Context, tx gdb.TX, tenantID int64, planID int6 // SubscribePlan 订阅套餐（公开函数，被自动续费、兑换码等外部调用） func SubscribePlan(ctx context.Context, tenantID int64, planID int64, months int, autoRenew bool) error { return g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - return subscribePlanTx(ctx, tx, tenantID, planID, months, autoRenew) + return subscribePlanTx(ctx, tenantID, planID, months, autoRenew) }) } -// creditWalletTx 在事务内钱包入账 -func creditWalletTx(ctx context.Context, tx gdb.TX, tenantID int64, amount float64, description string) error { +// creditWalletTx 在事务内钱包入账（依赖调用方传入携带事务的 ctx） +func creditWalletTx(ctx context.Context, tenantID int64, amount float64, description string) error { var w *struct { ID int64 `json:"id"` } - err := tx.Model("bil_wallets").Ctx(ctx). + err := dao.BilWallets.Ctx(ctx). Where("tenant_id", tenantID). Fields("id"). Scan(&w) @@ -143,7 +164,7 @@ func creditWalletTx(ctx context.Context, tx gdb.TX, tenantID int64, amount float return gerror.Newf("wallet not found for tenant %d", tenantID) } - _, err = tx.Ctx(ctx).Exec( + _, err = g.DB().Ctx(ctx).Exec(ctx, "UPDATE bil_wallets SET balance = balance + ?, cumulative_recharge = cumulative_recharge + ?, updated_at = NOW() WHERE id = ?", amount, amount, w.ID) if err != nil { @@ -154,7 +175,7 @@ func creditWalletTx(ctx context.Context, tx gdb.TX, tenantID int64, amount float Balance float64 `json:"balance"` FrozenBalance float64 `json:"frozen_balance"` } - err = tx.Model("bil_wallets").Ctx(ctx). + err = dao.BilWallets.Ctx(ctx). Where("id", w.ID). Fields("balance, frozen_balance"). Scan(&balance) @@ -165,7 +186,7 @@ func creditWalletTx(ctx context.Context, tx gdb.TX, tenantID int64, amount float return gerror.New("wallet not found after update") } - _, err = tx.Model("bil_transactions").Ctx(ctx).Insert(do.BilTransactions{ + _, err = dao.BilTransactions.Ctx(ctx).Insert(do.BilTransactions{ TenantId: tenantID, WalletId: w.ID, Type: "recharge", diff --git a/internal/logic/task/async_provider.go b/internal/logic/task/async_provider.go index d41428a..6fdea5a 100644 --- a/internal/logic/task/async_provider.go +++ b/internal/logic/task/async_provider.go @@ -217,6 +217,7 @@ func (p *AsyncProvider) GetTaskByPublicIDAndUser(ctx context.Context, publicTask BillingSettled bool `json:"billing_settled"` ResultURL string `json:"result_url"` Data json.RawMessage `json:"data"` + PrivateData json.RawMessage `json:"private_data"` SubmitTime *time.Time `json:"submit_time"` StartTime *time.Time `json:"start_time"` FinishTime *time.Time `json:"finish_time"` @@ -254,6 +255,7 @@ func (p *AsyncProvider) GetTaskByPublicIDAndUser(ctx context.Context, publicTask BillingSettled: row.BillingSettled, ResultURL: row.ResultURL, Data: row.Data, + PrivateData: row.PrivateData, SubmitTime: row.SubmitTime, StartTime: row.StartTime, FinishTime: row.FinishTime, diff --git a/internal/logic/task/sync_image_worker.go b/internal/logic/task/sync_image_worker.go index 7066ecd..0643bca 100644 --- a/internal/logic/task/sync_image_worker.go +++ b/internal/logic/task/sync_image_worker.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "math/rand" @@ -12,6 +13,7 @@ import ( "sync" "sync/atomic" "time" + "unicode/utf8" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gctx" @@ -68,10 +70,34 @@ var ( syncImageFailed atomic.Int64 // 累计 worker 处理失败 syncImageFileSvc *lcommon.FileService + syncImageFileSvcMu sync.Mutex // 保护 syncImageFileSvc 的惰性重建 syncImageBilling common.TaskBillingProvider syncImageRelayProv common.DataProvider ) +// acquireSyncImageFileSvc 返回共享的对象存储 FileService，惰性重建。 +// +// 对象存储配置存于数据库（sys_options），由管理后台**运行时**配置。若进程启动时存储尚未 +// 配置，StartSyncImageWorkers 首次构造会失败、syncImageFileSvc 为 nil。此处按需重建：一旦 +// 存储在管理后台配置好，下一个 b64_json 图片任务即可成功取到 FileService，**无需重启进程**。 +// 这与 admin / data_governance 每次按数据库配置重建 FileService 的行为对齐——只有本 worker +// 池此前把启动时的构造结果缓存成唯一真相源，才导致「配置后仍报未配置」。 +// +// 声明为包级变量，便于单测在无数据库环境下覆盖（避免触达 sys_options 查询）。 +var acquireSyncImageFileSvc = func(ctx context.Context) (*lcommon.FileService, error) { + syncImageFileSvcMu.Lock() + defer syncImageFileSvcMu.Unlock() + if syncImageFileSvc != nil { + return syncImageFileSvc, nil + } + fs, err := lcommon.NewFileServiceFromConfig(ctx) + if err != nil { + return nil, err + } + syncImageFileSvc = fs + return fs, nil +} + // SyncImageJob 同步图片任务的内存载荷（请求体只放内存，不落库，无需崩溃重放）。 type SyncImageJob struct { TaskID int64 // tsk_model_tasks 主键 @@ -99,12 +125,14 @@ func StartSyncImageWorkers(ctx context.Context) { syncImageBilling = billing.NewTaskBillingProvider() syncImageRelayProv = relay.NewDataProvider() - // 对象存储在启动时构造一次共享（无状态，可安全共享）。未配置时置 nil， - // b64_json 任务在无存储时会走 FAILURE+退款（见 buildImageResult）。 + // 对象存储配置存于数据库，可能在进程启动后才由管理后台配置。这里尝试预热一次； + // 未配置时保持 nil，后续任务运行时由 acquireSyncImageFileSvc 惰性重建（无需重启）。 if fs, err := lcommon.NewFileServiceFromConfig(ctx); err != nil { - g.Log().Warningf(ctx, "sync_image: object storage not configured (%v); b64_json image tasks will fail until storage is set", err) + g.Log().Warningf(ctx, "sync_image: object storage not yet configured (%v); will lazily rebuild when a b64_json image task runs", err) } else { + syncImageFileSvcMu.Lock() syncImageFileSvc = fs + syncImageFileSvcMu.Unlock() } for i := 0; i < syncImageWorkerCount; i++ { @@ -497,13 +525,24 @@ func runImagePipeline(ctx context.Context, job *SyncImageJob, sel *common.Channe func settleSyncImageSuccess(ctx context.Context, job *SyncImageJob, sel *common.ChannelSelection, memW *memResponseWriter) { resultURL, normalized, err := buildImageResult(ctx, job, memW.Bytes()) if err != nil { - failSyncImageJob(ctx, job, sel, fmt.Sprintf("build result: %v", err)) + failSyncImageJob(ctx, job, sel, imageFailReason(err)) return } + // 回填上游返回的 token 用量：gpt-image-1/2 等在响应体 usage 里带 input/output/total_tokens， + // DALL·E 等则没有。据此重算实际费用（token 计费模型多退少补），并写入计费/用量记录， + // 避免「同步转异步后 token 恒为 0、费用只等于预扣」的问题。 + promptTokens, completionTokens, totalTokens := extractImageUsage(memW.Bytes()) + now := time.Now() - // per_request 图片：实际费用 == 预扣（RecalculateByTokens 对 per_request 返回 0，无 token 重算）。 + // 默认实际费用 == 预扣（按次计费图片，如 DALL·E，无 token 用量）。 actualCost := job.PreDeductAmount + // 上游返回了 token 用量 → 按真实 token 重算；仅 token 计费模型返回 >0，按次模型返回 0 保持预扣。 + if totalTokens > 0 { + if tokenCost, rerr := syncImageBilling.RecalculateByTokens(ctx, job.TenantID, job.Model, totalTokens, job.Ratios); rerr == nil && tokenCost > 0 { + actualCost = tokenCost + } + } // 1. 赢得终态（billing_settled=false 先落） if err := DefaultAsyncProvider.UpdateTaskCAS(ctx, &common.AsyncTask{ @@ -519,9 +558,9 @@ func settleSyncImageSuccess(ctx context.Context, job *SyncImageJob, sel *common. return } - // 2. 结算钱包 + // 2. 结算钱包（传真实 token 用量：驱动 bil_records.output_tokens 与计费快照的 token 明细） settleResult, serr := syncImageBilling.SettleTaskSuccess(ctx, job.TenantID, job.UserID, job.ApiKeyID, sel.ChannelID, - job.Model, job.RequestID, actualCost, job.PreDeductAmount, 0, 0, job.Ratios, job.PublicTaskID) + job.Model, job.RequestID, actualCost, job.PreDeductAmount, totalTokens, completionTokens, job.Ratios, job.PublicTaskID) if serr != nil { // 保留 billing_settled=false，由未结算兜底网重放结算 g.Log().Warningf(ctx, "sync_image: task %s settle success failed (unsettled net will retry): %v", job.PublicTaskID, serr) @@ -548,6 +587,9 @@ func settleSyncImageSuccess(ctx context.Context, job *SyncImageJob, sel *common. chBasic := &common.ChannelBasicInfo{ID: sel.ChannelID, Type: sel.ChannelType, Name: sel.ChannelName} usageTask := buildUsageTask(job, sel, "SUCCESS", actualCost, now) + usageTask.PromptTokens = promptTokens + usageTask.CompletionTokens = completionTokens + usageTask.TotalTokens = totalTokens recordTaskUsage(usageTask, chBasic, true, "", settleResult) // 闭环审计：把提交阶段写入的 SUBMITTED 审计记录更新为终态（与 pollSingleTask 一致）， // 否则请求审计日志里的任务状态会一直停留在「已提交」。 @@ -600,8 +642,12 @@ func failSyncImageJob(ctx context.Context, job *SyncImageJob, sel *common.Channe recordTaskCompletionAudit(usageTask, "FAILURE", reason, nil) } -// buildImageResult 解析上游图片响应，返回结果 URL 与归一化后的响应体（供落库 Data）。 +// buildImageResult 解析上游图片响应，re-host **全部**图片，返回首图 URL（向后兼容单值 +// ResultURL）与归一化后的多图响应体（供落库 Data；fetch 端点据此吐出 data 数组）。 // b64_json 无条件 re-host；url 按配置开关（默认透传，开启则下载 re-host）。 +// +// 采用 all-or-nothing：任一张解码 / re-host 失败即整体返回错误，由调用方走 FAILURE + 全额 +// 退款（与单图行为一致）。计费按次扁平，多图不额外计费，与同步端点口径保持一致。 func buildImageResult(ctx context.Context, job *SyncImageJob, body []byte) (resultURL string, normalized []byte, err error) { var imgResp dto.ImageResponse if e := json.Unmarshal(body, &imgResp); e != nil { @@ -610,46 +656,84 @@ func buildImageResult(ctx context.Context, job *SyncImageJob, body []byte) (resu if len(imgResp.Data) == 0 { return "", nil, fmt.Errorf("empty image data") } - if len(imgResp.Data) > 1 { - g.Log().Warningf(ctx, "sync_image: task %s returned %d images; only the first is surfaced via ResultURL", job.PublicTaskID, len(imgResp.Data)) + + outData := make([]dto.ImageData, 0, len(imgResp.Data)) + for i, img := range imgResp.Data { + var url string + switch { + case img.B64JSON != "": + data, e := base64.StdEncoding.DecodeString(img.B64JSON) + if e != nil { + return "", nil, fmt.Errorf("decode b64_json[%d]: %w", i, e) + } + if url, e = rehostImage(ctx, job, data, "image/png", ".png", i); e != nil { + return "", nil, fmt.Errorf("rehost b64_json[%d]: %w", i, e) + } + case img.URL != "": + if rehostURLEnabled(ctx) { + if url, err = rehostFromURL(ctx, job, img.URL, i); err != nil { + return "", nil, fmt.Errorf("rehost url[%d]: %w", i, err) + } + } else { + url = img.URL + } + default: + return "", nil, fmt.Errorf("no url or b64_json in image response at index %d", i) + } + outData = append(outData, dto.ImageData{URL: url, RevisedPrompt: img.RevisedPrompt}) } - first := imgResp.Data[0] - switch { - case first.B64JSON != "": - data, e := base64.StdEncoding.DecodeString(first.B64JSON) - if e != nil { - return "", nil, fmt.Errorf("decode b64_json: %w", e) - } - resultURL, err = rehostImage(ctx, job, data, "image/png", ".png") - case first.URL != "": - if shouldRehostURL(ctx) { - resultURL, err = rehostFromURL(ctx, job, first.URL) - } else { - resultURL = first.URL - } - default: - return "", nil, fmt.Errorf("no url or b64_json in image response") - } - if err != nil { - return "", nil, err - } - - // 归一化落库：只存单张结果 URL，避免把大体积 b64 原文写进 data 列。 + resultURL = outData[0].URL + // 归一化落库：存全部结果 URL（不含大体积 b64 原文），供 fetch 吐 data 数组。 normalized, _ = json.Marshal(dto.ImageResponse{ Created: imgResp.Created, - Data: []dto.ImageData{{URL: resultURL, RevisedPrompt: first.RevisedPrompt}}, + Data: outData, }) return resultURL, normalized, nil } -func rehostImage(ctx context.Context, job *SyncImageJob, data []byte, contentType, ext string) (string, error) { - if syncImageFileSvc == nil { - return "", fmt.Errorf("object storage not configured, cannot re-host image") +// extractImageUsage 从上游图片响应体解析 token 用量。gpt-image-1/2 在 usage 里返回 +// input_tokens / output_tokens / total_tokens；DALL·E 等无 usage 字段，返回全 0。 +// total 缺失时用 input+output 兜底。映射：prompt=input、completion=output。 +func extractImageUsage(body []byte) (prompt, completion, total int) { + var resp struct { + Usage *dto.ImageUsage `json:"usage"` } - rec, err := syncImageFileSvc.Upload(ctx, &lcommon.FileUpload{ + if err := json.Unmarshal(body, &resp); err != nil || resp.Usage == nil { + return 0, 0, 0 + } + prompt = resp.Usage.InputTokens + completion = resp.Usage.OutputTokens + total = resp.Usage.TotalTokens + if total == 0 { + total = prompt + completion + } + return prompt, completion, total +} + +// rehostImage 把单张图片字节上传对象存储并返回下载 URL。index 用于区分同一任务的多张图片， +// 避免多图共用同一对象键相互覆盖。 +// imageFailReason 把「结果处理」阶段的内部错误映射为**面向用户**的失败原因（写入 task +// FailReason → fetch 端点的 error 字段 → 在线体验/客户端可见）。 +// +// 对象存储未配置是最常见、且用户可操作的错误：图片其实已在上游生成成功，只是网关无处保存 +// （b64_json 必须 re-host 到对象存储）。对这种情况给出中文友好提示，引导去系统设置配置 +// 对象存储；其余错误（上传/下载失败、上游响应异常等）保留技术细节，便于运维排查。 +func imageFailReason(err error) string { + if errors.Is(err, lcommon.ErrStorageNotConfigured) { + return "图片已生成，但平台尚未配置对象存储（OSS/S3/COS），无法保存图片。请联系管理员在系统设置中配置对象存储后重试。" + } + return "build result: " + err.Error() +} + +func rehostImage(ctx context.Context, job *SyncImageJob, data []byte, contentType, ext string, index int) (string, error) { + fs, err := acquireSyncImageFileSvc(ctx) + if err != nil { + return "", fmt.Errorf("object storage not configured, cannot re-host image: %w", err) + } + rec, err := fs.Upload(ctx, &lcommon.FileUpload{ Reader: bytes.NewReader(data), - Filename: job.PublicTaskID + ext, + Filename: fmt.Sprintf("%s_%d%s", job.PublicTaskID, index, ext), ContentType: contentType, Size: int64(len(data)), TenantID: job.TenantID, @@ -658,18 +742,26 @@ func rehostImage(ctx context.Context, job *SyncImageJob, data []byte, contentTyp if err != nil { return "", err } - return syncImageFileSvc.GetDownloadURL(ctx, rec.ID) + return fs.GetDownloadURL(ctx, rec.ID) } -func rehostFromURL(ctx context.Context, job *SyncImageJob, url string) (string, error) { - if syncImageFileSvc == nil { - return "", fmt.Errorf("object storage not configured, cannot re-host image") +// syncImageDownloadClient 用于从上游返回的图片 URL 下载**已生成好的**图片做 re-host。 +// 注意：这不是图片生成本身——生成走 adaptor 的 nonStreamClient（300s 超时），此处仅下载 +// 成品图片文件。显式设置超时，避免 http.DefaultClient 无超时时慢速/挂起的图片服务器长期 +// 占用 worker（io.LimitReader 只限体积不限时间）。120s 对 ≤20MB 的成品图很充裕，且远低于 +// 生成超时——放宽是为了避免「生成已成功却因下载慢被判失败+退款」的资损。 +var syncImageDownloadClient = &http.Client{Timeout: 120 * time.Second} + +func rehostFromURL(ctx context.Context, job *SyncImageJob, url string, index int) (string, error) { + // 先确认存储可用，避免存储未配置时白白下载一遍成品图。 + if _, err := acquireSyncImageFileSvc(ctx); err != nil { + return "", fmt.Errorf("object storage not configured, cannot re-host image: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return "", err } - resp, err := http.DefaultClient.Do(req) + resp, err := syncImageDownloadClient.Do(req) if err != nil { return "", err } @@ -685,10 +777,12 @@ func rehostFromURL(ctx context.Context, job *SyncImageJob, url string) (string, if contentType == "" { contentType = "image/png" } - return rehostImage(ctx, job, data, contentType, extFromContentType(contentType)) + return rehostImage(ctx, job, data, contentType, extFromContentType(contentType), index) } -func shouldRehostURL(ctx context.Context) bool { +// rehostURLEnabled 判断上游返回 url 时是否下载 re-host（默认透传）。 +// 声明为包级变量而非普通函数，便于单测覆盖多图 url 透传路径（无需初始化配置中心）。 +var rehostURLEnabled = func(ctx context.Context) bool { return lcommon.Config().GetBool(ctx, "sync_image_rehost_url") } @@ -743,11 +837,17 @@ func syncImageJitter() { time.Sleep(time.Duration(50+rand.Intn(150)) * time.Millisecond) } +// truncateStr 将字符串截断到不超过 max 字节，且按 rune 边界截断——避免把多字节 UTF-8 +// 字符（如中文）从中间切断，产生非法字节写入 DB 文本列。回退到 max 之内最后一个完整 rune。 func truncateStr(s string, max int) string { if len(s) <= max { return s } - return s[:max] + truncated := s[:max] + for len(truncated) > 0 && !utf8.ValidString(truncated) { + truncated = truncated[:len(truncated)-1] + } + return truncated } func extFromContentType(ct string) string { diff --git a/internal/logic/task/sync_image_worker_test.go b/internal/logic/task/sync_image_worker_test.go index 5b845f0..e1dc6eb 100644 --- a/internal/logic/task/sync_image_worker_test.go +++ b/internal/logic/task/sync_image_worker_test.go @@ -4,8 +4,15 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" + "fmt" "net/http" + "strings" "testing" + "unicode/utf8" + + lcommon "github.com/qianfree/team-api/internal/logic/common" + "github.com/qianfree/team-api/relay/dto" ) func TestMemResponseWriter(t *testing.T) { @@ -91,9 +98,49 @@ func TestBuildImageResult_EmptyData(t *testing.T) { } } +// TestExtractImageUsage_GptImage gpt-image 系列返回 usage，应被解析出 token 用量。 +func TestExtractImageUsage_GptImage(t *testing.T) { + body := []byte(`{"created":1,"data":[{"b64_json":"x"}],"usage":{"total_tokens":4160,"input_tokens":160,"output_tokens":4000}}`) + prompt, completion, total := extractImageUsage(body) + if prompt != 160 || completion != 4000 || total != 4160 { + t.Fatalf("got prompt=%d completion=%d total=%d, want 160/4000/4160", prompt, completion, total) + } +} + +// TestExtractImageUsage_TotalFallback total 缺失时用 input+output 兜底。 +func TestExtractImageUsage_TotalFallback(t *testing.T) { + body := []byte(`{"data":[{"b64_json":"x"}],"usage":{"input_tokens":100,"output_tokens":250}}`) + _, _, total := extractImageUsage(body) + if total != 350 { + t.Fatalf("total = %d, want 350 (input+output fallback)", total) + } +} + +// TestExtractImageUsage_NoUsage DALL·E 等无 usage 字段，返回全 0（保持按次计费预扣）。 +func TestExtractImageUsage_NoUsage(t *testing.T) { + body := []byte(`{"created":1,"data":[{"url":"https://example.com/a.png"}]}`) + prompt, completion, total := extractImageUsage(body) + if prompt != 0 || completion != 0 || total != 0 { + t.Fatalf("got prompt=%d completion=%d total=%d, want all 0", prompt, completion, total) + } +} + +// TestExtractImageUsage_Malformed 响应体非法 JSON 时安全返回全 0，不 panic。 +func TestExtractImageUsage_Malformed(t *testing.T) { + if p, c, tot := extractImageUsage([]byte("not json")); p != 0 || c != 0 || tot != 0 { + t.Fatalf("malformed body should yield 0/0/0, got %d/%d/%d", p, c, tot) + } +} + func TestBuildImageResult_B64WithoutStorageErrors(t *testing.T) { - // syncImageFileSvc 默认 nil（未启动 worker），b64_json 无法 re-host → 报错。 - syncImageFileSvc = nil + // 对象存储不可用时，b64_json 无法 re-host → 报错。覆盖 accessor 直接返回错误， + // 避免在无数据库的单测环境触达 sys_options 查询。 + orig := acquireSyncImageFileSvc + acquireSyncImageFileSvc = func(context.Context) (*lcommon.FileService, error) { + return nil, lcommon.ErrStorageNotConfigured + } + defer func() { acquireSyncImageFileSvc = orig }() + job := &SyncImageJob{PublicTaskID: "task_x", TenantID: 1, UserID: 2} raw, _ := json.Marshal(map[string]any{ "created": 1, @@ -104,6 +151,103 @@ func TestBuildImageResult_B64WithoutStorageErrors(t *testing.T) { } } +// TestBuildImageResult_MultiURLPassthrough 多图 url 透传：全部图片进 normalized，首图作 ResultURL。 +func TestBuildImageResult_MultiURLPassthrough(t *testing.T) { + orig := rehostURLEnabled + rehostURLEnabled = func(context.Context) bool { return false } // 透传，不下载 re-host + defer func() { rehostURLEnabled = orig }() + + job := &SyncImageJob{PublicTaskID: "task_multi"} + raw, _ := json.Marshal(map[string]any{ + "created": 123, + "data": []any{ + map[string]any{"url": "https://cdn.example.com/a.png", "revised_prompt": "a"}, + map[string]any{"url": "https://cdn.example.com/b.png"}, + map[string]any{"url": "https://cdn.example.com/c.png"}, + }, + }) + resultURL, normalized, err := buildImageResult(context.Background(), job, raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resultURL != "https://cdn.example.com/a.png" { + t.Fatalf("resultURL = %q, want first image url", resultURL) + } + var out dto.ImageResponse + if e := json.Unmarshal(normalized, &out); e != nil { + t.Fatalf("normalized not a valid ImageResponse: %v", e) + } + if len(out.Data) != 3 { + t.Fatalf("normalized data len = %d, want 3", len(out.Data)) + } + if out.Data[0].URL != "https://cdn.example.com/a.png" || out.Data[2].URL != "https://cdn.example.com/c.png" { + t.Fatalf("normalized urls not preserved in order: %+v", out.Data) + } + if out.Data[0].RevisedPrompt != "a" { + t.Fatalf("revised_prompt not preserved on first image") + } +} + +// TestBuildImageResult_AllOrNothingOnBadEntry 多图中任一条目无 url/b64 → 整体失败（all-or-nothing）。 +func TestBuildImageResult_AllOrNothingOnBadEntry(t *testing.T) { + orig := rehostURLEnabled + rehostURLEnabled = func(context.Context) bool { return false } + defer func() { rehostURLEnabled = orig }() + + job := &SyncImageJob{PublicTaskID: "task_bad"} + raw, _ := json.Marshal(map[string]any{ + "created": 1, + "data": []any{ + map[string]any{"url": "https://cdn.example.com/a.png"}, + map[string]any{}, // 无 url 无 b64 → 整单失败 + }, + }) + if _, _, err := buildImageResult(context.Background(), job, raw); err == nil { + t.Fatal("entry without url/b64 should fail the whole task (all-or-nothing)") + } +} + +// TestBuildImageResult_MultiB64WithoutStorageErrors 多图 b64 无对象存储 → 整体失败。 +func TestBuildImageResult_MultiB64WithoutStorageErrors(t *testing.T) { + orig := acquireSyncImageFileSvc + acquireSyncImageFileSvc = func(context.Context) (*lcommon.FileService, error) { + return nil, lcommon.ErrStorageNotConfigured + } + defer func() { acquireSyncImageFileSvc = orig }() + + job := &SyncImageJob{PublicTaskID: "task_b64multi", TenantID: 1, UserID: 2} + raw, _ := json.Marshal(map[string]any{ + "created": 1, + "data": []any{ + map[string]any{"b64_json": base64.StdEncoding.EncodeToString([]byte("PNG1"))}, + map[string]any{"b64_json": base64.StdEncoding.EncodeToString([]byte("PNG2"))}, + }, + }) + if _, _, err := buildImageResult(context.Background(), job, raw); err == nil { + t.Fatal("multi b64 without storage should error") + } +} + +// TestImageFailReason_StorageNotConfiguredFriendly 存储未配置的错误（穿过多层 %w 包裹） +// 应映射为中文友好提示；其它错误保留技术细节。 +func TestImageFailReason_StorageNotConfiguredFriendly(t *testing.T) { + // 模拟 buildImageResult → rehostImage 的多层包裹，验证 errors.Is 仍能识别哨兵。 + wrapped := fmt.Errorf("rehost b64_json[0]: %w", + fmt.Errorf("object storage not configured, cannot re-host image: %w", lcommon.ErrStorageNotConfigured)) + msg := imageFailReason(wrapped) + if !strings.Contains(msg, "对象存储") { + t.Fatalf("storage-not-configured should map to friendly message, got %q", msg) + } + + other := imageFailReason(errors.New("upstream returned garbage")) + if strings.Contains(other, "对象存储") { + t.Fatalf("non-storage error must not map to storage message, got %q", other) + } + if !strings.Contains(other, "upstream returned garbage") { + t.Fatalf("non-storage error should retain technical detail, got %q", other) + } +} + func TestExtFromContentType(t *testing.T) { cases := map[string]string{ "image/png": ".png", @@ -126,4 +270,26 @@ func TestTruncateStr(t *testing.T) { if got := truncateStr("ab", 5); got != "ab" { t.Fatalf("truncate = %q, want ab", got) } + + // UTF-8 边界：不得把多字节字符（中文每字 3 字节）从中间切断。 + utf8Cases := []struct { + s string + max int + want string + }{ + {"中文", 3, "中"}, // 恰好一个完整字 + {"中文", 4, "中"}, // 第 4 字节落在「文」中间 → 回退到「中」 + {"中文", 5, "中"}, // 第 5 字节仍在「文」中间 → 回退到「中」 + {"中文", 6, "中文"}, // 恰好容纳两字 + {"中", 2, ""}, // 连一个字都放不下 → 空串（而非非法字节） + } + for _, c := range utf8Cases { + got := truncateStr(c.s, c.max) + if got != c.want { + t.Fatalf("truncateStr(%q, %d) = %q, want %q", c.s, c.max, got, c.want) + } + if !utf8.ValidString(got) { + t.Fatalf("truncateStr(%q, %d) = %q is not valid UTF-8", c.s, c.max, got) + } + } } diff --git a/internal/logic/tenant/api_key.go b/internal/logic/tenant/api_key.go index ba615a2..9cd5af4 100644 --- a/internal/logic/tenant/api_key.go +++ b/internal/logic/tenant/api_key.go @@ -523,7 +523,7 @@ func (s *sTenant) ApiKeyUpdateScopes(ctx context.Context, req *v1.TenantApiKeyUp // updateApiKeyModelScopes 在事务内更新模型范围（先删后插） func updateApiKeyModelScopes(ctx context.Context, keyID int64, modelNames []string) error { return g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - _, err := tx.Ctx(ctx).Model("api_key_model_scopes"). + _, err := dao.ApiKeyModelScopes.Ctx(ctx). Where("api_key_id", keyID). Delete() if err != nil { @@ -532,7 +532,7 @@ func updateApiKeyModelScopes(ctx context.Context, keyID int64, modelNames []stri for _, modelName := range modelNames { if modelName != "" { - if _, err := tx.Ctx(ctx).Model("api_key_model_scopes").Insert(do.ApiKeyModelScopes{ + if _, err := dao.ApiKeyModelScopes.Ctx(ctx).Insert(do.ApiKeyModelScopes{ ApiKeyId: keyID, ModelName: modelName, }); err != nil { diff --git a/internal/logic/tenant/auth.go b/internal/logic/tenant/auth.go index 76ad473..913c030 100644 --- a/internal/logic/tenant/auth.go +++ b/internal/logic/tenant/auth.go @@ -132,9 +132,9 @@ func (s *sTenant) Register(ctx context.Context, req *v1.TenantRegisterReq) (*v1. var tenantID, ownerUserID int64 - err = dao.TntTenants.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Create tenant - tenantResult, err := tx.Model("tnt_tenants").Ctx(ctx).Data(do.TntTenants{ + tenantResult, err := dao.TntTenants.Ctx(ctx).Data(do.TntTenants{ Name: tenantName, Code: tenantCode, // MaxMembers/MaxConcurrency: nil = 跟随等级配置 @@ -152,7 +152,7 @@ func (s *sTenant) Register(ctx context.Context, req *v1.TenantRegisterReq) (*v1. } // Create owner user - userResult, err := tx.Model("tnt_users").Ctx(ctx).Data(do.TntUsers{ + userResult, err := dao.TntUsers.Ctx(ctx).Data(do.TntUsers{ TenantId: tenantID, Username: username, Email: email, @@ -169,7 +169,7 @@ func (s *sTenant) Register(ctx context.Context, req *v1.TenantRegisterReq) (*v1. } // Update tenant owner - _, err = tx.Model("tnt_tenants").Ctx(ctx). + _, err = dao.TntTenants.Ctx(ctx). Where("id", tenantID). Data(do.TntTenants{ OwnerUserId: ownerUserID, @@ -180,7 +180,7 @@ func (s *sTenant) Register(ctx context.Context, req *v1.TenantRegisterReq) (*v1. } // Create wallet for tenant - _, err = tx.Model("bil_wallets").Ctx(ctx).Data(do.BilWallets{ + _, err = dao.BilWallets.Ctx(ctx).Data(do.BilWallets{ TenantId: tenantID, Balance: 0, FrozenBalance: 0, @@ -194,7 +194,7 @@ func (s *sTenant) Register(ctx context.Context, req *v1.TenantRegisterReq) (*v1. var defaultGroups []struct { Id int64 } - if err := tx.Model("mdl_model_groups").Ctx(ctx). + if err := dao.MdlModelGroups.Ctx(ctx). Where("is_default", true).Where("status", "active"). Fields("id").Scan(&defaultGroups); err == nil && len(defaultGroups) > 0 { insertData := make([]do.MdlTenantGroups, 0, len(defaultGroups)) @@ -204,7 +204,7 @@ func (s *sTenant) Register(ctx context.Context, req *v1.TenantRegisterReq) (*v1. GroupId: dg.Id, }) } - if _, err := tx.Model("mdl_tenant_groups").Ctx(ctx). + if _, err := dao.MdlTenantGroups.Ctx(ctx). Batch(len(insertData)).Insert(insertData); err != nil { g.Log().Warningf(ctx, "assign default model groups failed: %v", err) } diff --git a/internal/logic/tenant/member.go b/internal/logic/tenant/member.go index d7207f3..170a091 100644 --- a/internal/logic/tenant/member.go +++ b/internal/logic/tenant/member.go @@ -254,9 +254,9 @@ func (s *sTenant) JoinByInvite(ctx context.Context, req *v1.TenantMemberJoinReq) var userID int64 - err = dao.TntTenants.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Check member limit - memberCount, err := tx.Model("tnt_users").Ctx(ctx). + memberCount, err := dao.TntUsers.Ctx(ctx). Where("tenant_id", tenantID). Where("status", "active"). Count() @@ -275,7 +275,7 @@ func (s *sTenant) JoinByInvite(ctx context.Context, req *v1.TenantMemberJoinReq) } // Check username uniqueness within tenant - count, err := tx.Model("tnt_users").Ctx(ctx). + count, err := dao.TntUsers.Ctx(ctx). Where("tenant_id", tenantID). Where("username", username). Count() @@ -288,7 +288,7 @@ func (s *sTenant) JoinByInvite(ctx context.Context, req *v1.TenantMemberJoinReq) // Check email uniqueness within tenant (email is optional — skip when empty) if email != "" { - count, err = tx.Model("tnt_users").Ctx(ctx). + count, err = dao.TntUsers.Ctx(ctx). Where("tenant_id", tenantID). Where("email", email). Count() @@ -306,7 +306,7 @@ func (s *sTenant) JoinByInvite(ctx context.Context, req *v1.TenantMemberJoinReq) displayName = username } - userResult, err := tx.Model("tnt_users").Ctx(ctx).Data(do.TntUsers{ + userResult, err := dao.TntUsers.Ctx(ctx).Data(do.TntUsers{ TenantId: tenantID, Username: username, Email: nullableEmail(email), @@ -329,7 +329,7 @@ func (s *sTenant) JoinByInvite(ctx context.Context, req *v1.TenantMemberJoinReq) } // Increment invitation use count - _, err = tx.Model("tnt_invitations").Ctx(ctx). + _, err = dao.TntInvitations.Ctx(ctx). Where("id", invitation.ID). Data(do.TntInvitations{ UseCount: gdb.Raw("use_count + 1"), @@ -427,9 +427,9 @@ func (s *sTenant) CreateMember(ctx context.Context, req *v1.TenantMemberCreateRe var userID int64 - err = dao.TntTenants.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Check member limit - memberCount, err := tx.Model("tnt_users").Ctx(ctx). + memberCount, err := dao.TntUsers.Ctx(ctx). Where("tenant_id", tenantID). Where("status", "active"). Count() @@ -448,7 +448,7 @@ func (s *sTenant) CreateMember(ctx context.Context, req *v1.TenantMemberCreateRe } // Check username uniqueness within tenant - count, err := tx.Model("tnt_users").Ctx(ctx). + count, err := dao.TntUsers.Ctx(ctx). Where("tenant_id", tenantID). Where("username", username). Count() @@ -461,7 +461,7 @@ func (s *sTenant) CreateMember(ctx context.Context, req *v1.TenantMemberCreateRe // Check email uniqueness within tenant (skip if empty) if email != "" { - count, err = tx.Model("tnt_users").Ctx(ctx). + count, err = dao.TntUsers.Ctx(ctx). Where("tenant_id", tenantID). Where("email", email). Count() @@ -474,7 +474,7 @@ func (s *sTenant) CreateMember(ctx context.Context, req *v1.TenantMemberCreateRe } // Create user - userResult, err := tx.Model("tnt_users").Ctx(ctx).Data(do.TntUsers{ + userResult, err := dao.TntUsers.Ctx(ctx).Data(do.TntUsers{ TenantId: tenantID, Username: username, Email: nullableEmail(email), @@ -553,39 +553,46 @@ func (s *sTenant) RemoveMember(ctx context.Context, req *v1.TenantMemberRemoveRe removedDisplayName = "[已移除成员]" } - // Revoke all sessions + // Revoke all sessions（Redis + DB 会话表）。会话撤销属于尽力而为的安全动作， + // 放在事务外先行执行：即便后续 DB 事务回滚，也已让该成员的活跃会话失效（fail-safe 方向）。 common.RevokeAllSessions(ctx, "tenant", memberID) - // Revoke all API keys for this user - _, err = dao.ApiKeys.Ctx(ctx). - Where("tenant_id", tenantID). - Where("user_id", memberID). - Data(do.ApiKeys{ - Status: "revoked", - }).Update() - if err != nil { - return nil, err - } + // 撤销 API Key → 删除成员模型范围 → 匿名化个人数据，三步写操作放入同一事务， + // 避免中途失败导致「Key 已撤销但用户未匿名化」等半完成的不一致状态。 + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // Revoke all API keys for this user + if _, err := dao.ApiKeys.Ctx(ctx). + Where("tenant_id", tenantID). + Where("user_id", memberID). + Data(do.ApiKeys{ + Status: "revoked", + }).Update(); err != nil { + return err + } - // Remove member model scopes - _, err = dao.TntMemberModelScopes.Ctx(ctx). - Where("tenant_id", tenantID). - Where("user_id", memberID). - Delete() - if err != nil { - return nil, err - } + // Remove member model scopes + if _, err := dao.TntMemberModelScopes.Ctx(ctx). + Where("tenant_id", tenantID). + Where("user_id", memberID). + Delete(); err != nil { + return err + } - // Anonymize personal data - _, err = dao.TntUsers.Ctx(ctx). - Where("id", memberID). - Where("tenant_id", tenantID). - Data(do.TntUsers{ - Status: "removed", - Email: fmt.Sprintf("deleted_%d@removed.local", memberID), - DisplayName: removedDisplayName, - Username: fmt.Sprintf("deleted_%d", memberID), - }).Update() + // Anonymize personal data + if _, err := dao.TntUsers.Ctx(ctx). + Where("id", memberID). + Where("tenant_id", tenantID). + Data(do.TntUsers{ + Status: "removed", + Email: fmt.Sprintf("deleted_%d@removed.local", memberID), + DisplayName: removedDisplayName, + Username: fmt.Sprintf("deleted_%d", memberID), + }).Update(); err != nil { + return err + } + + return nil + }) if err != nil { return nil, err } @@ -616,25 +623,27 @@ func (s *sTenant) DisableMember(ctx context.Context, tenantID, userID int64) err return nil // already disabled } - // Revoke all active API keys - _, err = dao.ApiKeys.Ctx(ctx). - Where("tenant_id", tenantID). - Where("user_id", userID). - Where("status", "active"). - Data(do.ApiKeys{ - Status: "disabled", - }).Update() - if err != nil { - return err - } + // 撤销 API Key 与更新成员状态放入同一事务，避免「Key 已禁用但成员状态未更新」的不一致。 + return g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // Revoke all active API keys + if _, err := dao.ApiKeys.Ctx(ctx). + Where("tenant_id", tenantID). + Where("user_id", userID). + Where("status", "active"). + Data(do.ApiKeys{ + Status: "disabled", + }).Update(); err != nil { + return err + } - _, err = dao.TntUsers.Ctx(ctx). - Where("id", userID). - Where("tenant_id", tenantID). - Data(do.TntUsers{ - Status: "disabled", - }).Update() - return err + _, err := dao.TntUsers.Ctx(ctx). + Where("id", userID). + Where("tenant_id", tenantID). + Data(do.TntUsers{ + Status: "disabled", + }).Update() + return err + }) } // EnableMember re-enables a member and restores their API keys. @@ -657,25 +666,27 @@ func (s *sTenant) EnableMember(ctx context.Context, tenantID, userID int64) erro return nil // already active or removed } - // Restore disabled API keys - _, err = dao.ApiKeys.Ctx(ctx). - Where("tenant_id", tenantID). - Where("user_id", userID). - Where("status", "disabled"). - Data(do.ApiKeys{ - Status: "active", - }).Update() - if err != nil { - return err - } + // 恢复 API Key 与更新成员状态放入同一事务，避免「Key 已恢复但成员状态未更新」的不一致。 + return g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // Restore disabled API keys + if _, err := dao.ApiKeys.Ctx(ctx). + Where("tenant_id", tenantID). + Where("user_id", userID). + Where("status", "disabled"). + Data(do.ApiKeys{ + Status: "active", + }).Update(); err != nil { + return err + } - _, err = dao.TntUsers.Ctx(ctx). - Where("id", userID). - Where("tenant_id", tenantID). - Data(do.TntUsers{ - Status: "active", - }).Update() - return err + _, err := dao.TntUsers.Ctx(ctx). + Where("id", userID). + Where("tenant_id", tenantID). + Data(do.TntUsers{ + Status: "active", + }).Update() + return err + }) } // UpdateMemberRole updates a member's role. diff --git a/internal/logic/tenant/member_model_scope.go b/internal/logic/tenant/member_model_scope.go index daef672..7505fdf 100644 --- a/internal/logic/tenant/member_model_scope.go +++ b/internal/logic/tenant/member_model_scope.go @@ -59,6 +59,11 @@ var memberModelScopeCache = lcommon.NewCache("member_model", 60*time.Second) // MemberModelScopesSet sets the available models for a member (full replace). func (s *sTenant) MemberModelScopesSet(ctx context.Context, req *v1.TenantMemberModelScopesSetReq) (*v1.TenantMemberModelScopesSetRes, error) { + // 仅 owner/admin 可管理成员的模型访问范围，防止普通成员给自己或他人越权开通模型 + role := middleware.GetUserRole(ctx) + if role != "owner" && role != "admin" { + return nil, lcommon.NewForbiddenError("需要 owner 或 admin 权限") + } if err := requireTeamEnabled(ctx); err != nil { return nil, err } @@ -79,9 +84,10 @@ func (s *sTenant) MemberModelScopesSet(ctx context.Context, req *v1.TenantMember return nil, lcommon.NewNotFoundError("成员") } + // 事务采用 ctx 传播式写法：闭包内统一使用 dao.Xxx.Ctx(ctx)，事务由 ctx 自动挂载。 err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Delete existing scopes - _, err := tx.Model("tnt_member_model_scopes").Ctx(ctx). + _, err := dao.TntMemberModelScopes.Ctx(ctx). Where("tenant_id", tenantID). Where("user_id", req.Id). Delete() @@ -99,13 +105,13 @@ func (s *sTenant) MemberModelScopesSet(ctx context.Context, req *v1.TenantMember ModelId: mID, } } - _, err = tx.Model("tnt_member_model_scopes").Ctx(ctx).Data(data).Insert() + _, err = dao.TntMemberModelScopes.Ctx(ctx).Data(data).Insert() if err != nil { return err } } else { // 空列表表示禁止所有模型，插入哨兵记录（model_id = -1） - _, err = tx.Model("tnt_member_model_scopes").Ctx(ctx).Data(do.TntMemberModelScopes{ + _, err = dao.TntMemberModelScopes.Ctx(ctx).Data(do.TntMemberModelScopes{ TenantId: tenantID, UserId: req.Id, ModelId: -1, diff --git a/internal/logic/tenant/model.go b/internal/logic/tenant/model.go index 5e65dd9..9c28272 100644 --- a/internal/logic/tenant/model.go +++ b/internal/logic/tenant/model.go @@ -362,9 +362,9 @@ func (s *sTenant) ListAvailableModels(ctx context.Context, req *v1.TenantAvailab } } - // 标记图片模型是否必须走异步端点（DashScope 等异步图片 provider）。 - // 判定与同步端点拦截 gate 同源（constant.IsAsyncImageProvider），保证在线体验示例 - // 与后端实际行为一致。仅对图片分类模型跑一次批量查询。 + // 标记图片模型可用的调用模式（同步端点 / 异步端点），供在线体验决定同步/异步开关。 + // 判定与同步端点拦截 gate 同源（constant.IsAsyncImageModel），保证在线体验示例与后端实际 + // 行为一致。仅对图片分类模型跑一次批量查询。 imageModelIDs := make([]string, 0) for _, item := range list { if item.Category == "image" { @@ -372,14 +372,20 @@ func (s *sTenant) ListAvailableModels(ctx context.Context, req *v1.TenantAvailab } } if len(imageModelIDs) > 0 { - asyncSet, err := lcommon.GetAsyncImageModelSet(ctx, imageModelIDs) + modes, err := lcommon.GetImageModelModes(ctx, imageModelIDs) if err != nil { return nil, err } for i := range list { - if list[i].Category == "image" && asyncSet[list[i].ModelId] { - list[i].AsyncImage = true + if list[i].Category != "image" { + continue } + m, ok := modes[list[i].ModelId] + if !ok { + continue + } + list[i].AsyncImage = m.AsyncSupported + list[i].ImageSyncSupported = m.SyncSupported } } diff --git a/internal/logic/tenant/open_platform.go b/internal/logic/tenant/open_platform.go index 8e51587..c429df0 100644 --- a/internal/logic/tenant/open_platform.go +++ b/internal/logic/tenant/open_platform.go @@ -157,8 +157,8 @@ func (s *sTenant) OpenAppCreate(ctx context.Context, req *v1.OpenAppCreateReq) ( } var id int64 - err = dao.OpnApps.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - result, err := tx.Model("opn_apps").Ctx(ctx).Data(do.OpnApps{ + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + result, err := dao.OpnApps.Ctx(ctx).Data(do.OpnApps{ TenantId: tenantID, Name: req.Name, Description: req.Description, @@ -176,7 +176,7 @@ func (s *sTenant) OpenAppCreate(ctx context.Context, req *v1.OpenAppCreateReq) ( if err != nil { return err } - _, err = tx.Model("opn_apps").Ctx(ctx). + _, err = dao.OpnApps.Ctx(ctx). Where("id", id). Data(opnAppEncryptedSecretData{EncryptedSecret: encryptedSecret}). Update() diff --git a/internal/logic/tenant/organization.go b/internal/logic/tenant/organization.go index fa9e3b4..1819fda 100644 --- a/internal/logic/tenant/organization.go +++ b/internal/logic/tenant/organization.go @@ -206,9 +206,9 @@ func (s *sTenant) TransferOwnership(ctx context.Context, req *v1.TenantOrgTransf return nil, common.NewNotFoundError("用户") } - err = dao.TntTenants.Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + err = g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { // Demote current owner to admin - _, err := tx.Model("tnt_users").Ctx(ctx). + _, err := dao.TntUsers.Ctx(ctx). Where("id", currentOwnerID). Data(do.TntUsers{ Role: "admin", @@ -218,7 +218,7 @@ func (s *sTenant) TransferOwnership(ctx context.Context, req *v1.TenantOrgTransf } // Promote new owner - _, err = tx.Model("tnt_users").Ctx(ctx). + _, err = dao.TntUsers.Ctx(ctx). Where("id", req.NewOwnerID). Data(do.TntUsers{ Role: "owner", @@ -228,7 +228,7 @@ func (s *sTenant) TransferOwnership(ctx context.Context, req *v1.TenantOrgTransf } // Update tenant owner reference - _, err = tx.Model("tnt_tenants").Ctx(ctx). + _, err = dao.TntTenants.Ctx(ctx). Where("id", tenantID). Data(do.TntTenants{ OwnerUserId: req.NewOwnerID, diff --git a/internal/logic/tenant/project.go b/internal/logic/tenant/project.go index 0c94e26..74e4dd8 100644 --- a/internal/logic/tenant/project.go +++ b/internal/logic/tenant/project.go @@ -127,8 +127,10 @@ func projectTotalCost(ctx context.Context, tenantID, projectID int64) (float64, } func exhaustProjectBudget(ctx context.Context, tenantID, projectID int64) error { + // 事务采用 ctx 传播式写法：闭包内所有 dao 操作统一使用 dao.Xxx.Ctx(ctx)， + // 由 ctx 自动挂载当前事务（不直接使用 tx 句柄）。务必保证 ctx 逐层传递。 return g.DB().Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { - _, err := tx.Model("tnt_projects").Ctx(ctx). + _, err := dao.TntProjects.Ctx(ctx). Where("id", projectID). Where("tenant_id", tenantID). Where("status", "active"). @@ -138,7 +140,7 @@ func exhaustProjectBudget(ctx context.Context, tenantID, projectID int64) error return err } - _, err = tx.Model("api_keys").Ctx(ctx). + _, err = dao.ApiKeys.Ctx(ctx). Where("tenant_id", tenantID). Where("project_id", projectID). Where("status", "active"). diff --git a/internal/logic/tenant/redemption.go b/internal/logic/tenant/redemption.go index e557ec8..874a0db 100644 --- a/internal/logic/tenant/redemption.go +++ b/internal/logic/tenant/redemption.go @@ -12,6 +12,7 @@ import ( do "github.com/qianfree/team-api/internal/model/do" + "github.com/qianfree/team-api/internal/dao" "github.com/qianfree/team-api/internal/logic/billing" lcommon "github.com/qianfree/team-api/internal/logic/common" "github.com/qianfree/team-api/internal/logic/payment" @@ -45,7 +46,7 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( Status string `json:"status"` ExpiresAt time.Time `json:"expires_at"` } - err := tx.Ctx(ctx).Model("ord_redemptions"). + err := dao.OrdRedemptions.Ctx(ctx). Where("code", req.Code). LockUpdate(). Scan(&redemption) @@ -59,7 +60,7 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( return gerror.Newf("兑换码状态为%s", redemption.Status) } if !redemption.ExpiresAt.IsZero() && redemption.ExpiresAt.Before(time.Now()) { - _, updateErr := tx.Ctx(ctx).Model("ord_redemptions"). + _, updateErr := dao.OrdRedemptions.Ctx(ctx). Where("id", redemption.ID). Data(do.OrdRedemptions{Status: "expired"}). Update() @@ -78,7 +79,7 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( switch redemption.Type { case "quota": - txID, err = creditWalletForRedemptionTx(ctx, tx, tenantID, redemption.Value, redemption.ID) + txID, err = creditWalletForRedemptionTx(ctx, tenantID, redemption.Value, redemption.ID) if err != nil { return err } @@ -107,7 +108,7 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( if redemption.DurationDays <= 0 { return lcommon.NewBusinessError(422, "时长兑换码缺少duration_days") } - err = extendPlanDurationTx(ctx, tx, tenantID, redemption.DurationDays) + err = extendPlanDurationTx(ctx, tenantID, redemption.DurationDays) if err != nil { return gerror.Wrapf(err, "延长套餐时长失败") } @@ -118,7 +119,7 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( } // 记录兑换使用记录 - _, err = tx.Ctx(ctx).Model("ord_redemption_usages").Insert(do.OrdRedemptionUsages{ + _, err = dao.OrdRedemptionUsages.Ctx(ctx).Insert(do.OrdRedemptionUsages{ RedemptionId: redemption.ID, TenantId: tenantID, UserId: userID, @@ -131,7 +132,7 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( } // 原子递增 used_count - _, err = tx.Ctx(ctx).Model("ord_redemptions"). + _, err = dao.OrdRedemptions.Ctx(ctx). Where("id", redemption.ID). Data(do.OrdRedemptions{ UsedCount: gdb.Raw("used_count + 1"), @@ -157,13 +158,13 @@ func (s *sTenant) RedeemCode(ctx context.Context, req *v1.TenantRedeemCodeReq) ( return res, nil } -// creditWalletForRedemptionTx 在事务内为租户钱包充值 -func creditWalletForRedemptionTx(ctx context.Context, tx gdb.TX, tenantID int64, amount float64, redemptionID int64) (int64, error) { +// creditWalletForRedemptionTx 在事务内为租户钱包充值（依赖调用方传入携带事务的 ctx） +func creditWalletForRedemptionTx(ctx context.Context, tenantID int64, amount float64, redemptionID int64) (int64, error) { type walletRow struct { ID int64 `json:"id"` } var w *walletRow - err := tx.Ctx(ctx).Model("bil_wallets"). + err := dao.BilWallets.Ctx(ctx). Where("tenant_id", tenantID). Fields("id"). Scan(&w) @@ -174,10 +175,11 @@ func creditWalletForRedemptionTx(ctx context.Context, tx gdb.TX, tenantID int64, return 0, nil } - _, err = tx.Ctx(ctx).Model("bil_wallets"). - Where("id", w.ID). - Data(do.BilWallets{Balance: gdb.Raw(fmt.Sprintf("balance + %v", amount))}). - Update() + // 钱包自增用参数化原生 SQL，避免用 fmt.Sprintf 拼 float 带来的精度丢失与注入隐患； + // updated_at 手动置为 NOW()，补回原 dao.Update() 自动填充的时间字段。 + _, err = g.DB().Ctx(ctx).Exec(ctx, + "UPDATE bil_wallets SET balance = balance + ?, updated_at = NOW() WHERE id = ?", + amount, w.ID) if err != nil { return 0, err } @@ -186,7 +188,7 @@ func creditWalletForRedemptionTx(ctx context.Context, tx gdb.TX, tenantID int64, Balance float64 `json:"balance"` FrozenBalance float64 `json:"frozen_balance"` } - err = tx.Ctx(ctx).Model("bil_wallets"). + err = dao.BilWallets.Ctx(ctx). Where("id", w.ID). Fields("balance, frozen_balance"). Scan(&balance) @@ -197,7 +199,7 @@ func creditWalletForRedemptionTx(ctx context.Context, tx gdb.TX, tenantID int64, return 0, gerror.New("wallet not found after update") } - id, err := tx.Ctx(ctx).Model("bil_transactions").InsertAndGetId(do.BilTransactions{ + id, err := dao.BilTransactions.Ctx(ctx).InsertAndGetId(do.BilTransactions{ TenantId: tenantID, WalletId: w.ID, Type: "recharge", @@ -215,9 +217,9 @@ func creditWalletForRedemptionTx(ctx context.Context, tx gdb.TX, tenantID int64, return id, nil } -// extendPlanDurationTx 在事务内延长套餐时长 -func extendPlanDurationTx(ctx context.Context, tx gdb.TX, tenantID int64, days int) error { - _, err := tx.Ctx(ctx).Exec( +// extendPlanDurationTx 在事务内延长套餐时长（依赖调用方传入携带事务的 ctx） +func extendPlanDurationTx(ctx context.Context, tenantID int64, days int) error { + _, err := g.DB().Ctx(ctx).Exec(ctx, "UPDATE pln_tenant_plans SET end_at = end_at + ?::integer * INTERVAL '1 day' WHERE tenant_id = ? AND status = ?", days, tenantID, "active") return err diff --git a/internal/logic/tenant/security.go b/internal/logic/tenant/security.go index ecca52d..665f2f8 100644 --- a/internal/logic/tenant/security.go +++ b/internal/logic/tenant/security.go @@ -355,6 +355,11 @@ func getEncKey(ctx context.Context) []byte { // GetIPWhitelist returns the tenant's IP whitelist configuration. func (s *sTenant) GetIPWhitelist(ctx context.Context, _ *v1.TenantIPWhitelistGetReq) (*v1.TenantIPWhitelistGetRes, error) { + // IP 白名单是租户级组织安全配置，仅 owner/admin 可查看 + role := middleware.GetUserRole(ctx) + if role != "owner" && role != "admin" { + return nil, common.NewForbiddenError("需要 owner 或 admin 权限") + } tenantID := middleware.GetTenantID(ctx) var tenant *entity.TntTenants err := dao.TntTenants.Ctx(ctx).Where("id", tenantID).Scan(&tenant) @@ -388,6 +393,11 @@ func (s *sTenant) GetIPWhitelist(ctx context.Context, _ *v1.TenantIPWhitelistGet // UpdateIPWhitelist updates the tenant's IP whitelist configuration. func (s *sTenant) UpdateIPWhitelist(ctx context.Context, req *v1.TenantIPWhitelistUpdateReq) (*v1.TenantIPWhitelistUpdateRes, error) { + // IP 白名单是租户级组织安全配置，仅 owner/admin 可修改，防止普通成员削弱整个组织的访问控制 + role := middleware.GetUserRole(ctx) + if role != "owner" && role != "admin" { + return nil, common.NewForbiddenError("需要 owner 或 admin 权限") + } tenantID := middleware.GetTenantID(ctx) var tenant *entity.TntTenants err := dao.TntTenants.Ctx(ctx).Where("id", tenantID).Scan(&tenant) diff --git a/internal/logic/tenant/task.go b/internal/logic/tenant/task.go index 91e582a..c32ba0e 100644 --- a/internal/logic/tenant/task.go +++ b/internal/logic/tenant/task.go @@ -170,6 +170,15 @@ func (s *sTenant) TenantTaskDetail(ctx context.Context, req *v1.TenantTaskDetail item.CreatedAt = *task.CreatedAt } + // 对 re-host 到对象存储的结果图，生成新鲜的缩略图 URL 供详情弹窗内联预览，并刷新原图 URL + // （规避 result_url 24h 预签名过期）。tenantID 已限定为当前租户，不跨租户。 + if thumb, orig := common.TaskResultImageURLs(ctx, tenantID, task.PublicTaskId, 600); thumb != "" { + item.ResultThumbURL = thumb + if orig != "" { + item.ResultURL = orig + } + } + return &v1.TenantTaskDetailRes{ Task: item, }, nil diff --git a/internal/middleware/admin_auth.go b/internal/middleware/admin_auth.go index 52158cc..1e2beab 100644 --- a/internal/middleware/admin_auth.go +++ b/internal/middleware/admin_auth.go @@ -199,7 +199,9 @@ func GetProjectID(ctx context.Context) int64 { } // extractBearerToken extracts the Bearer token from the Authorization header. -// Falls back to "token" query parameter for WebSocket connections. +// 仅在 WebSocket 握手请求上回退到 "token" query 参数——浏览器无法为 WebSocket 握手 +// 自定义 Authorization 头，只能用 query 传 token（如 /v1/realtime）。普通 REST 请求 +// 不走此回退，避免 token 泄漏进访问日志/代理日志/浏览器历史/Referer。 func extractBearerToken(r *ghttp.Request) string { authHeader := r.GetHeader("Authorization") if authHeader != "" { @@ -209,10 +211,19 @@ func extractBearerToken(r *ghttp.Request) string { } } - // Fallback: check query parameter for WebSocket (browsers can't set headers) - if token := r.Get("token").String(); token != "" { - return token + // 仅 WebSocket 升级握手才允许 query 参数回退（浏览器无法为 WS 握手设置请求头） + if isWebSocketUpgrade(r) { + if token := r.Get("token").String(); token != "" { + return token + } } return "" } + +// isWebSocketUpgrade 判断请求是否为 WebSocket 升级握手。 +// 浏览器发起 WebSocket 握手时会自动带上 "Upgrade: websocket" 头（无法禁用）， +// 据此把 query token 回退严格限定在 WS 场景，普通 HTTP 请求一律不生效。 +func isWebSocketUpgrade(r *ghttp.Request) bool { + return strings.EqualFold(r.GetHeader("Upgrade"), "websocket") +} diff --git a/internal/middleware/idempotency.go b/internal/middleware/idempotency.go index 05dc9b6..9a7bc6e 100644 --- a/internal/middleware/idempotency.go +++ b/internal/middleware/idempotency.go @@ -1,6 +1,7 @@ package middleware import ( + "fmt" "net/http" "time" @@ -25,15 +26,25 @@ const ( // Idempotency ensures that requests with the same Idempotency-Key // are processed only once. Subsequent requests with the same key // return the cached response. +// +// 幂等键按认证主体（userType + tenantID + userID）隔离：不同租户/用户即便复用同一 +// 客户端 Idempotency-Key，也各自独立，避免命中彼此缓存的响应体（跨租户数据泄漏） +// 或彼此误触 409。因此本中间件必须挂载在认证中间件（如 TenantAuth）之后。 func Idempotency(r *ghttp.Request) { - idempotencyKey := r.GetHeader(IdempotencyHeader) - if idempotencyKey == "" { + clientKey := r.GetHeader(IdempotencyHeader) + if clientKey == "" { r.Middleware.Next() return } ctx := r.Context() + // 组合出按主体隔离的存储键（仍是单一字符串，不改表结构） + tenantID := r.GetCtxVar(CtxKeyTenantID).Int64() + userID := r.GetCtxVar(CtxKeyUserID).Int64() + userType := r.GetCtxVar(CtxKeyUserType).String() + idempotencyKey := buildIdempotencyKey(userType, tenantID, userID, clientKey) + // Check if this key has been processed before var record struct { Id int64 `json:"id"` @@ -106,6 +117,13 @@ func Idempotency(r *ghttp.Request) { }).Update() } +// buildIdempotencyKey 组合出按认证主体隔离的幂等存储键。 +// 加入 userType/tenantID/userID 前缀，使不同主体即便复用同一客户端 key 也互不干扰， +// 杜绝跨租户命中彼此缓存响应体（数据泄漏）或误触 409。 +func buildIdempotencyKey(userType string, tenantID, userID int64, clientKey string) string { + return fmt.Sprintf("%s:%d:%d:%s", userType, tenantID, userID, clientKey) +} + // writeConflictResponse writes a 409 Conflict response for duplicate idempotency keys. func writeConflictResponse(r *ghttp.Request) { r.Response.WriteStatus(409) diff --git a/internal/middleware/open_auth.go b/internal/middleware/open_auth.go index 11f2994..7f55e8b 100644 --- a/internal/middleware/open_auth.go +++ b/internal/middleware/open_auth.go @@ -121,6 +121,12 @@ func OpenPlatformAuth(r *ghttp.Request) { return } + // 防重放：签名校验通过后，将 nonce 存入 Redis 去重。放在签名校验之后， + // 避免未认证攻击者用伪造 nonce 刷爆 Redis。命中已存在的 nonce 即判为重放，拒绝。 + if !checkOpenAppNonce(r, app.Id, nonce) { + return + } + // Inject context variables using typed keys ctx := r.Context() ctx = context.WithValue(ctx, ctxKeyOpenAppID, app.Id) @@ -132,6 +138,48 @@ func OpenPlatformAuth(r *ghttp.Request) { r.Middleware.Next() } +// checkOpenAppNonce enforces replay protection by recording each nonce in Redis. +// 用 SET NX 原子写入 nonce：首次出现返回 OK 放行；已存在说明是重放，写 401 并返回 false。 +// Redis 故障时 fail-open（与限流一致），仅记告警，因为时间戳窗口本身已把可重放时间 +// 限制在数分钟内。 +func checkOpenAppNonce(r *ghttp.Request, appDBID int64, nonce string) bool { + ctx := r.Context() + isNew, err := recordOpenNonce(ctx, appDBID, nonce) + if err != nil { + g.Log().Warningf(ctx, "[OpenAuth] nonce SET NX failed app=%d: %v", appDBID, err) + // Fail open on Redis error + return true + } + if !isNew { + // nonce 已存在 → 重放请求 + response.Error(r, consts.ErrUnauthorized) + return false + } + return true +} + +// recordOpenNonce 原子记录 nonce，返回 (isNew, err)。 +// isNew==true 表示该 nonce 首次出现（应放行）；false 表示已存在（重放）。 +func recordOpenNonce(ctx context.Context, appDBID int64, nonce string) (bool, error) { + res, err := g.Redis().Do(ctx, "SET", openNonceKey(appDBID, nonce), "1", "NX", "EX", openNonceTTLSeconds()) + if err != nil { + return false, err + } + return !res.IsNil(), nil +} + +// openNonceKey 构造按应用隔离的 nonce 去重键。 +func openNonceKey(appDBID int64, nonce string) string { + return fmt.Sprintf("open:nonce:%d:%s", appDBID, nonce) +} + +// openNonceTTLSeconds 返回 nonce 记录的存活秒数。 +// 取 2 倍时间戳偏移窗口——请求在 [ts-skew, ts+skew] 内都可能通过时间戳校验， +// 记住这么久即可覆盖同一 nonce 的全部可重放区间。 +func openNonceTTLSeconds() int { + return int(2 * openPlatformMaxSkew / time.Second) +} + func computeHMACSignature(secret, timestamp, nonce, method, path string, body []byte) string { bodyHash := sha256.Sum256(body) bodyHashHex := hex.EncodeToString(bodyHash[:]) diff --git a/internal/middleware/operation_log.go b/internal/middleware/operation_log.go index 07f3ce4..701d6f1 100644 --- a/internal/middleware/operation_log.go +++ b/internal/middleware/operation_log.go @@ -25,6 +25,9 @@ var sensitiveFields = map[string]bool{ "secret": true, "api_key": true, "encrypted_key": true, + // 存储配置凭证（系统设置更新 / 存储连通性测试的请求体中携带） + "storage_access_key_id": true, + "storage_access_key_secret": true, } // resourcePluralToSingular maps URL path resource names to singular form. diff --git a/internal/middleware/security_fixes_test.go b/internal/middleware/security_fixes_test.go new file mode 100644 index 0000000..2e28f7b --- /dev/null +++ b/internal/middleware/security_fixes_test.go @@ -0,0 +1,92 @@ +package middleware + +import ( + "testing" + "time" +) + +// TestBuildIdempotencyKey 验证幂等键按认证主体隔离（对应安全修复 #7）。 +// 核心不变量：不同租户/用户即便复用同一客户端 Idempotency-Key，也必须得到不同的存储键， +// 否则会命中彼此缓存的响应体（跨租户数据泄漏）或互相误触 409。 +func TestBuildIdempotencyKey(t *testing.T) { + const clientKey = "client-supplied-uuid" + + t.Run("相同主体+相同客户端key -> 相同存储键（幂等生效）", func(t *testing.T) { + a := buildIdempotencyKey("tenant", 1, 10, clientKey) + b := buildIdempotencyKey("tenant", 1, 10, clientKey) + if a != b { + t.Fatalf("same principal must yield same key: %q != %q", a, b) + } + }) + + t.Run("不同租户复用同一客户端key -> 不同存储键（防跨租户泄漏）", func(t *testing.T) { + tenantA := buildIdempotencyKey("tenant", 1, 10, clientKey) + tenantB := buildIdempotencyKey("tenant", 2, 10, clientKey) + if tenantA == tenantB { + t.Fatalf("different tenants must not collide on the same client key: %q", tenantA) + } + }) + + t.Run("同租户不同用户 -> 不同存储键", func(t *testing.T) { + userA := buildIdempotencyKey("tenant", 1, 10, clientKey) + userB := buildIdempotencyKey("tenant", 1, 11, clientKey) + if userA == userB { + t.Fatalf("different users must not collide: %q", userA) + } + }) + + t.Run("不同用户类型 -> 不同存储键（admin 与 tenant 不混）", func(t *testing.T) { + admin := buildIdempotencyKey("admin", 1, 10, clientKey) + tenant := buildIdempotencyKey("tenant", 1, 10, clientKey) + if admin == tenant { + t.Fatalf("different user types must not collide: %q", admin) + } + }) + + t.Run("不同客户端key -> 不同存储键", func(t *testing.T) { + k1 := buildIdempotencyKey("tenant", 1, 10, "key-1") + k2 := buildIdempotencyKey("tenant", 1, 10, "key-2") + if k1 == k2 { + t.Fatalf("different client keys must differ: %q", k1) + } + }) +} + +// TestOpenNonceKey 验证 nonce 去重键按应用隔离（对应安全修复 #4）。 +func TestOpenNonceKey(t *testing.T) { + t.Run("相同 app+nonce -> 相同键（确定性）", func(t *testing.T) { + k1 := openNonceKey(7, "abc") + k2 := openNonceKey(7, "abc") + if k1 != k2 { + t.Fatal("nonce key must be deterministic") + } + }) + + t.Run("不同 app 相同 nonce -> 不同键（按应用隔离）", func(t *testing.T) { + if openNonceKey(7, "abc") == openNonceKey(8, "abc") { + t.Fatal("same nonce under different apps must not collide") + } + }) + + t.Run("相同 app 不同 nonce -> 不同键", func(t *testing.T) { + if openNonceKey(7, "abc") == openNonceKey(7, "xyz") { + t.Fatal("different nonces must differ") + } + }) +} + +// TestOpenNonceTTL 验证 nonce 记录存活时间足以覆盖整个可重放窗口（对应安全修复 #4）。 +// 请求在 [ts-skew, ts+skew] 内均可能通过时间戳校验，因此 nonce 至少要记住 2×skew， +// 才能保证同一 nonce 在其所有可被接受的时刻都已存在于 Redis。 +func TestOpenNonceTTL(t *testing.T) { + ttl := openNonceTTLSeconds() + + minRequired := int(2 * openPlatformMaxSkew / time.Second) + if ttl < minRequired { + t.Fatalf("nonce TTL %ds must cover the full replay window (>= %ds)", ttl, minRequired) + } + // 当前配置：skew=5min → TTL=600s + if ttl != 600 { + t.Fatalf("expected TTL 600s for a 5min skew window, got %ds", ttl) + } +} diff --git a/internal/model/do/aud_request_logs.go b/internal/model/do/aud_request_logs.go index 6d6dbbd..3cd909a 100644 --- a/internal/model/do/aud_request_logs.go +++ b/internal/model/do/aud_request_logs.go @@ -42,5 +42,4 @@ type AudRequestLogs struct { TaskResult any // 异步任务完成时上游返回的原始响应体 TaskUpstreamHeaders any // 异步任务完成时上游返回的响应头（仅审计级别为 full 时记录） TaskCompletedAt *gtime.Time // 异步任务达到终态的时间 - AttachmentIds any // 提取的媒体附件 fil_files ID 数组（JSONB: [1,2,3]） } diff --git a/internal/model/entity/aud_request_logs.go b/internal/model/entity/aud_request_logs.go index bf20994..dc48e93 100644 --- a/internal/model/entity/aud_request_logs.go +++ b/internal/model/entity/aud_request_logs.go @@ -40,5 +40,4 @@ type AudRequestLogs struct { TaskResult string `json:"task_result" orm:"task_result" description:"异步任务完成时上游返回的原始响应体"` // 异步任务完成时上游返回的原始响应体 TaskUpstreamHeaders string `json:"task_upstream_headers" orm:"task_upstream_headers" description:"异步任务完成时上游返回的响应头（仅审计级别为 full 时记录）"` // 异步任务完成时上游返回的响应头（仅审计级别为 full 时记录） TaskCompletedAt *gtime.Time `json:"task_completed_at" orm:"task_completed_at" description:"异步任务达到终态的时间"` // 异步任务达到终态的时间 - AttachmentIds string `json:"attachment_ids" orm:"attachment_ids" description:"提取的媒体附件 fil_files ID 数组（JSONB: [1,2,3]）"` // 提取的媒体附件 fil_files ID 数组（JSONB: [1,2,3]） } diff --git a/internal/model/entity/sys_admin_users.go b/internal/model/entity/sys_admin_users.go index 5839ac5..8c934c8 100644 --- a/internal/model/entity/sys_admin_users.go +++ b/internal/model/entity/sys_admin_users.go @@ -10,20 +10,20 @@ import ( // SysAdminUsers is the golang structure for table sys_admin_users. type SysAdminUsers struct { - Id int64 `json:"id" orm:"id" description:"主键ID"` // 主键ID - Username string `json:"username" orm:"username" description:"登录用户名"` // 登录用户名 - PasswordHash string `json:"password_hash" orm:"password_hash" description:"密码哈希（bcrypt）"` // 密码哈希（bcrypt） - Email string `json:"email" orm:"email" description:"邮箱地址（可选，可为NULL）"` // 邮箱地址（可选，可为NULL） - DisplayName string `json:"display_name" orm:"display_name" description:"显示名称"` // 显示名称 - Role string `json:"role" orm:"role" description:"角色：super_admin（全权限）/ admin（可配置权限）"` // 角色：super_admin（全权限）/ admin（可配置权限） - Status string `json:"status" orm:"status" description:"状态：active（启用）/ disabled（禁用）"` // 状态：active（启用）/ disabled（禁用） - LastLoginAt *gtime.Time `json:"last_login_at" orm:"last_login_at" description:"最后登录时间"` // 最后登录时间 - LastLoginIp string `json:"last_login_ip" orm:"last_login_ip" description:"最后登录IP"` // 最后登录IP - CreatedAt *gtime.Time `json:"created_at" orm:"created_at" description:"创建时间"` // 创建时间 - UpdatedAt *gtime.Time `json:"updated_at" orm:"updated_at" description:"更新时间"` // 更新时间 - TotpSecret string `json:"totp_secret" orm:"totp_secret" description:"TOTP 密钥（AES-256 加密存储）"` // TOTP 密钥（AES-256 加密存储） - TotpEnabled bool `json:"totp_enabled" orm:"totp_enabled" description:"是否启用双因素认证"` // 是否启用双因素认证 - BackupCodes string `json:"backup_codes" orm:"backup_codes" description:"备用恢复码（bcrypt 哈希存储）"` // 备用恢复码（bcrypt 哈希存储） - FailedAttempts int `json:"failed_attempts" orm:"failed_attempts" description:"连续登录失败次数（成功登录后归零）"` // 连续登录失败次数（成功登录后归零） - LockedUntil *gtime.Time `json:"locked_until" orm:"locked_until" description:"锁定截止时间（连续5次失败后锁定30分钟）"` // 锁定截止时间（连续5次失败后锁定30分钟） + Id int64 `json:"id" orm:"id" description:"主键ID"` // 主键ID + Username string `json:"username" orm:"username" description:"登录用户名"` // 登录用户名 + PasswordHash string `json:"password_hash" orm:"password_hash" description:"密码哈希（bcrypt）"` // 密码哈希（bcrypt） + Email string `json:"email" orm:"email" description:"邮箱地址（可选，可为NULL）"` // 邮箱地址（可选，可为NULL） + DisplayName string `json:"display_name" orm:"display_name" description:"显示名称"` // 显示名称 + Role string `json:"role" orm:"role" description:"角色：super_admin（全权限）/ admin（可配置权限）"` // 角色：super_admin（全权限）/ admin（可配置权限） + Status string `json:"status" orm:"status" description:"状态：active（启用）/ disabled（禁用）"` // 状态：active（启用）/ disabled（禁用） + LastLoginAt *gtime.Time `json:"last_login_at" orm:"last_login_at" description:"最后登录时间"` // 最后登录时间 + LastLoginIp string `json:"last_login_ip" orm:"last_login_ip" description:"最后登录IP"` // 最后登录IP + CreatedAt *gtime.Time `json:"created_at" orm:"created_at" description:"创建时间"` // 创建时间 + UpdatedAt *gtime.Time `json:"updated_at" orm:"updated_at" description:"更新时间"` // 更新时间 + TotpSecret string `json:"totp_secret" orm:"totp_secret" description:"TOTP 密钥（AES-256 加密存储）"` // TOTP 密钥（AES-256 加密存储） + TotpEnabled bool `json:"totp_enabled" orm:"totp_enabled" description:"是否启用双因素认证"` // 是否启用双因素认证 + BackupCodes string `json:"backup_codes" orm:"backup_codes" description:"备用恢复码（bcrypt 哈希存储）"` // 备用恢复码（bcrypt 哈希存储） + FailedAttempts int `json:"failed_attempts" orm:"failed_attempts" description:"连续登录失败次数（成功登录后归零）"` // 连续登录失败次数（成功登录后归零） + LockedUntil *gtime.Time `json:"locked_until" orm:"locked_until" description:"锁定截止时间（连续5次失败后锁定30分钟）"` // 锁定截止时间（连续5次失败后锁定30分钟） } diff --git a/internal/service/admin.go b/internal/service/admin.go index d60e2b2..e9b212c 100644 --- a/internal/service/admin.go +++ b/internal/service/admin.go @@ -403,6 +403,13 @@ type ( GetSettings(ctx context.Context, req *v1.AdminSettingsGetReq) (*v1.AdminSettingsGetRes, error) // UpdateSettings batch-updates settings for a given category. UpdateSettings(ctx context.Context, req *v1.AdminSettingsUpdateReq) (*v1.AdminSettingsUpdateRes, error) + // TestStorageConfig 测试对象存储配置连通性：用生成的测试图片走一次「上传 → 下载校验 → 删除」 + // 完整往返，验证凭证 / 桶 / 端点 / 读写删权限是否正常。 + // + // access_key_id / secret 为空或掩码（"******"）时回落到已保存的配置值——与 GET 接口对敏感字段 + // 的掩码逻辑（config.GetCategoryWithValues）以及 Turnstile 校验的 "******" 处理保持一致， + // 从而支持管理员在「未点保存」的状态下直接测试表单中新填/改动的配置。 + TestStorageConfig(ctx context.Context, req *v1.AdminStorageTestReq) (*v1.AdminStorageTestRes, error) // TaskList 大模型异步任务列表 TaskList(ctx context.Context, req *v1.TaskListReq) (*v1.TaskListRes, error) // TaskDetail 大模型异步任务详情 diff --git a/internal/utility/crypto/crypto.go b/internal/utility/crypto/crypto.go index d5965df..02040c1 100644 --- a/internal/utility/crypto/crypto.go +++ b/internal/utility/crypto/crypto.go @@ -91,15 +91,26 @@ func DecryptString(key []byte, encoded string) (string, error) { return string(plain), nil } -// MustGetEncryptionKey returns the encryption key from a hex-encoded string. -// Panics if the key is invalid. -func MustGetEncryptionKey(hexKey string) []byte { +// GetEncryptionKey decodes and validates a hex-encoded AES-256 key. +// Returns an error instead of panicking, so callers can validate at startup. +func GetEncryptionKey(hexKey string) ([]byte, error) { key, err := hex.DecodeString(hexKey) if err != nil { - panic(fmt.Sprintf("invalid encryption key hex: %v", err)) + return nil, fmt.Errorf("invalid encryption key hex: %w", err) } if len(key) != 32 { - panic(fmt.Sprintf("encryption key must be 32 bytes, got %d", len(key))) + return nil, fmt.Errorf("%w: got %d", ErrInvalidKeyLength, len(key)) + } + return key, nil +} + +// MustGetEncryptionKey returns the encryption key from a hex-encoded string. +// Panics if the key is invalid. Prefer validating the key at startup (see +// GetEncryptionKey) so this never panics on a live request path. +func MustGetEncryptionKey(hexKey string) []byte { + key, err := GetEncryptionKey(hexKey) + if err != nil { + panic(err.Error()) } return key } diff --git a/internal/utility/export/csv.go b/internal/utility/export/csv.go index 29c4438..9725b5e 100644 --- a/internal/utility/export/csv.go +++ b/internal/utility/export/csv.go @@ -39,7 +39,8 @@ func StreamCSV(r *ghttp.Request, config Config, queryFn func(yield func(map[stri queryFn(func(row map[string]any) bool { record := make([]string, len(config.Columns)) for i, col := range config.Columns { - record[i] = GetCellValue(row, col) + // B14：CSV 公式注入防御，中和以 =+-@ 等开头的用户可控字段 + record[i] = sanitizeCSVField(GetCellValue(row, col)) } _ = writer.Write(record) rowCount++ diff --git a/internal/utility/export/helper.go b/internal/utility/export/helper.go index a159527..d48696e 100644 --- a/internal/utility/export/helper.go +++ b/internal/utility/export/helper.go @@ -3,6 +3,7 @@ package export import ( "fmt" "net/url" + "strconv" "strings" "time" @@ -57,3 +58,23 @@ func GetCellValue(row map[string]any, col Column) string { } return fmt.Sprintf("%v", val) } + +// sanitizeCSVField 防御 CSV 公式注入（CWE-1236）。 +// Excel/LibreOffice/Google Sheets 打开 CSV 时，会把以 = + - @ 及 TAB(0x09)/CR(0x0D) 开头的 +// 单元格当作公式求值，用户可控字段（租户名/成员名/工单标题等）可借此注入 DDE/命令执行。 +// 处理策略：对以危险字符开头且【整体不是合法数字】的值，前置单引号中和为纯文本； +// 合法数字（含负号/正号/科学计数法）保持原样，避免破坏数值列的可计算性。 +// 仅用于 CSV 写入；XLSX 经 excelize 以字符串类型单元格存储，Excel 不会将其当公式求值，无需处理。 +func sanitizeCSVField(s string) string { + if s == "" { + return s + } + switch s[0] { + case '=', '+', '-', '@', '\t', '\r': + if _, err := strconv.ParseFloat(strings.TrimSpace(s), 64); err == nil { + return s // 纯数字，Excel 不会当公式，安全 + } + return "'" + s + } + return s +} diff --git a/migrations/000012_v0_1_14_version.sql b/migrations/000012_v0_1_14_version.sql new file mode 100644 index 0000000..f6110f1 --- /dev/null +++ b/migrations/000012_v0_1_14_version.sql @@ -0,0 +1,18 @@ +-- +goose Up + +-- A3 修复：结算幂等 —— bil_records.request_id 由普通 btree 索引升级为唯一索引。 +-- 原 idx_bil_records_request 无唯一性保护，同一 request_id 重复结算会写入多条计费记录 +-- 并重复扣款。唯一索引作为 DB 层最终兜底：重复结算的第二次 INSERT 触发 23505 唯一冲突， +-- 结算事务整体回滚（钱包扣款一并撤销），应用层据此识别为幂等空操作。 +-- +-- 前置约束：若历史数据因该 bug 已产生重复 request_id，本迁移会失败（这是有意为之—— +-- 金融记录不做静默删除，应由运维先人工核对重复计费记录再重跑）。bil_prededuct_tracks +-- 早已对 request_id 施加 UNIQUE，证明「一次请求一条记录」本就是系统不变式。 +DROP INDEX IF EXISTS idx_bil_records_request; +CREATE UNIQUE INDEX IF NOT EXISTS uk_bil_records_request ON bil_records USING btree (request_id); + +-- +goose Down + +-- 还原为非唯一 btree 索引 +DROP INDEX IF EXISTS uk_bil_records_request; +CREATE INDEX IF NOT EXISTS idx_bil_records_request ON bil_records USING btree (request_id); diff --git a/relay/constant/errors.go b/relay/constant/errors.go index a868f8f..6704d0f 100644 --- a/relay/constant/errors.go +++ b/relay/constant/errors.go @@ -103,6 +103,20 @@ func NewModelGoneError(modelName, sunsetDate string) *RelayError { } } +// IsResponseWritten 判断错误对应的响应体是否已由 adaptor 直接写入客户端。 +// 一旦上游的原生错误体已透传给客户端，重试后再写成功体会造成「错误体+成功体」拼接污染， +// 因此重试循环命中此情况必须终止重试（即使 StatusCode 本身可重试）。 +func IsResponseWritten(err error) bool { + if err == nil { + return false + } + var relayErr *RelayError + if errors.As(err, &relayErr) { + return relayErr.ResponseWritten + } + return false +} + // IsRetryable 判断错误是否可重试 func IsRetryable(err error) bool { if err == nil { @@ -159,3 +173,46 @@ func isTransientNetworkError(err error) bool { return false } + +// isAmbiguousDelivery 判断连接层错误是否属于「请求可能已送达上游」的模糊情形。 +// 这类错误（io.EOF/ErrUnexpectedEOF/ECONNRESET/超时）发生时，上游有可能已收到并开始 +// 处理请求，只是响应在回传途中中断——对非幂等且高成本的生成（图片/视频）重试会造成 +// 上游重复生成 + 重复计费。 +// 反之不属于模糊情形（重试安全）：① 连接被拒绝 / DNS 解析失败——请求确定未送达； +// ② 状态码类 RelayError——已拿到上游 HTTP 响应，说明上游已明确返回错误、未成功生成。 +func isAmbiguousDelivery(err error) bool { + if err == nil { + return false + } + // 拿到了 HTTP 响应（哪怕是错误状态码）→ 非模糊。 + var relayErr *RelayError + if errors.As(err, &relayErr) { + return false + } + // 连接被拒绝：上游从未接受连接，请求确定未送达 → 非模糊。 + if errors.Is(err, syscall.ECONNREFUSED) { + return false + } + // DNS 解析失败：请求确定未送达 → 非模糊。 + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return false + } + // 其余瞬时网络错误（EOF/RST/i/o timeout/DeadlineExceeded）：可能已送达 → 模糊。 + return isTransientNetworkError(err) +} + +// IsRetryableForRequest 在 DoRequest（发送请求）阶段判断错误是否可重试。 +// 相比 IsRetryable 多一个 nonIdempotentExpensive 参数：对非幂等且高成本的生成端点 +// （图片/视频），「可能已送达上游」的模糊连接层错误不重试，避免上游重复生成 + 重复计费； +// 「确定未送达」的错误（连接被拒/DNS 失败）与状态码类错误仍照常重试。其它端点（chat 等） +// 行为与 IsRetryable 完全一致。 +func IsRetryableForRequest(err error, nonIdempotentExpensive bool) bool { + if !IsRetryable(err) { + return false + } + if nonIdempotentExpensive && isAmbiguousDelivery(err) { + return false + } + return true +} diff --git a/relay/constant/errors_test.go b/relay/constant/errors_test.go new file mode 100644 index 0000000..29721e2 --- /dev/null +++ b/relay/constant/errors_test.go @@ -0,0 +1,51 @@ +package constant + +import ( + "io" + "syscall" + "testing" +) + +// TestIsRetryableForRequest 覆盖 DoRequest 阶段的 mode 感知重试判定： +// 对高成本非幂等生成（图片/视频），「可能已送达上游」的模糊网络错误不重试； +// 「确定未送达」的错误与状态码类错误仍照常重试；其它端点（chat）行为不变。 +func TestIsRetryableForRequest(t *testing.T) { + relay502 := &RelayError{StatusCode: 502} + relay400 := &RelayError{StatusCode: 400} + + cases := []struct { + name string + err error + expensive bool + want bool + }{ + // chat（expensive=false）：与 IsRetryable 一致，模糊错误照常重试。 + {"chat/eof", io.EOF, false, true}, + {"chat/reset", syscall.ECONNRESET, false, true}, + {"chat/refused", syscall.ECONNREFUSED, false, true}, + + // 图片/视频（expensive=true）：模糊「可能已送达」错误不重试。 + {"image/eof", io.EOF, true, false}, + {"image/unexpected_eof", io.ErrUnexpectedEOF, true, false}, + {"image/reset", syscall.ECONNRESET, true, false}, + + // 图片/视频：确定未送达 → 仍重试。 + {"image/refused", syscall.ECONNREFUSED, true, true}, + + // 状态码类错误：已拿到上游响应（未成功生成）→ 图片也可重试。 + {"image/status_502", relay502, true, true}, + + // 本就不可重试的错误：任何情况都不重试。 + {"image/status_400", relay400, true, false}, + {"chat/status_400", relay400, false, false}, + {"nil", nil, true, false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsRetryableForRequest(c.err, c.expensive); got != c.want { + t.Fatalf("IsRetryableForRequest(%v, expensive=%v) = %v, want %v", c.err, c.expensive, got, c.want) + } + }) + } +} diff --git a/relay/handler/realtime_handler.go b/relay/handler/realtime_handler.go index cd7ec10..1ca86d6 100644 --- a/relay/handler/realtime_handler.go +++ b/relay/handler/realtime_handler.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "os" "strings" "time" @@ -19,7 +21,42 @@ import ( // websocketUpgrader HTTP → WebSocket 升级器 var websocketUpgrader = websocket.Upgrader{ Subprotocols: []string{"realtime"}, - CheckOrigin: func(r *http.Request) bool { return true }, + CheckOrigin: checkRealtimeOrigin, +} + +// checkRealtimeOrigin 校验 WebSocket 握手来源，防跨站 WebSocket 劫持（CSWSH，B15）。 +// 策略： +// - 无 Origin 头：放行。Realtime API 的主要客户端是原生 SDK/CLI/服务端，不发送 Origin； +// 浏览器才强制带 Origin，据此区分。 +// - 同源（Origin.host == 请求 Host）：放行。 +// - 命中 REALTIME_ALLOWED_ORIGINS（逗号分隔的可信来源，如租户控制台/playground 域名）：放行。 +// - 其余跨源浏览器请求：拒绝。 +func checkRealtimeOrigin(r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin == "" { + return true // 非浏览器客户端 + } + + u, err := url.Parse(origin) + if err != nil || u.Host == "" { + return false + } + + if strings.EqualFold(u.Host, r.Host) { + return true // 同源 + } + + for _, allowed := range strings.Split(os.Getenv("REALTIME_ALLOWED_ORIGINS"), ",") { + allowed = strings.TrimSpace(allowed) + if allowed == "" { + continue + } + if strings.EqualFold(origin, allowed) || strings.EqualFold(u.Host, allowed) { + return true + } + } + + return false } // RealtimeContext Realtime 请求上下文 diff --git a/relay/handler/relay_handler.go b/relay/handler/relay_handler.go index dd3aa5f..b2b3223 100644 --- a/relay/handler/relay_handler.go +++ b/relay/handler/relay_handler.go @@ -434,7 +434,7 @@ func RelayHandler(ctx context.Context, body []byte, path string, headers http.He "this image model uses asynchronous generation on Alibaba DashScope; submit via POST /v1/images/generations/async and poll for the result", nil) } - info := buildRelayInfo(ctx, rc, v, selection, path, headers) + info := buildRelayInfo(ctx, rc, v, selection, path, headers, attempt) if tr := monitor.GetTrackedRequest(rc.RequestID); tr != nil { tr.ChannelID = selection.ChannelID @@ -473,8 +473,15 @@ func RelayHandler(ctx context.Context, body []byte, path string, headers http.He // 发送请求到上游 resp, err := adaptor.DoRequest(upstreamCtx, info, convertedBody) if err != nil { + // 高成本非幂等生成（图片/视频）：DoRequest 阶段「可能已送达上游」的模糊网络错误 + // （EOF/RST/超时）不重试，避免上游重复生成 + 重复计费；连接被拒/DNS 失败等确定 + // 未送达的错误与状态码类错误仍照常重试。chat 等端点行为不变。 + expensiveGen := v.relayMode == constant.RelayModeImagesGenerations || + v.relayMode == constant.RelayModeImagesEdits || + v.relayMode == constant.RelayModeVideoGenerations + retryable := constant.IsRetryableForRequest(err, expensiveGen) failReason := fmt.Sprintf("attempt=%d channel=%d(%s) model=%s upstreamModel=%s error=[%v] latency=%.0fms retryable=%v", - attempt, selection.ChannelID, selection.ChannelName, v.modelName, selection.UpstreamModelName, err, info.LatencyMs(), constant.IsRetryable(err)) + attempt, selection.ChannelID, selection.ChannelName, v.modelName, selection.UpstreamModelName, err, info.LatencyMs(), retryable) channelErrors = append(channelErrors, failReason) g.Log().Warningf(ctx, "[RelayHandler] Upstream request failed: %s", failReason) @@ -483,7 +490,7 @@ func RelayHandler(ctx context.Context, body []byte, path string, headers http.He excludeChannelIDs = append(excludeChannelIDs, selection.ChannelID) scheduler.GetGlobalAffinity().Delete(rc.TenantID, rc.UserID, v.modelName) - if constant.IsRetryable(err) && attempt < maxRetries { + if retryable && attempt < maxRetries { recordChannelError(rc, selection, v.modelName, attempt, false, err, info.LatencyMs()) appendHop(trace, hop, false, err.Error(), info.LatencyMs()) settleCancel() @@ -547,7 +554,9 @@ func RelayHandler(ctx context.Context, body []byte, path string, headers http.He channelErrors = append(channelErrors, failReason) scheduler.GetGlobalAffinity().Delete(rc.TenantID, rc.UserID, v.modelName) - if v.isStream || !constant.IsRetryable(err) || attempt >= maxRetries { + // 响应已由 adaptor 直接写入客户端（原生错误透传）时，即使 StatusCode 可重试也必须 + // 终止重试：否则重试成功会把成功体追加到已写的错误体后，造成响应污染（错误体+成功体拼接）。 + if v.isStream || !constant.IsRetryable(err) || attempt >= maxRetries || constant.IsResponseWritten(err) { if billing != nil && preDeductAmount > 0 { _ = billing.SettleFailed(settleCtx, rc.TenantID, rc.RequestID, preDeductAmount) } @@ -581,7 +590,8 @@ func RelayHandler(ctx context.Context, body []byte, path string, headers http.He } // buildRelayInfo 从渠道选择结果构建 RelayInfo -func buildRelayInfo(ctx context.Context, rc *RelayContext, v *relayValidation, selection *common.ChannelSelection, path string, headers http.Header) *common.RelayInfo { +// attempt 为当前重试轮次（0=首次），写入 RetryIndex 供 ParamOverride「是否重试」规则与 bil_usage_logs.retry_index 使用（C3）。 +func buildRelayInfo(ctx context.Context, rc *RelayContext, v *relayValidation, selection *common.ChannelSelection, path string, headers http.Header, attempt int) *common.RelayInfo { return &common.RelayInfo{ Context: ctx, TenantID: rc.TenantID, @@ -589,6 +599,7 @@ func buildRelayInfo(ctx context.Context, rc *RelayContext, v *relayValidation, s ApiKeyID: rc.ApiKeyID, ProjectID: rc.ProjectID, RequestID: rc.RequestID, + RetryIndex: attempt, RelayMode: int(v.relayMode), IsStream: v.isStream, OriginModelName: v.modelName, diff --git a/relay/handler/task_handler.go b/relay/handler/task_handler.go index 73bccf3..9517a99 100644 --- a/relay/handler/task_handler.go +++ b/relay/handler/task_handler.go @@ -15,6 +15,7 @@ import ( "github.com/gogf/gf/v2/frame/g" "github.com/qianfree/team-api/relay/common" "github.com/qianfree/team-api/relay/constant" + "github.com/qianfree/team-api/relay/dto" "github.com/qianfree/team-api/relay/taskchannel" _ "github.com/qianfree/team-api/relay/taskchannel/ali" _ "github.com/qianfree/team-api/relay/taskchannel/gemini" @@ -338,6 +339,15 @@ func HandleTaskFetch( if task.ResultURL != "" { respBody["url"] = task.ResultURL } + // sync_image「同步图片异步化」任务支持多图：把归一化的全部图片作为 data 数组吐出， + // url 仍保留首图（向后兼容）。仅对 sync_image 任务开启——其 Data 为标准 OpenAI + // ImageResponse；视频 / suno / mj 等平台的 Data 结构不同，不能误吐。 + if isSyncImageResultTask(task) && len(task.Data) > 0 { + var img dto.ImageResponse + if json.Unmarshal(task.Data, &img) == nil && len(img.Data) > 0 { + respBody["data"] = img.Data + } + } if task.FailReason != "" { respBody["error"] = task.FailReason } @@ -348,6 +358,22 @@ func HandleTaskFetch( writeJSON(rc.Writer, http.StatusOK, respBody) } +// isSyncImageResultTask 判断任务是否为「同步图片异步化」任务（private_data.task_type == sync_image）。 +// 仅这类任务的 Data 为归一化的 OpenAI ImageResponse，可安全地在 fetch 响应里吐 data 数组； +// 其他平台（视频 / suno / mj）的 Data 结构不同，不在此列，避免误吐内部数据。 +func isSyncImageResultTask(t *common.AsyncTask) bool { + if len(t.PrivateData) == 0 { + return false + } + var pd struct { + TaskType string `json:"task_type"` + } + if json.Unmarshal(t.PrivateData, &pd) != nil { + return false + } + return pd.TaskType == string(constant.TaskPlatformSyncImage) +} + // writeTaskError 写入错误响应 func writeTaskError(w http.ResponseWriter, statusCode int, message, code string) { resp := map[string]any{ diff --git a/relay/scheduler/affinity_test.go b/relay/scheduler/affinity_test.go index e15910e..54fea73 100644 --- a/relay/scheduler/affinity_test.go +++ b/relay/scheduler/affinity_test.go @@ -3,8 +3,6 @@ package scheduler import ( "testing" "time" - - "github.com/qianfree/team-api/relay/constant" ) func TestAffinityStore_SetGet(t *testing.T) { @@ -144,28 +142,3 @@ func TestAffinityStore_CleanExpired(t *testing.T) { t.Errorf("expected 2 remaining entries, got %d", store.Size()) } } - -func TestRetryStrategy_ShouldRetry(t *testing.T) { - tests := []struct { - name string - err error - retryCount int - maxRetries int - shouldRetry bool - }{ - {"nil error", nil, 0, 3, false}, - {"non-retryable", constant.NewRequestError("bad request", nil), 0, 3, false}, - {"retryable first attempt", constant.NewUpstreamError(500, "server error", nil), 0, 3, true}, - {"retryable at max", constant.NewUpstreamError(500, "server error", nil), 3, 3, false}, - {"429 rate limit", constant.NewUpstreamError(429, "rate limit", nil), 1, 3, true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := ShouldRetry(tt.err, tt.retryCount, tt.maxRetries) - if result != tt.shouldRetry { - t.Errorf("ShouldRetry() = %v, want %v", result, tt.shouldRetry) - } - }) - } -} diff --git a/relay/scheduler/retry.go b/relay/scheduler/retry.go deleted file mode 100644 index 0b7a895..0000000 --- a/relay/scheduler/retry.go +++ /dev/null @@ -1,30 +0,0 @@ -package scheduler - -import ( - "github.com/qianfree/team-api/relay/constant" -) - -// MaxRetries 最大重试次数 -const MaxRetries = 3 - -// RetryStrategy 重试策略 -type RetryStrategy struct { - MaxRetries int - ExcludeChannel int64 -} - -// ShouldRetry 判断是否应该重试 -func ShouldRetry(err error, retryCount int, maxRetries int) bool { - if retryCount >= maxRetries { - return false - } - return constant.IsRetryable(err) -} - -// DefaultRetryStrategy 默认重试策略 -func DefaultRetryStrategy(excludeChannelID int64) *RetryStrategy { - return &RetryStrategy{ - MaxRetries: MaxRetries, - ExcludeChannel: excludeChannelID, - } -} diff --git a/web/admin/src/views/ai/ChannelsPage.vue b/web/admin/src/views/ai/ChannelsPage.vue index 2b14f63..6fadfe3 100644 --- a/web/admin/src/views/ai/ChannelsPage.vue +++ b/web/admin/src/views/ai/ChannelsPage.vue @@ -7,6 +7,7 @@ import type { FormInstance } from '@arco-design/web-vue' import PageHeader from '@/components/PageHeader.vue' import request from '@/utils/request' import { useExport } from '@/composables/useExport' +import { hasPermission } from '@/utils/permission' const router = useRouter() const message = Message @@ -106,17 +107,23 @@ const columns: TableColumnData[] = [ render({ record }) { const isActive = record.status === 'active' const action = isActive ? '禁用' : '启用' - return h(Space, { size: 4 }, () => [ - h(Popconfirm, { - content: `确定${action}该渠道？`, - onOk: () => toggleStatus(record), - }, () => h(Button, { - size: 'small', - status: isActive ? 'normal' : 'success', - }, () => action)), + // 破坏性写操作按钮按权限点渲染（super_admin 在 hasPermission 内直接放行）。 + const nodes = [ + hasPermission('channel:edit') + ? h(Popconfirm, { + content: `确定${action}该渠道？`, + onOk: () => toggleStatus(record), + }, () => h(Button, { + size: 'small', + status: isActive ? 'normal' : 'success', + }, () => action)) + : null, h(Button, { size: 'small', type: 'primary', onClick: () => openDetail(record) }, () => '详情'), - h(Popconfirm, { content: '确定删除该渠道？此操作不可撤销。', onOk: () => deleteChannel(record) }, () => h(Button, { size: 'small', status: 'danger' }, () => '删除')), - ]) + hasPermission('channel:delete') + ? h(Popconfirm, { content: '确定删除该渠道？', onOk: () => deleteChannel(record) }, () => h(Button, { size: 'small', status: 'danger' }, () => '删除')) + : null, + ].filter(Boolean) + return h(Space, { size: 4 }, () => nodes) }, }, ] diff --git a/web/admin/src/views/ai/TaskLogsPage.vue b/web/admin/src/views/ai/TaskLogsPage.vue index 2f3d738..188842e 100644 --- a/web/admin/src/views/ai/TaskLogsPage.vue +++ b/web/admin/src/views/ai/TaskLogsPage.vue @@ -319,12 +319,20 @@ onMounted(() => { diff --git a/web/admin/src/views/system/SettingsPage.vue b/web/admin/src/views/system/SettingsPage.vue index 1df6082..3703997 100644 --- a/web/admin/src/views/system/SettingsPage.vue +++ b/web/admin/src/views/system/SettingsPage.vue @@ -1,5 +1,6 @@