diff --git a/api/admin/admin.go b/api/admin/admin.go index ae6ebff..9dc5151 100644 --- a/api/admin/admin.go +++ b/api/admin/admin.go @@ -25,6 +25,7 @@ type IAdminV1 interface { RequestAuditLogList(ctx context.Context, req *v1.RequestAuditLogListReq) (res *v1.RequestAuditLogListRes, err error) RequestAuditLogDetail(ctx context.Context, req *v1.RequestAuditLogDetailReq) (res *v1.RequestAuditLogDetailRes, err error) OperationLogExport(ctx context.Context, req *v1.OperationLogExportReq) (res *v1.OperationLogExportRes, err error) + ContentFilterLogList(ctx context.Context, req *v1.ContentFilterLogListReq) (res *v1.ContentFilterLogListRes, err error) AdminLogin(ctx context.Context, req *v1.AdminLoginReq) (res *v1.AdminLoginRes, err error) AdminLogout(ctx context.Context, req *v1.AdminLogoutReq) (res *v1.AdminLogoutRes, err error) AdminRefresh(ctx context.Context, req *v1.AdminRefreshReq) (res *v1.AdminRefreshRes, err error) diff --git a/api/admin/v1/audit.go b/api/admin/v1/audit.go index 2781c62..d41737b 100644 --- a/api/admin/v1/audit.go +++ b/api/admin/v1/audit.go @@ -95,3 +95,24 @@ type OperationLogExportReq struct { } type OperationLogExportRes struct{} + +// === 内容过滤拦截日志 === + +type ContentFilterLogListReq struct { + g.Meta `path:"/audit/content-filter-logs" method:"get" mime:"json" tags:"管理后台-审计" summary:"内容过滤拦截日志"` + Page int `json:"page" d:"1"` + PageSize int `json:"page_size" d:"20"` + TenantID int `json:"tenant_id" dc:"租户ID"` + Mode string `json:"mode" dc:"过滤模式：log/replace/block"` + Blocked string `json:"blocked" dc:"是否被拦截：true/false"` + StartDate string `json:"start_date" dc:"开始日期"` + EndDate string `json:"end_date" dc:"结束日期"` + Keyword string `json:"keyword" dc:"关键词搜索（敏感词/路径）"` +} + +type ContentFilterLogListRes struct { + List []map[string]any `json:"list"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} diff --git a/internal/controller/admin/admin_v1_content_filter_log_list.go b/internal/controller/admin/admin_v1_content_filter_log_list.go new file mode 100644 index 0000000..2d6abeb --- /dev/null +++ b/internal/controller/admin/admin_v1_content_filter_log_list.go @@ -0,0 +1,12 @@ +package admin + +import ( + "context" + + v1 "github.com/qianfree/team-api/api/admin/v1" + "github.com/qianfree/team-api/internal/service" +) + +func (c *ControllerV1) ContentFilterLogList(ctx context.Context, req *v1.ContentFilterLogListReq) (res *v1.ContentFilterLogListRes, err error) { + return service.Admin().ContentFilterLogList(ctx, req) +} diff --git a/internal/logic/admin/audit.go b/internal/logic/admin/audit.go index 90bf516..44242a5 100644 --- a/internal/logic/admin/audit.go +++ b/internal/logic/admin/audit.go @@ -457,3 +457,87 @@ func (s *sAdmin) ExportOperationLogs(ctx context.Context, req *v1.OperationLogEx } }) } + +// ContentFilterLogList returns a paginated list of content filter interception logs. +func (s *sAdmin) ContentFilterLogList(ctx context.Context, req *v1.ContentFilterLogListReq) (*v1.ContentFilterLogListRes, error) { + page, pageSize := common.NormalizePagination(req.Page, req.PageSize) + + var conditions []string + var args []any + + if req.TenantID > 0 { + conditions = append(conditions, "l.tenant_id = ?") + args = append(args, int64(req.TenantID)) + } + if req.Mode != "" { + conditions = append(conditions, "l.filter_mode = ?") + args = append(args, req.Mode) + } + if req.Blocked == "true" { + conditions = append(conditions, "l.blocked = true") + } else if req.Blocked == "false" { + conditions = append(conditions, "l.blocked = false") + } + if req.StartDate != "" { + conditions = append(conditions, "l.created_at >= ?") + args = append(args, req.StartDate+" 00:00:00") + } + if req.EndDate != "" { + conditions = append(conditions, "l.created_at <= ?") + args = append(args, req.EndDate+" 23:59:59") + } + if req.Keyword != "" { + conditions = append(conditions, "(l.matched_words::text ILIKE ? OR l.path ILIKE ?)") + kw := "%" + req.Keyword + "%" + args = append(args, kw, kw) + } + + where := "" + if len(conditions) > 0 { + where = " WHERE " + strings.Join(conditions, " AND ") + } + + fields := `l.id, l.tenant_id, l.user_id, l.api_key_id, l.project_id, l.request_id, l.method, l.path, l.client_ip, + l.filter_mode, l.matched_words, l.original_snippet, l.blocked, l.created_at, + t.name AS tenant_name, u.username AS user_name, k.name AS api_key_name, p.name AS project_name` + + // Count + countSQL := "SELECT COUNT(*) AS total FROM aud_content_filter_logs l" + where + countResult, err := g.DB().Ctx(ctx).Query(ctx, countSQL, args...) + if err != nil { + return nil, err + } + total := 0 + if len(countResult) > 0 { + total = countResult[0]["total"].Int() + } + + // Query with JOINs + dataSQL := fmt.Sprintf(`SELECT %s FROM aud_content_filter_logs l + LEFT JOIN tnt_tenants t ON l.tenant_id = t.id + LEFT JOIN tnt_users u ON l.user_id = u.id + LEFT JOIN api_keys k ON l.api_key_id = k.id + LEFT JOIN tnt_projects p ON l.project_id = p.id + %s ORDER BY l.created_at DESC LIMIT %d OFFSET %d`, fields, where, pageSize, (page-1)*pageSize) + + result, err := g.DB().Ctx(ctx).Query(ctx, dataSQL, args...) + if err != nil { + return nil, err + } + + items := make([]map[string]any, 0, len(result)) + for _, row := range result { + m := make(map[string]any, len(row)) + for k, v := range row { + m[k] = v.Val() + } + items = append(items, m) + } + + return &v1.ContentFilterLogListRes{ + List: items, + Total: total, + Page: page, + PageSize: pageSize, + }, nil +} diff --git a/internal/logic/common/content_filter.go b/internal/logic/common/content_filter.go index 09f4a90..fd451c3 100644 --- a/internal/logic/common/content_filter.go +++ b/internal/logic/common/content_filter.go @@ -1,9 +1,11 @@ package common import ( + "bytes" "context" "strings" "sync" + "sync/atomic" "time" "github.com/cloudflare/ahocorasick" @@ -17,15 +19,20 @@ type FilterResult struct { FilteredText string } -// ContentFilterEngine manages the Aho-Corasick automaton for multi-pattern matching. -// It is safe for concurrent use via sync.RWMutex. -type ContentFilterEngine struct { - mu sync.RWMutex +// filterSnapshot is an immutable snapshot of the filter engine state. +// Stored in atomic.Value for lock-free reads in the hot path. +type filterSnapshot struct { + mode string + repl string matcher *ahocorasick.Matcher - literalPatterns []string // patterns fed into the AC matcher (no wildcards) - words []string // original word list (including wildcard patterns) - mode string // off/log/replace/block - repl string // replacement string for "replace" mode + literalPatterns []string + wildcardInners []string // precomputed inner literals from wildcard patterns +} + +// ContentFilterEngine manages the Aho-Corasick automaton for multi-pattern matching. +type ContentFilterEngine struct { + snapshot atomic.Value // stores *filterSnapshot + mu sync.Mutex // protects Rebuild } var ( @@ -36,17 +43,15 @@ var ( // ContentFilter returns the global ContentFilterEngine singleton. func ContentFilter() *ContentFilterEngine { contentFilterEngineOnce.Do(func() { - contentFilterEngine = &ContentFilterEngine{ - mode: "off", - repl: "***", - } + eng := &ContentFilterEngine{} + eng.snapshot.Store(&filterSnapshot{mode: "off", repl: "***"}) + contentFilterEngine = eng }) return contentFilterEngine } // InitContentFilter initializes the content filter engine and starts the // Redis Pub/Sub subscriber for auto-rebuild on settings changes. -// Call this once at startup. func InitContentFilter(ctx context.Context) { engine := ContentFilter() engine.Rebuild(ctx) @@ -64,20 +69,23 @@ func (e *ContentFilterEngine) Rebuild(ctx context.Context) { repl = "***" } - var rawWords []string - if err := cfg.GetOptionJSON(ctx, "content_filter_words", &rawWords); err != nil { + var words []string + if err := cfg.GetOptionJSON(ctx, "content_filter_words", &words); err != nil { g.Log().Warningf(ctx, "[ContentFilter] failed to parse content_filter_words: %v", err) - rawWords = nil + words = nil } - // Wildcard patterns like "*赌博*" cannot be matched directly by AC, - // so we separate them and handle via strings.Contains in Check(). var literalPatterns []string - for _, w := range rawWords { + var wildcardInners []string + for _, w := range words { if w == "" { continue } if strings.Contains(w, "*") { + inner := strings.TrimLeft(strings.TrimRight(w, "*"), "*") + if inner != "" { + wildcardInners = append(wildcardInners, inner) + } continue } literalPatterns = append(literalPatterns, w) @@ -89,66 +97,62 @@ func (e *ContentFilterEngine) Rebuild(ctx context.Context) { } e.mu.Lock() - e.matcher = matcher - e.literalPatterns = literalPatterns - e.words = rawWords - e.mode = mode - e.repl = repl + e.snapshot.Store(&filterSnapshot{ + mode: mode, + repl: repl, + matcher: matcher, + literalPatterns: literalPatterns, + wildcardInners: wildcardInners, + }) e.mu.Unlock() - g.Log().Infof(ctx, "[ContentFilter] rebuilt: mode=%s, words=%d, literals=%d", mode, len(rawWords), len(literalPatterns)) + g.Log().Infof(ctx, "[ContentFilter] rebuilt: mode=%s, words=%d, literals=%d, wildcards=%d", mode, len(words), len(literalPatterns), len(wildcardInners)) } -// Check scans the given text for sensitive words and returns a FilterResult. -func (e *ContentFilterEngine) Check(text string) *FilterResult { - e.mu.RLock() - mode := e.mode - matcher := e.matcher - literalPatterns := e.literalPatterns - words := e.words - repl := e.repl - e.mu.RUnlock() +// Check scans the given body for sensitive words and returns a FilterResult. +// Accepts []byte to avoid string conversion on the hot path. +func (e *ContentFilterEngine) Check(body []byte) *FilterResult { + snap := e.snapshot.Load().(*filterSnapshot) - if mode == "off" || text == "" { - return &FilterResult{ - Matched: false, - MatchedWords: nil, - FilteredText: text, - } - } - - result := &FilterResult{ - Matched: false, - MatchedWords: nil, - FilteredText: text, + if snap.mode == "off" || len(body) == 0 { + return &FilterResult{FilteredText: string(body)} } matchedSet := make(map[string]bool) // Phase 1: Aho-Corasick multi-pattern matching for literal words. - // MatchThreadSafe returns indices into the original dictionary (literalPatterns). - if matcher != nil { - hits := matcher.MatchThreadSafe([]byte(text)) + if snap.matcher != nil { + hits := snap.matcher.MatchThreadSafe(body) for _, idx := range hits { - if idx >= 0 && idx < len(literalPatterns) { - matchedSet[literalPatterns[idx]] = true + if idx >= 0 && idx < len(snap.literalPatterns) { + matchedSet[snap.literalPatterns[idx]] = true } } } - // Phase 2: Wildcard pattern matching via strings.Contains. - // Patterns like "*赌博*" mean "contains 赌博". - for _, w := range words { - if !strings.Contains(w, "*") { - continue + // Phase 2: Wildcard pattern matching via bytes.Contains. + // For non-replace modes, skip if already matched; early-exit on first hit. + if snap.mode != "replace" { + if len(matchedSet) > 0 { + // Already have a match, no need for wildcard phase + } else { + for _, inner := range snap.wildcardInners { + if bytes.Contains(body, []byte(inner)) { + matchedSet[inner] = true + break // one hit is enough for log/block + } + } } - // Strip leading/trailing wildcards to get the inner literal. - inner := strings.TrimLeft(strings.TrimRight(w, "*"), "*") - if inner != "" && strings.Contains(text, inner) { - matchedSet[inner] = true + } else { + // Replace mode: must find ALL matches for complete replacement + for _, inner := range snap.wildcardInners { + if bytes.Contains(body, []byte(inner)) { + matchedSet[inner] = true + } } } + result := &FilterResult{FilteredText: string(body)} if len(matchedSet) == 0 { return result } @@ -158,21 +162,19 @@ func (e *ContentFilterEngine) Check(text string) *FilterResult { result.MatchedWords = append(result.MatchedWords, w) } - // Build filtered text by replacing all matched words. - filtered := text + // Build filtered text (only needed for replace mode, but compute for completeness) + filtered := string(body) for w := range matchedSet { - filtered = strings.ReplaceAll(filtered, w, repl) + filtered = strings.ReplaceAll(filtered, w, snap.repl) } result.FilteredText = filtered return result } -// GetMode returns the current filter mode thread-safely. +// GetMode returns the current filter mode (lock-free via atomic). func (e *ContentFilterEngine) GetMode() string { - e.mu.RLock() - defer e.mu.RUnlock() - return e.mode + return e.snapshot.Load().(*filterSnapshot).mode } // startSubscriber subscribes to Redis Pub/Sub "settings:changed" channel diff --git a/internal/middleware/content_filter.go b/internal/middleware/content_filter.go index 8e25987..87661fd 100644 --- a/internal/middleware/content_filter.go +++ b/internal/middleware/content_filter.go @@ -40,7 +40,7 @@ func ContentFilter(r *ghttp.Request) { return } - result := common.ContentFilter().Check(string(body)) + result := common.ContentFilter().Check(body) if !result.Matched { r.Middleware.Next() return @@ -50,6 +50,9 @@ func ContentFilter(r *ghttp.Request) { r.SetCtxVar(CtxKeyContentFilterMatched, true) r.SetCtxVar(CtxKeyContentFilterWords, result.MatchedWords) + // Asynchronously persist the interception log + insertContentFilterLog(r, mode, result.MatchedWords, string(body)) + switch mode { case "log": g.Log().Infof(r.Context(), "[ContentFilter] matched words: %v", result.MatchedWords) @@ -72,6 +75,46 @@ func ContentFilter(r *ghttp.Request) { } } +// insertContentFilterLog persists a content filter interception record asynchronously. +func insertContentFilterLog(r *ghttp.Request, mode string, matchedWords []string, body string) { + tenantId := r.GetCtxVar(CtxKeyTenantID).Int64() + userId := r.GetCtxVar(CtxKeyUserID).Int64() + apiKeyId := r.GetCtxVar(CtxKeyApiKeyID).Int64() + projectId := r.GetCtxVar(CtxKeyProjectID).Int64() + requestId := r.GetCtxVar("RequestId").String() + method := r.Method + path := r.URL.Path + clientIp := r.GetClientIp() + + // Truncate body snippet for replace mode only + snippet := "" + if mode == "replace" && len(body) > 0 { + const maxSnippet = 500 + if len(body) > maxSnippet { + snippet = body[:maxSnippet] + } else { + snippet = body + } + } + + blocked := mode == "block" + wordsJSON, _ := json.Marshal(matchedWords) + + go func() { + _, err := g.DB().Exec(context.Background(), + `INSERT INTO aud_content_filter_logs + (tenant_id, user_id, api_key_id, project_id, request_id, method, path, client_ip, + filter_mode, matched_words, original_snippet, blocked, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now())`, + tenantId, userId, apiKeyId, projectId, requestId, method, path, clientIp, + mode, string(wordsJSON), snippet, blocked) + if err != nil { + g.Log().Warningf(context.Background(), + "[ContentFilter] failed to insert filter log: %v", err) + } + }() +} + // writeContentFilterError writes an error response in the relay-native format. // /v1/messages uses Claude format, all other paths use OpenAI format. func writeContentFilterError(r *ghttp.Request, statusCode int, errType, message string) { diff --git a/internal/service/admin.go b/internal/service/admin.go index 9678c97..ca37559 100644 --- a/internal/service/admin.go +++ b/internal/service/admin.go @@ -41,6 +41,8 @@ type ( GetRequestAuditLogDetail(ctx context.Context, req *v1.RequestAuditLogDetailReq) (*v1.RequestAuditLogDetailRes, error) // ExportOperationLogs exports operation logs to CSV or Excel. ExportOperationLogs(ctx context.Context, req *v1.OperationLogExportReq) (*v1.OperationLogExportRes, error) + // ContentFilterLogList returns a paginated list of content filter interception logs. + ContentFilterLogList(ctx context.Context, req *v1.ContentFilterLogListReq) (*v1.ContentFilterLogListRes, error) // Login handles admin login. Login(ctx context.Context, req *v1.AdminLoginReq) (*v1.AdminLoginRes, error) // Logout handles admin logout. diff --git a/migrations/000002_tenant_model_pricing_and_task_table_rename.sql b/migrations/000002_v0_1_2_version_changed.sql similarity index 72% rename from migrations/000002_tenant_model_pricing_and_task_table_rename.sql rename to migrations/000002_v0_1_2_version_changed.sql index 33bd714..d341eac 100644 --- a/migrations/000002_tenant_model_pricing_and_task_table_rename.sql +++ b/migrations/000002_v0_1_2_version_changed.sql @@ -1,5 +1,5 @@ -- +goose Up --- 租户模型自定义定价 + 异步任务表重命名 +-- 租户模型自定义定价 + 异步任务表重命名 + 内容过滤日志表 -- 1. 租户模型分配增加自定义缓存定价覆盖字段和阶梯定价支持 ALTER TABLE mdl_tenant_models @@ -50,8 +50,49 @@ COMMENT ON COLUMN tsk_model_tasks.finish_time IS '任务完成（成功或失败 COMMENT ON COLUMN tsk_model_tasks.created_at IS '记录创建时间'; COMMENT ON COLUMN tsk_model_tasks.updated_at IS '记录更新时间'; +-- 3. 创建内容过滤拦截日志表 +CREATE TABLE aud_content_filter_logs ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT, + user_id BIGINT, + api_key_id BIGINT, + project_id BIGINT, + request_id VARCHAR(64), + method VARCHAR(10), + path VARCHAR(500), + client_ip VARCHAR(45), + filter_mode VARCHAR(20) NOT NULL, + matched_words JSONB NOT NULL, + original_snippet TEXT, + blocked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ DEFAULT now() NOT NULL +); + +CREATE INDEX idx_aud_content_filter_logs_created_brin ON aud_content_filter_logs USING brin (created_at); +CREATE INDEX idx_aud_content_filter_logs_tenant ON aud_content_filter_logs USING btree (tenant_id, created_at); +CREATE INDEX idx_aud_content_filter_logs_mode ON aud_content_filter_logs (filter_mode); + +COMMENT ON TABLE aud_content_filter_logs IS '内容过滤拦截日志，记录所有命中敏感词的请求'; +COMMENT ON COLUMN aud_content_filter_logs.id IS '主键ID'; +COMMENT ON COLUMN aud_content_filter_logs.tenant_id IS '租户ID'; +COMMENT ON COLUMN aud_content_filter_logs.user_id IS '用户ID'; +COMMENT ON COLUMN aud_content_filter_logs.api_key_id IS 'API Key ID'; +COMMENT ON COLUMN aud_content_filter_logs.project_id IS '项目ID'; +COMMENT ON COLUMN aud_content_filter_logs.request_id IS '请求唯一ID'; +COMMENT ON COLUMN aud_content_filter_logs.method IS 'HTTP 方法'; +COMMENT ON COLUMN aud_content_filter_logs.path IS '请求路径'; +COMMENT ON COLUMN aud_content_filter_logs.client_ip IS '客户端 IP'; +COMMENT ON COLUMN aud_content_filter_logs.filter_mode IS '过滤模式：log / replace / block'; +COMMENT ON COLUMN aud_content_filter_logs.matched_words IS '命中的敏感词列表（JSONB 数组）'; +COMMENT ON COLUMN aud_content_filter_logs.original_snippet IS '原始请求体片段（截断存储，仅 replace 模式）'; +COMMENT ON COLUMN aud_content_filter_logs.blocked IS '是否被拦截（mode=block 时为 true）'; +COMMENT ON COLUMN aud_content_filter_logs.created_at IS '创建时间'; + -- +goose Down --- 移除注释 +-- 3. 删除内容过滤日志表 +DROP TABLE IF EXISTS aud_content_filter_logs; + +-- 2. 移除注释 COMMENT ON TABLE tsk_model_tasks IS NULL; COMMENT ON COLUMN tsk_model_tasks.id IS NULL; COMMENT ON COLUMN tsk_model_tasks.public_task_id IS NULL; @@ -90,7 +131,7 @@ ALTER TABLE tsk_model_tasks RENAME CONSTRAINT uk_tsk_model_tasks_public_id TO uk -- 恢复表名 ALTER TABLE tsk_model_tasks RENAME TO tsk_async_tasks; --- 移除定价字段 +-- 1. 移除定价字段 ALTER TABLE mdl_tenant_models DROP COLUMN IF EXISTS custom_cache_read_price, DROP COLUMN IF EXISTS custom_cache_creation_price, diff --git a/web/admin/src/router/admin.ts b/web/admin/src/router/admin.ts index aee1047..ddf408d 100644 --- a/web/admin/src/router/admin.ts +++ b/web/admin/src/router/admin.ts @@ -252,6 +252,12 @@ const adminRoutes: RouteRecordRaw[] = [ component: () => import('@/views/monitor/ErrorLogsPage.vue'), meta: { title: '系统错误日志' }, }, + { + path: 'content-filter-logs', + name: 'AdminContentFilterLogs', + component: () => import('@/views/monitor/ContentFilterLogsPage.vue'), + meta: { title: '拦截日志' }, + }, { path: 'cron-jobs', name: 'AdminCronJobs', diff --git a/web/admin/src/views/AdminLayout.vue b/web/admin/src/views/AdminLayout.vue index 13af984..767948e 100644 --- a/web/admin/src/views/AdminLayout.vue +++ b/web/admin/src/views/AdminLayout.vue @@ -214,6 +214,7 @@ const menuGroups = [ { name: 'AdminAlertRules', label: '告警规则', icon: IconNotification }, { name: 'AdminAlertEvents', label: '告警记录', icon: IconFile }, { name: 'AdminErrorLogs', label: '错误日志', icon: IconFile }, + { name: 'AdminContentFilterLogs', label: '拦截日志', icon: IconFile }, { name: 'AdminCronJobs', label: '定时任务', icon: IconFile }, ], }, diff --git a/web/admin/src/views/monitor/ContentFilterLogsPage.vue b/web/admin/src/views/monitor/ContentFilterLogsPage.vue new file mode 100644 index 0000000..f78e185 --- /dev/null +++ b/web/admin/src/views/monitor/ContentFilterLogsPage.vue @@ -0,0 +1,288 @@ + + + + +