feat(logs): add model consistency filter (#675)

This commit is contained in:
tbphp authored and GitHub committed 2026-09-17 17:13:46 +08:00
1 parent dad1f05016
commit a2f0c141b7
8 files changed
+82 -1

No files matched your search

+14 -1
View File
@@ -309,6 +309,7 @@ func requestLogQueryUsesInternalFields(rawQuery string) bool {
"channel_id",
"credential_id",
"upstream_model",
"model_consistency",
"access_key_id",
"attempt_status_code",
"failure_category",
@@ -347,7 +348,7 @@ func parseRequestLogQuery(rawQuery string) (requestlog.ListQuery, *app_errors.AP
}
allowed := map[string]struct{}{
"from_ms": {}, "to_ms": {}, "group_id": {}, "channel_id": {}, "credential_id": {},
"client_model": {}, "upstream_model": {}, "access_key_id": {},
"client_model": {}, "upstream_model": {}, "model_consistency": {}, "access_key_id": {},
"status": {}, "request_id": {}, "protocol": {}, "stream": {}, "final_status_code": {},
"usage_state": {}, "cost_state": {}, "pricing_completeness": {}, "cache_present": {},
"attempt_status_code": {}, "failure_category": {}, "error_code": {},
@@ -416,6 +417,18 @@ func parseRequestLogQuery(rawQuery string) (requestlog.ListQuery, *app_errors.AP
}
query.UpstreamModel = value
}
if value, ok := singleQueryValue(values, "model_consistency"); ok {
consistency := telemetry.ModelConsistency(value)
switch consistency {
case telemetry.ModelConsistencyNotApplicable,
telemetry.ModelConsistencyMatch,
telemetry.ModelConsistencyUnknown,
telemetry.ModelConsistencyMismatch:
query.ModelConsistency = consistency
default:
return requestlog.ListQuery{}, app_errors.ErrValidation
}
}
if value, ok := singleQueryValue(values, "access_key_id"); ok {
parsed, apiErr := parseRequestLogID(value)
if apiErr != nil {
+23
View File
@@ -265,6 +265,7 @@ func TestRequestLogEndpointParsesAdvancedFilters(t *testing.T) {
"cache_present=true",
"channel_id=openai",
"credential_id=9",
"model_consistency=mismatch",
"attempt_status_code=429",
"failure_category=rate_limited",
"error_code=provider_rate_limit",
@@ -297,6 +298,7 @@ func TestRequestLogEndpointParsesAdvancedFilters(t *testing.T) {
got.CachePresent == nil || !*got.CachePresent ||
got.ChannelID != channel.OpenAI ||
got.CredentialID == nil || *got.CredentialID != 9 ||
got.ModelConsistency != telemetry.ModelConsistencyMismatch ||
got.AttemptStatusCode == nil || *got.AttemptStatusCode != 429 ||
got.FailureCategory != telemetry.FailureCategoryRateLimited ||
got.AttemptErrorCode != "provider_rate_limit" || got.RetryState != requestlog.RetryStateRetried ||
@@ -316,6 +318,25 @@ func TestRequestLogEndpointParsesAdvancedFilters(t *testing.T) {
}
}
func TestRequestLogEndpointAcceptsModelConsistencyMismatchFilter(t *testing.T) {
t.Parallel()
reader := &recordingRequestLogReader{}
recorder := performRequestLogRequest(
newRequestLogTestEngine(t, reader),
"test-auth-key",
"model_consistency=mismatch",
)
if recorder.Code != http.StatusOK {
t.Fatalf("response = %d %s, want 200", recorder.Code, recorder.Body.String())
}
if len(reader.queries) != 1 {
t.Fatalf("Reader calls = %d, want one", len(reader.queries))
}
if reader.queries[0].ModelConsistency != telemetry.ModelConsistencyMismatch {
t.Fatalf("ModelConsistency = %q, want mismatch", reader.queries[0].ModelConsistency)
}
}
func TestRequestLogEndpointRejectsInvalidAdvancedFilters(t *testing.T) {
t.Parallel()
tests := []string{
@@ -328,6 +349,7 @@ func TestRequestLogEndpointRejectsInvalidAdvancedFilters(t *testing.T) {
"cache_present=yes",
"channel_id=unknown",
"credential_id=0",
"model_consistency=invalid",
"attempt_status_code=-1",
"failure_category=unknown",
"error_code=",
@@ -1143,6 +1165,7 @@ func TestRequestLogEndpointsBindAccessKeyScopeAndRedactRoutingInternals(t *testi
"channel_id=openai",
"credential_id=101",
"upstream_model=private-upstream-model",
"model_consistency=mismatch",
"retry_state=retried",
} {
recorder := performRequestLogRequest(engine, current.Key, query)
+3
View File
@@ -40,6 +40,9 @@ func (service *Service) List(ctx context.Context, input ListQuery) (Page, error)
if input.ClientModel != "" {
query = query.Where("client_model = ?", input.ClientModel)
}
if input.ModelConsistency != "" {
query = query.Where("model_consistency = ?", input.ModelConsistency)
}
if input.AccessKeyID != nil {
query = query.Where("access_key_id = ?", *input.AccessKeyID)
}
+28
View File
@@ -467,6 +467,34 @@ func TestServiceListAppliesAllFiltersAndGroupJSON(t *testing.T) {
}
}
func TestServiceListFiltersModelConsistency(t *testing.T) {
db := openRequestLogQueryDB(t)
base := time.Date(2026, time.July, 24, 12, 0, 0, 0, time.UTC)
mismatch := requestLogQueryRow(
"00000000-0000-4000-8000-000000000218", base, 71, "client-model", nil,
)
mismatch.UpstreamModel = "expected-model"
mismatch.UpstreamReportedModel = "returned-model"
mismatch.ModelConsistency = string(telemetry.ModelConsistencyMismatch)
createRequestLogQueryRow(t, db, mismatch)
match := requestLogQueryRow(
"00000000-0000-4000-8000-000000000219", base.Add(time.Second), 71, "client-model", nil,
)
createRequestLogQueryRow(t, db, match)
page, err := newRequestLogTestService(db).List(context.Background(), ListQuery{
ModelConsistency: telemetry.ModelConsistencyMismatch,
Limit: 50,
})
if err != nil {
t.Fatalf("List() error = %v", err)
}
if got, want := requestIDs(page.Items), []string{mismatch.ID}; !reflect.DeepEqual(got, want) {
t.Fatalf("mismatch IDs = %v, want %v", got, want)
}
}
func TestServiceListGroupFilterUsesAnyAttemptWhileAttributionUsesFinalGroup(t *testing.T) {
db := openRequestLogQueryDB(t)
event := testEvent("00000000-0000-4000-8000-000000000210")
+1
View File
@@ -77,6 +77,7 @@ type ListQuery struct {
ChannelID channel.ID
ClientModel string
UpstreamModel string
ModelConsistency telemetry.ModelConsistency
AccessKeyID *uint
Status telemetry.RequestStatus
RequestID string
+2
View File
@@ -12,6 +12,7 @@ export const logFilterNames = [
'credential_id',
'client_model',
'upstream_model',
'model_consistency',
'access_key_id',
'status',
'request_id',
@@ -46,6 +47,7 @@ export const internalLogFilters: readonly LogFilterName[] = [
'channel_id',
'credential_id',
'upstream_model',
'model_consistency',
'access_key_id',
'attempt_status_code',
'failure_category',
@@ -34,6 +34,7 @@ export const logFilterOptions: Partial<Record<LogFilterName, readonly string[]>>
usage_state: ['complete', 'partial', 'missing', 'not_applicable'],
cost_state: ['priced', 'unpriced', 'not_applicable'],
pricing_completeness: ['complete', 'partial', 'unavailable', 'not_applicable'],
model_consistency: ['match', 'mismatch', 'unknown'],
retry_state: ['retried', 'not_retried'],
failure_category: [
'ok',
@@ -62,6 +63,13 @@ export const advancedLogFilters: readonly LogFilterDefinition[] = [
{ key: 'retry_count_min', section: 'routing', kind: 'number', admin: true },
{ key: 'retry_count_max', section: 'routing', kind: 'number', admin: true },
{ key: 'final_status_code', section: 'result', kind: 'number' },
{
key: 'model_consistency',
section: 'result',
kind: 'select',
values: logFilterOptions.model_consistency,
admin: true,
},
{ key: 'attempt_status_code', section: 'result', kind: 'number', admin: true },
{
key: 'failure_category',
@@ -153,6 +153,7 @@ export const zhCN = {
access_key_id: '访问密钥',
client_model: '模型',
upstream_model: '上游模型',
model_consistency: '返回模型',
status: '结果',
request_id: '请求 ID',
protocol: '协议',
@@ -437,6 +438,7 @@ export const enUS: typeof zhCN = {
access_key_id: 'Access key',
client_model: 'Model',
upstream_model: 'Upstream model',
model_consistency: 'Returned model',
status: 'Outcome',
request_id: 'Request ID',
protocol: 'Protocol',
@@ -719,6 +721,7 @@ export const jaJP: typeof zhCN = {
access_key_id: 'アクセスキー',
client_model: '要求モデル',
upstream_model: '上流モデル',
model_consistency: '応答モデル',
status: '結果',
request_id: 'リクエスト ID',
protocol: 'プロトコル',