feat(logs): add model consistency filter (#675)
This commit is contained in:
1 parent
dad1f05016
commit
a2f0c141b7
8 files changed
+82
-1
No files matched your search
@@ -309,6 +309,7 @@ func requestLogQueryUsesInternalFields(rawQuery string) bool {
|
||||
"channel_id",
|
||||
"credential_id",
|
||||
"upstream_model",
|
||||
"model_consistency",
|
||||
"access_key_id",
|
||||
"attempt_status_code",
|
||||
"failure_category",
|
||||
@@ -347,7 +348,7 @@ func parseRequestLogQuery(rawQuery string) (requestlog.ListQuery, *app_errors.AP
|
||||
}
|
||||
allowed := map[string]struct{}{
|
||||
"from_ms": {}, "to_ms": {}, "group_id": {}, "channel_id": {}, "credential_id": {},
|
||||
"client_model": {}, "upstream_model": {}, "access_key_id": {},
|
||||
"client_model": {}, "upstream_model": {}, "model_consistency": {}, "access_key_id": {},
|
||||
"status": {}, "request_id": {}, "protocol": {}, "stream": {}, "final_status_code": {},
|
||||
"usage_state": {}, "cost_state": {}, "pricing_completeness": {}, "cache_present": {},
|
||||
"attempt_status_code": {}, "failure_category": {}, "error_code": {},
|
||||
@@ -416,6 +417,18 @@ func parseRequestLogQuery(rawQuery string) (requestlog.ListQuery, *app_errors.AP
|
||||
}
|
||||
query.UpstreamModel = value
|
||||
}
|
||||
if value, ok := singleQueryValue(values, "model_consistency"); ok {
|
||||
consistency := telemetry.ModelConsistency(value)
|
||||
switch consistency {
|
||||
case telemetry.ModelConsistencyNotApplicable,
|
||||
telemetry.ModelConsistencyMatch,
|
||||
telemetry.ModelConsistencyUnknown,
|
||||
telemetry.ModelConsistencyMismatch:
|
||||
query.ModelConsistency = consistency
|
||||
default:
|
||||
return requestlog.ListQuery{}, app_errors.ErrValidation
|
||||
}
|
||||
}
|
||||
if value, ok := singleQueryValue(values, "access_key_id"); ok {
|
||||
parsed, apiErr := parseRequestLogID(value)
|
||||
if apiErr != nil {
|
||||
|
||||
@@ -265,6 +265,7 @@ func TestRequestLogEndpointParsesAdvancedFilters(t *testing.T) {
|
||||
"cache_present=true",
|
||||
"channel_id=openai",
|
||||
"credential_id=9",
|
||||
"model_consistency=mismatch",
|
||||
"attempt_status_code=429",
|
||||
"failure_category=rate_limited",
|
||||
"error_code=provider_rate_limit",
|
||||
@@ -297,6 +298,7 @@ func TestRequestLogEndpointParsesAdvancedFilters(t *testing.T) {
|
||||
got.CachePresent == nil || !*got.CachePresent ||
|
||||
got.ChannelID != channel.OpenAI ||
|
||||
got.CredentialID == nil || *got.CredentialID != 9 ||
|
||||
got.ModelConsistency != telemetry.ModelConsistencyMismatch ||
|
||||
got.AttemptStatusCode == nil || *got.AttemptStatusCode != 429 ||
|
||||
got.FailureCategory != telemetry.FailureCategoryRateLimited ||
|
||||
got.AttemptErrorCode != "provider_rate_limit" || got.RetryState != requestlog.RetryStateRetried ||
|
||||
@@ -316,6 +318,25 @@ func TestRequestLogEndpointParsesAdvancedFilters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestLogEndpointAcceptsModelConsistencyMismatchFilter(t *testing.T) {
|
||||
t.Parallel()
|
||||
reader := &recordingRequestLogReader{}
|
||||
recorder := performRequestLogRequest(
|
||||
newRequestLogTestEngine(t, reader),
|
||||
"test-auth-key",
|
||||
"model_consistency=mismatch",
|
||||
)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("response = %d %s, want 200", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if len(reader.queries) != 1 {
|
||||
t.Fatalf("Reader calls = %d, want one", len(reader.queries))
|
||||
}
|
||||
if reader.queries[0].ModelConsistency != telemetry.ModelConsistencyMismatch {
|
||||
t.Fatalf("ModelConsistency = %q, want mismatch", reader.queries[0].ModelConsistency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestLogEndpointRejectsInvalidAdvancedFilters(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []string{
|
||||
@@ -328,6 +349,7 @@ func TestRequestLogEndpointRejectsInvalidAdvancedFilters(t *testing.T) {
|
||||
"cache_present=yes",
|
||||
"channel_id=unknown",
|
||||
"credential_id=0",
|
||||
"model_consistency=invalid",
|
||||
"attempt_status_code=-1",
|
||||
"failure_category=unknown",
|
||||
"error_code=",
|
||||
@@ -1143,6 +1165,7 @@ func TestRequestLogEndpointsBindAccessKeyScopeAndRedactRoutingInternals(t *testi
|
||||
"channel_id=openai",
|
||||
"credential_id=101",
|
||||
"upstream_model=private-upstream-model",
|
||||
"model_consistency=mismatch",
|
||||
"retry_state=retried",
|
||||
} {
|
||||
recorder := performRequestLogRequest(engine, current.Key, query)
|
||||
|
||||
@@ -40,6 +40,9 @@ func (service *Service) List(ctx context.Context, input ListQuery) (Page, error)
|
||||
if input.ClientModel != "" {
|
||||
query = query.Where("client_model = ?", input.ClientModel)
|
||||
}
|
||||
if input.ModelConsistency != "" {
|
||||
query = query.Where("model_consistency = ?", input.ModelConsistency)
|
||||
}
|
||||
if input.AccessKeyID != nil {
|
||||
query = query.Where("access_key_id = ?", *input.AccessKeyID)
|
||||
}
|
||||
|
||||
@@ -467,6 +467,34 @@ func TestServiceListAppliesAllFiltersAndGroupJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceListFiltersModelConsistency(t *testing.T) {
|
||||
db := openRequestLogQueryDB(t)
|
||||
base := time.Date(2026, time.July, 24, 12, 0, 0, 0, time.UTC)
|
||||
mismatch := requestLogQueryRow(
|
||||
"00000000-0000-4000-8000-000000000218", base, 71, "client-model", nil,
|
||||
)
|
||||
mismatch.UpstreamModel = "expected-model"
|
||||
mismatch.UpstreamReportedModel = "returned-model"
|
||||
mismatch.ModelConsistency = string(telemetry.ModelConsistencyMismatch)
|
||||
createRequestLogQueryRow(t, db, mismatch)
|
||||
|
||||
match := requestLogQueryRow(
|
||||
"00000000-0000-4000-8000-000000000219", base.Add(time.Second), 71, "client-model", nil,
|
||||
)
|
||||
createRequestLogQueryRow(t, db, match)
|
||||
|
||||
page, err := newRequestLogTestService(db).List(context.Background(), ListQuery{
|
||||
ModelConsistency: telemetry.ModelConsistencyMismatch,
|
||||
Limit: 50,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("List() error = %v", err)
|
||||
}
|
||||
if got, want := requestIDs(page.Items), []string{mismatch.ID}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("mismatch IDs = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceListGroupFilterUsesAnyAttemptWhileAttributionUsesFinalGroup(t *testing.T) {
|
||||
db := openRequestLogQueryDB(t)
|
||||
event := testEvent("00000000-0000-4000-8000-000000000210")
|
||||
|
||||
@@ -77,6 +77,7 @@ type ListQuery struct {
|
||||
ChannelID channel.ID
|
||||
ClientModel string
|
||||
UpstreamModel string
|
||||
ModelConsistency telemetry.ModelConsistency
|
||||
AccessKeyID *uint
|
||||
Status telemetry.RequestStatus
|
||||
RequestID string
|
||||
|
||||
@@ -12,6 +12,7 @@ export const logFilterNames = [
|
||||
'credential_id',
|
||||
'client_model',
|
||||
'upstream_model',
|
||||
'model_consistency',
|
||||
'access_key_id',
|
||||
'status',
|
||||
'request_id',
|
||||
@@ -46,6 +47,7 @@ export const internalLogFilters: readonly LogFilterName[] = [
|
||||
'channel_id',
|
||||
'credential_id',
|
||||
'upstream_model',
|
||||
'model_consistency',
|
||||
'access_key_id',
|
||||
'attempt_status_code',
|
||||
'failure_category',
|
||||
|
||||
@@ -34,6 +34,7 @@ export const logFilterOptions: Partial<Record<LogFilterName, readonly string[]>>
|
||||
usage_state: ['complete', 'partial', 'missing', 'not_applicable'],
|
||||
cost_state: ['priced', 'unpriced', 'not_applicable'],
|
||||
pricing_completeness: ['complete', 'partial', 'unavailable', 'not_applicable'],
|
||||
model_consistency: ['match', 'mismatch', 'unknown'],
|
||||
retry_state: ['retried', 'not_retried'],
|
||||
failure_category: [
|
||||
'ok',
|
||||
@@ -62,6 +63,13 @@ export const advancedLogFilters: readonly LogFilterDefinition[] = [
|
||||
{ key: 'retry_count_min', section: 'routing', kind: 'number', admin: true },
|
||||
{ key: 'retry_count_max', section: 'routing', kind: 'number', admin: true },
|
||||
{ key: 'final_status_code', section: 'result', kind: 'number' },
|
||||
{
|
||||
key: 'model_consistency',
|
||||
section: 'result',
|
||||
kind: 'select',
|
||||
values: logFilterOptions.model_consistency,
|
||||
admin: true,
|
||||
},
|
||||
{ key: 'attempt_status_code', section: 'result', kind: 'number', admin: true },
|
||||
{
|
||||
key: 'failure_category',
|
||||
|
||||
@@ -153,6 +153,7 @@ export const zhCN = {
|
||||
access_key_id: '访问密钥',
|
||||
client_model: '模型',
|
||||
upstream_model: '上游模型',
|
||||
model_consistency: '返回模型',
|
||||
status: '结果',
|
||||
request_id: '请求 ID',
|
||||
protocol: '协议',
|
||||
@@ -437,6 +438,7 @@ export const enUS: typeof zhCN = {
|
||||
access_key_id: 'Access key',
|
||||
client_model: 'Model',
|
||||
upstream_model: 'Upstream model',
|
||||
model_consistency: 'Returned model',
|
||||
status: 'Outcome',
|
||||
request_id: 'Request ID',
|
||||
protocol: 'Protocol',
|
||||
@@ -719,6 +721,7 @@ export const jaJP: typeof zhCN = {
|
||||
access_key_id: 'アクセスキー',
|
||||
client_model: '要求モデル',
|
||||
upstream_model: '上流モデル',
|
||||
model_consistency: '応答モデル',
|
||||
status: '結果',
|
||||
request_id: 'リクエスト ID',
|
||||
protocol: 'プロトコル',
|
||||
|
||||
Reference in new issue
Block a user