Files
gpt-load/internal/gateway/execution_forward_test.go

1951 lines
72 KiB
Go

package gateway
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
"unsafe"
"gpt-load/internal/dialect"
"gpt-load/internal/execution"
"gpt-load/internal/health"
"gpt-load/internal/platform/contentcoding"
"gpt-load/internal/protocol"
"gpt-load/internal/reasoning"
"gpt-load/internal/state"
"gpt-load/internal/usage"
)
type fakeExecutionExecutor struct {
unary func(context.Context, execution.AttemptSpec) execution.AttemptResult
stream func(context.Context, execution.AttemptSpec, execution.StreamSink) execution.StreamResult
}
type observingUsageDialect struct {
expectedBody []byte
matchedBody bool
}
type observingUsageStreamExtractor struct {
expectedBody []byte
matchedBody bool
}
func (*observingUsageDialect) Protocol() protocol.Protocol {
return protocol.OpenAIImages
}
func (*observingUsageDialect) InspectRequest(
*dialect.ParsedRequest,
) (dialect.RequestMetadata, error) {
return dialect.RequestMetadata{}, nil
}
func (d *observingUsageDialect) ExtractUsage(
body []byte,
) (usage.Result, error) {
d.matchedBody = len(body) == len(d.expectedBody) &&
len(body) > 0 && &body[0] == &d.expectedBody[0]
return usage.Result{State: usage.StateComplete}, nil
}
func (*observingUsageDialect) NewUsageStreamExtractor() dialect.UsageStreamExtractor {
return nil
}
func (e *observingUsageStreamExtractor) Observe(body []byte) error {
e.matchedBody = len(body) == len(e.expectedBody) &&
len(body) > 0 && &body[0] == &e.expectedBody[0]
return nil
}
func (*observingUsageStreamExtractor) Finalize() (usage.Result, bool) {
return usage.Result{State: usage.StateComplete}, true
}
func (value fakeExecutionExecutor) Execute(
ctx context.Context,
spec execution.AttemptSpec,
) execution.AttemptResult {
return value.unary(ctx, spec)
}
func (value fakeExecutionExecutor) ExecuteStream(
ctx context.Context,
spec execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
return value.stream(ctx, spec, sink)
}
func TestExecutionForwarderBuildsFrozenAttemptAndMapsUnaryResult(t *testing.T) {
t.Parallel()
wantUsage := usage.Result{State: usage.StateComplete, Tokens: usage.Tokens{UncachedInput: 7, Output: 3}}
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
spec execution.AttemptSpec,
) execution.AttemptResult {
if spec.RequestID != "request-1" || spec.AttemptID != "attempt-1" || spec.Sequence != 2 ||
spec.ChannelID != "openai" ||
spec.RouteMode != execution.RouteNative ||
spec.RouteRequirement != execution.RouteRequirementNative ||
spec.ClientProtocol != protocol.OpenAICompletions ||
spec.Operation != execution.OperationChatCompletion || spec.ClientModel != "public" ||
spec.UpstreamModel != "upstream" || spec.Method != http.MethodPost ||
spec.Path != "/v1/chat/completions" || spec.RawQuery != "trace=kept" ||
string(spec.Body) != `{"model":"public"}` || spec.Credential.ID != 7 {
t.Fatalf("attempt spec = %#v", spec)
}
if spec.Header.Get("Authorization") != "" || spec.Header.Get("X-Remove") != "" ||
spec.Header.Get("X-Template") != "Bearer secret" || spec.Header.Get("X-Test") != "kept" {
t.Fatalf("attempt headers = %#v", spec.Header)
}
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent,
ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"X-Request-Id": {"upstream-1"}},
Body: []byte(`{"ok":true}`),
Model: "upstream",
UpstreamRequestID: "upstream-1",
Usage: &execution.UsageEvidence{Normalized: wantUsage},
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), executionForwardInput())
if result.Err != nil || result.StatusCode != http.StatusOK ||
string(result.Body) != `{"ok":true}` || !reflect.DeepEqual(result.Usage, wantUsage) ||
result.DispatchState != execution.DispatchMaybeSent || !result.ResponseStarted ||
result.UpstreamRequestID != "upstream-1" || result.UpstreamReportedModel != "upstream" {
t.Fatalf("Forward() = %#v", result)
}
}
func TestExecutionForwarderNormalizesDowngradedResponsesStoreUnary(t *testing.T) {
for _, test := range []struct {
name string
request string
response string
}{
{
name: "store true",
request: `{"model":"public","input":"hello","store":true}`,
response: `{"id":"resp_1","object":"response","store":true,"output":[]}`,
},
{
name: "store omitted",
request: `{"model":"public","input":"hello"}`,
response: `{"id":"resp_1","object":"response","output":[]}`,
},
} {
t.Run(test.name, func(t *testing.T) {
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.Request.Body = []byte(test.request)
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
spec execution.AttemptSpec,
) execution.AttemptResult {
var request map[string]json.RawMessage
if err := json.Unmarshal(spec.Body, &request); err != nil || string(request["store"]) != "false" {
t.Fatalf("upstream request = %s, err=%v", spec.Body, err)
}
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}},
Body: []byte(test.response),
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
var response map[string]json.RawMessage
if result.Err != nil || json.Unmarshal(result.Body, &response) != nil ||
string(response["store"]) != "false" {
t.Fatalf("Forward() = %#v, body=%s", result, result.Body)
}
if string(input.Request.Body) != test.request {
t.Fatalf("input request mutated = %s", input.Request.Body)
}
})
}
}
func TestExecutionForwarderPreservesUnrecognizedDowngradedResponsesSuccess(t *testing.T) {
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.ExternalModel = "same-model"
input.UpstreamModelID = "same-model"
input.Request.Body = []byte(`{"model":"same-model","input":"hello","store":true}`)
want := []byte(`not-json`)
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
_ execution.AttemptSpec,
) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/octet-stream"}},
Body: want,
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
if result.Err != nil || !bytes.Equal(result.Body, want) {
t.Fatalf("Forward() = %#v, want unchanged body %q", result, want)
}
}
func TestExecutionForwarderRejectsDowngradedResponsesStoreEncodingFailure(t *testing.T) {
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.ExternalModel = "same-model"
input.UpstreamModelID = "same-model"
input.Request.Body = []byte(`{"model":"same-model","input":"hello","store":true}`)
response := []byte(
`{"` + strings.Repeat("\u2028", maxResponsesStoreRewriteGrowthBytes/3+1) +
`":0,"store":true}`,
)
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
_ execution.AttemptSpec,
) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}},
Body: response,
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
if !errors.Is(result.Err, ErrUpstreamProtocol) || result.ExecutionError == nil ||
result.ExecutionError.Code != "response_representation_invalid" || len(result.Body) != 0 {
t.Fatalf("Forward() = %#v, body=%s", result, result.Body)
}
}
func TestExecutionForwarderCapturesImagesUsageFromUnaryBody(t *testing.T) {
t.Parallel()
const responseBody = `{"created":123,"model":"upstream-image","data":[{"b64_json":"AA=="}],"usage":{"input_tokens":100,"input_tokens_details":{"text_tokens":4,"image_tokens":96},"output_tokens":30,"total_tokens":130}}`
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
_ execution.AttemptSpec,
) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent,
ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"application/json"}},
Body: []byte(responseBody),
Model: "upstream-image",
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIImages()
input.ClientProtocol = protocol.OpenAIImages
input.ObserveUsage = true
input.Operation = execution.OperationImagesGenerate
input.RouteRequirement = execution.RouteRequirementNative
input.Request.Path = "/v1/images/generations"
input.Request.Body = []byte(`{"model":"public-image","prompt":"draw"}`)
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
want := usage.Result{
State: usage.StateComplete,
Tokens: usage.Tokens{UncachedInput: 100, Output: 30},
}
if result.Err != nil || result.Usage != want {
t.Fatalf("Forward() usage = %#v, want %#v; result=%#v", result.Usage, want, result)
}
}
func TestExecutionForwarderCapturesEmbeddingsUsageFromUnaryBody(t *testing.T) {
t.Parallel()
responseBody := []byte(`{"object":"list","model":"upstream-embedding","data":[{"object":"embedding","index":0,"embedding":[0.1,0.2]}],"usage":{"prompt_tokens":7,"total_tokens":7}}`)
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
_ execution.AttemptSpec,
) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent,
ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"application/json"}},
Body: responseBody,
Model: "upstream-embedding",
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIEmbeddings()
input.ClientProtocol = protocol.OpenAIEmbeddings
input.ObserveUsage = true
input.Operation = execution.OperationEmbeddingsCreate
input.RouteRequirement = execution.RouteRequirementNative
input.ExternalModel = "upstream-embedding"
input.UpstreamModelID = "upstream-embedding"
input.Request.Path = "/v1/embeddings"
input.Request.Body = []byte(`{"model":"upstream-embedding","input":"hello"}`)
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
want := usage.Result{
State: usage.StateComplete,
Tokens: usage.Tokens{UncachedInput: 7},
}
if result.Err != nil || result.Usage != want || result.ClassificationBody != nil ||
len(result.Body) == 0 || unsafe.SliceData(result.Body) != unsafe.SliceData(responseBody) {
t.Fatalf("Forward() usage = %#v, want %#v; result=%#v", result.Usage, want, result)
}
}
func TestExecutionForwarderCapturesCompressedImagesUsageAfterDecoding(t *testing.T) {
t.Parallel()
plain := []byte(`{"created":123,"model":"upstream-image","data":[{"b64_json":"AA=="}],"usage":{"input_tokens":100,"input_tokens_details":{"text_tokens":4,"image_tokens":96},"output_tokens":30,"total_tokens":130}}`)
for _, encoding := range []contentcoding.Encoding{
contentcoding.Gzip,
contentcoding.Brotli,
} {
t.Run(string(encoding), func(t *testing.T) {
compressed := encodeContentCodingForGatewayTest(t, encoding, plain)
executor := fakeExecutionExecutor{unary: func(
_ context.Context,
_ execution.AttemptSpec,
) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent,
ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{
"Content-Type": {"application/json"},
"Content-Encoding": {string(encoding)},
},
Body: compressed,
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIImages()
input.ClientProtocol = protocol.OpenAIImages
input.ObserveUsage = true
input.Operation = execution.OperationImagesGenerate
input.RouteRequirement = execution.RouteRequirementNative
input.ExternalModel = "upstream-image"
input.UpstreamModelID = "upstream-image"
input.Request.Path = "/v1/images/generations"
input.Request.Body = []byte(`{"model":"upstream-image","prompt":"draw"}`)
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
want := usage.Result{
State: usage.StateComplete,
Tokens: usage.Tokens{UncachedInput: 100, Output: 30},
}
if result.Err != nil || result.Usage != want ||
!bytes.Equal(result.Body, plain) ||
result.Header.Get("Content-Encoding") != "" {
t.Fatalf("Forward() = %#v", result)
}
})
}
}
func TestUsageCaptureNonStreamingPlainDoesNotCloneBody(t *testing.T) {
t.Parallel()
body := []byte(`{"usage":{"input_tokens":1,"output_tokens":1}}`)
selected := &observingUsageDialect{expectedBody: body}
result := newUsageCaptureBoundary().extractNonStreamingPlain(selected, body)
if result.State != usage.StateComplete || !selected.matchedBody {
t.Fatal("extractNonStreamingPlain() did not borrow the original body")
}
}
func TestUsageCaptureStreamEventDoesNotCloneBody(t *testing.T) {
t.Parallel()
body := []byte(`{"type":"image_generation.partial_image","b64_json":"AA=="}`)
extractor := &observingUsageStreamExtractor{expectedBody: body}
capture := &streamUsageCapture{
boundary: newUsageCaptureBoundary(),
protocol: protocol.OpenAIImages,
extractor: extractor,
active: true,
}
capture.observeEvent(dialect.StreamEvent{Payload: body})
if !capture.active || !extractor.matchedBody {
t.Fatal("observeEvent() did not borrow the original body")
}
}
func TestExecutionForwarderKeepsHTTPFailureAsUncommittedResponse(t *testing.T) {
t.Parallel()
evidence := execution.ErrorEvidence{
Kind: execution.ErrorKindHTTP, StatusCode: http.StatusTooManyRequests,
Summary: "upstream rejected request", RetryAfter: 3 * time.Second,
}
executor := fakeExecutionExecutor{unary: func(context.Context, execution.AttemptSpec) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusTooManyRequests,
Header: http.Header{"Retry-After": {"3"}},
Body: []byte(`{"error":{"type":"rate_limit"}}`),
Error: &evidence,
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), executionForwardInput())
if result.Err != nil || !result.HasResponse() || result.ExecutionError == nil ||
result.ExecutionError.Kind != execution.ErrorKindHTTP || result.ErrorSummary != evidence.Summary {
t.Fatalf("Forward() = %#v", result)
}
result.ExecutionError.Summary = "changed"
if evidence.Summary != "upstream rejected request" {
t.Fatal("Forward() retained executor-owned error evidence")
}
}
func TestExecutionForwarderPreservesNotSentRefreshFailure(t *testing.T) {
t.Parallel()
evidence := execution.ErrorEvidence{
Kind: execution.ErrorKindHTTP, Hint: execution.FailureHintRefreshUnavailable,
OriginHint: execution.ErrorOriginUpstream, ScopeHint: execution.ErrorScopeCredential,
Code: "refresh_temporarily_unavailable", Summary: "credential refresh is unavailable",
RetryAfter: 30 * time.Minute, ReplaySafety: execution.ReplaySafetyRejectedBeforeProcessing,
}
executor := fakeExecutionExecutor{unary: func(context.Context, execution.AttemptSpec) execution.AttemptResult {
return execution.AttemptResult{DispatchState: execution.DispatchNotSent, Error: &evidence}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), executionForwardInput())
if result.DispatchState != execution.DispatchNotSent || result.ExecutionError == nil ||
result.ExecutionError.Code != "refresh_temporarily_unavailable" ||
result.ExecutionError.RetryAfter != 30*time.Minute {
t.Fatalf("Forward() = %#v", result)
}
now := time.Date(2026, time.August, 30, 8, 0, 0, 0, time.UTC)
decision := judgeUpstreamResult(
result,
now,
health.DecisionContext{DefaultRateLimitCooldown: 10 * time.Minute},
)
if decision.Category != health.FailureCategoryUpstreamHostError ||
decision.Retry != health.RetryNextCandidate ||
decision.Effect != health.EffectCooldownCredential ||
!decision.CooldownUntil.Equal(now.Add(30*time.Minute)) {
t.Fatalf("health decision = %#v", decision)
}
}
func TestExecutionForwarderRejectsInvalidUnaryTerminalContract(t *testing.T) {
t.Parallel()
executor := fakeExecutionExecutor{unary: func(context.Context, execution.AttemptSpec) execution.AttemptResult {
return execution.AttemptResult{
Body: []byte("private invalid terminal body"),
Error: &execution.ErrorEvidence{
Kind: execution.ErrorKindHTTP, Summary: "private invalid terminal summary",
},
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), executionForwardInput())
if result.DispatchState != execution.DispatchMaybeSent || result.ExecutionError == nil ||
result.ExecutionError.Kind != execution.ErrorKindInternal ||
result.ExecutionError.Code != "attempt_result_contract_invalid" ||
result.ExecutionError.Summary != "Attempt executor returned an invalid result." ||
result.Body != nil || result.ClassificationBody != nil {
t.Fatalf("Forward() = %#v", result)
}
for _, value := range []string{result.ErrorSummary, result.ExecutionError.Summary, fmt.Sprint(result.Err)} {
if strings.Contains(value, "private") {
t.Fatalf("invalid contract leaked private detail: %q", value)
}
}
}
func TestExecutionForwarderRejectsInvalidStreamTerminalContract(t *testing.T) {
t.Parallel()
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData,
Data: []byte("data: {\"id\":\"response\",\"choices\":[]}\n\n"),
}); err != nil {
t.Fatalf("data sink: %v", err)
}
return execution.StreamResult{
Error: &execution.ErrorEvidence{
Kind: execution.ErrorKindProvider, Summary: "private invalid stream terminal",
},
}
}}
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), executionForwardInput(), httptest.NewRecorder(),
)
if !result.Committed || result.ExecutionError == nil ||
result.ExecutionError.Kind != execution.ErrorKindInternal ||
result.ExecutionError.Code != "attempt_result_contract_invalid" ||
result.Stream.EndReason != StreamEndUpstreamProtocolError {
t.Fatalf("ForwardStream() = %#v", result)
}
for _, value := range []string{result.ErrorSummary, result.ExecutionError.Summary, fmt.Sprint(result.Err)} {
if strings.Contains(value, "private") {
t.Fatalf("invalid stream contract leaked private detail: %q", value)
}
}
}
func TestExecutionForwarderKeepsExecutionObservationOnRepresentationFailure(t *testing.T) {
t.Parallel()
budget := int64(4096)
executor := fakeExecutionExecutor{unary: func(context.Context, execution.AttemptSpec) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent,
ResponseStarted: true,
UpstreamProtocol: protocol.Anthropic,
AppliedReasoning: &reasoning.Config{Mode: "enabled", BudgetTokens: &budget},
StatusCode: http.StatusOK,
Header: http.Header{"Content-Encoding": {"unsupported"}},
Body: []byte(`{"ok":true}`),
UpstreamRequestID: "upstream-observation",
}
}}
result := NewExecutionForwarder(executor).Forward(context.Background(), executionForwardInput())
if !errors.Is(result.Err, ErrUpstreamProtocol) ||
result.DispatchState != execution.DispatchMaybeSent || !result.ResponseStarted ||
result.UpstreamProtocol != protocol.Anthropic ||
result.UpstreamRequestID != "upstream-observation" ||
result.AppliedReasoning.Mode != "enabled" || result.AppliedReasoning.BudgetTokens == nil ||
*result.AppliedReasoning.BudgetTokens != budget || result.ExecutionError == nil ||
result.ExecutionError.Kind != execution.ErrorKindInternal ||
result.ExecutionError.Code != "response_representation_invalid" {
t.Fatalf("Forward() representation failure = %#v", result)
}
}
func TestExecutionForwarderProjectsConvertedErrorsToClientProtocol(t *testing.T) {
t.Parallel()
tests := []struct {
name string
protocol protocol.Protocol
dialect dialect.Dialect
contains []string
}{
{name: "OpenAI", protocol: protocol.OpenAIResponses, dialect: dialect.NewOpenAI(), contains: []string{`"error"`, `"type":"rate_limit_error"`}},
{name: "Anthropic", protocol: protocol.Anthropic, dialect: dialect.NewAnthropic(), contains: []string{`"type":"error"`, `"type":"rate_limit_error"`}},
{name: "Gemini", protocol: protocol.Gemini, dialect: dialect.NewGemini(), contains: []string{`"code":429`, `"status":"RESOURCE_EXHAUSTED"`}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
executor := fakeExecutionExecutor{unary: func(context.Context, execution.AttemptSpec) execution.AttemptResult {
return execution.AttemptResult{
DispatchState: execution.DispatchMaybeSent,
ResponseStarted: true,
StatusCode: http.StatusTooManyRequests,
Header: http.Header{"Content-Type": {"application/json"}},
Body: []byte(`{"error":{"message":"provider-shaped"}}`),
Error: &execution.ErrorEvidence{
Kind: execution.ErrorKindHTTP, Hint: execution.FailureHintRateLimited,
StatusCode: http.StatusTooManyRequests, Type: "rate_limit_error",
Code: "RESOURCE_EXHAUSTED", Summary: "upstream returned HTTP 429",
},
}
}}
input := executionForwardInput()
input.ClientProtocol = test.protocol
input.Dialect = test.dialect
input.RouteMode = execution.RouteConverted
result := NewExecutionForwarder(executor).Forward(context.Background(), input)
for _, want := range test.contains {
if !strings.Contains(string(result.Body), want) {
t.Fatalf("converted %s error body = %s, want %q", test.name, result.Body, want)
}
}
if result.Header.Get("Content-Type") != "application/json" {
t.Fatalf("content type = %q", result.Header.Get("Content-Type"))
}
})
}
}
func TestExecutionForwarderCommitsSuccessfulStreamOnlyOnFirstData(t *testing.T) {
t.Parallel()
readyObserved := false
firstResponseObserved := false
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}, "X-Request-Id": {"stream-1"}},
UpstreamRequestID: "stream-1",
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if readyObserved || firstResponseObserved {
t.Fatal("ready metadata committed downstream before first data")
}
if err := sink(execution.StreamEvent{Sequence: 2, Kind: execution.StreamEventData, Data: []byte("data: {\"id\":\"chat_1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"one\"},\"finish_reason\":null}]}\n\n")}); err != nil {
t.Fatalf("data sink: %v", err)
}
if !readyObserved || !firstResponseObserved {
t.Fatal("first data did not cross commit boundary")
}
if err := sink(execution.StreamEvent{Sequence: 3, Kind: execution.StreamEventData, Data: []byte("data: [DONE]\n\n")}); err != nil {
t.Fatalf("done sink: %v", err)
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
UpstreamRequestID: "stream-1",
}
}}
input := executionForwardInput()
input.OnStreamReady = func() { readyObserved = true }
input.OnFirstResponse = func() { firstResponseObserved = true }
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(context.Background(), input, recorder)
if result.Err != nil || !result.Committed || result.Stream.EndReason != StreamEndCleanEOF ||
recorder.Code != http.StatusOK || recorder.Body.String() != "data: {\"id\":\"chat_1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"one\"},\"finish_reason\":null}]}\n\ndata: [DONE]\n\n" {
t.Fatalf("ForwardStream() result=%#v response=%d %q", result, recorder.Code, recorder.Body.String())
}
}
func TestExecutionForwarderAllowsImagesEventAboveDefaultLimit(t *testing.T) {
partial := "event: image_generation.partial_image\n" +
"data: {\"type\":\"image_generation.partial_image\",\"b64_json\":\"" +
strings.Repeat("A", maxSSEEventBytes+1) + "\"}\n\n"
completed := "event: image_generation.completed\n" +
"data: {\"type\":\"image_generation.completed\",\"b64_json\":\"AA==\",\"usage\":{\"input_tokens\":100,\"input_tokens_details\":{\"text_tokens\":4,\"image_tokens\":96},\"output_tokens\":30,\"total_tokens\":130}}\n\n"
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
for _, event := range []execution.StreamEvent{
{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
},
{Sequence: 2, Kind: execution.StreamEventData, Data: []byte(partial)},
{Sequence: 3, Kind: execution.StreamEventData, Data: []byte(completed)},
} {
if err := sink(event); err != nil {
t.Fatalf("stream sink: %v", err)
}
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIImages()
input.ClientProtocol = protocol.OpenAIImages
input.ObserveUsage = true
input.Operation = execution.OperationImagesGenerate
input.Request.Path = "/v1/images/generations"
input.Request.RawQuery = ""
input.Request.Body = []byte(`{"model":"public","stream":true}`)
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(context.Background(), input, recorder)
if result.Err != nil || !result.Committed || result.Stream.EndReason != StreamEndCleanEOF ||
result.Usage.State != usage.StateComplete ||
result.Usage.Tokens != (usage.Tokens{UncachedInput: 100, Output: 30}) {
t.Fatalf("ForwardStream() = %#v", result)
}
if !bytes.Equal(recorder.Body.Bytes(), []byte(partial+completed)) {
t.Fatalf("stream body length = %d, want %d", recorder.Body.Len(), len(partial)+len(completed))
}
}
func TestExecutionForwarderChecksImagesStreamCredentialsBeforeForwarding(t *testing.T) {
const (
partial = "event: image_generation.partial_image\n" +
"data: {\"type\":\"image_generation.partial_image\",\"b64_json\":\"AAAA\",\"A\":1}\n\n"
completedStart = "event: image_generation.completed\n" +
"data: {\"type\":\"image_generation.completed\",\"b64_json\":\"AA"
completedEnd = "AA\",\"A\":1}\n\n"
completed = completedStart + completedEnd
apiKeyLeakStart = "event: image_generation.completed\n" +
"data: {\"type\":\"image_generation.completed\",\"revised_prompt\":\"A"
credentialLeakStart = "event: image_generation.completed\n" +
"data: {\"type\":\"image_generation.completed\",\"metadata\":{\"token\":\"oauth-secret"
leakEnd = "\"}}\n\n"
)
tests := []struct {
name string
chunks []string
wantSinkError bool
wantCommitted bool
wantEndReason StreamEndReason
wantBodies []string
}{
{
name: "split media and structure collisions remain valid",
chunks: []string{completedStart, completedEnd},
wantCommitted: true,
wantEndReason: StreamEndCleanEOF,
wantBodies: []string{"", completed},
},
{
name: "split credential leak before commit fails closed",
chunks: []string{credentialLeakStart, leakEnd},
wantSinkError: true,
wantBodies: []string{"", ""},
},
{
name: "split credential leak after commit drops offending event",
chunks: []string{partial, apiKeyLeakStart, "\"}\n\n"},
wantSinkError: true,
wantCommitted: true,
wantEndReason: StreamEndUpstreamProtocolError,
wantBodies: []string{partial, partial, partial},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var sinkErr error
recorder := httptest.NewRecorder()
bodies := make([]string, 0, len(test.chunks))
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
for index, chunk := range test.chunks {
sinkErr = sink(execution.StreamEvent{
Sequence: uint64(index + 2), Kind: execution.StreamEventData, Data: []byte(chunk),
})
bodies = append(bodies, recorder.Body.String())
if sinkErr != nil {
break
}
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIImages()
input.ClientProtocol = protocol.OpenAIImages
input.Operation = execution.OperationImagesGenerate
input.Request.Path = "/v1/images/generations"
input.Request.RawQuery = ""
input.Request.Body = []byte(`{"model":"public","stream":true}`)
input.APIKey = "A"
input.CredentialSecrets = []string{"oauth-secret"}
result := NewExecutionForwarder(executor).ForwardStream(context.Background(), input, recorder)
if test.wantSinkError {
if !errors.Is(sinkErr, ErrUpstreamProtocol) || !errors.Is(result.Err, ErrUpstreamProtocol) {
t.Fatalf("sink/result errors = %v / %v, want upstream protocol errors", sinkErr, result.Err)
}
} else if sinkErr != nil || result.Err != nil {
t.Fatalf("sink/result errors = %v / %v", sinkErr, result.Err)
}
if result.Committed != test.wantCommitted {
t.Fatalf("Committed = %t, want %t", result.Committed, test.wantCommitted)
}
if test.wantCommitted && result.Stream.EndReason != test.wantEndReason {
t.Fatalf("EndReason = %q, want %q", result.Stream.EndReason, test.wantEndReason)
}
if !reflect.DeepEqual(bodies, test.wantBodies) {
t.Fatalf("response bodies = %#v, want %#v", bodies, test.wantBodies)
}
})
}
}
func TestExecutionForwarderClassifiesImagesCompletedErrorObject(t *testing.T) {
tests := []struct {
name string
frames []string
wantEndReason StreamEndReason
}{
{
name: "empty error completes",
frames: []string{
"event: image_generation.completed\n" +
"data: {\"type\":\"image_generation.completed\",\"error\":{}}\n\n",
},
wantEndReason: StreamEndCleanEOF,
},
{
name: "non-empty error fails",
frames: []string{
"event: image_generation.partial_image\n" +
"data: {\"type\":\"image_generation.partial_image\",\"b64_json\":\"AA==\"}\n\n",
"event: image_generation.completed\n" +
"data: {\"type\":\"image_generation.completed\",\"error\":{\"message\":\"failed\"}}\n\n",
},
wantEndReason: StreamEndSSEError,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
for index, frame := range test.frames {
if err := sink(execution.StreamEvent{
Sequence: uint64(index + 2), Kind: execution.StreamEventData, Data: []byte(frame),
}); err != nil {
t.Fatalf("data sink: %v", err)
}
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIImages()
input.ClientProtocol = protocol.OpenAIImages
input.Operation = execution.OperationImagesGenerate
input.Request.Path = "/v1/images/generations"
input.Request.RawQuery = ""
input.Request.Body = []byte(`{"model":"public","stream":true}`)
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(context.Background(), input, recorder)
if result.Err != nil || !result.Committed || result.Stream.EndReason != test.wantEndReason {
t.Fatalf("ForwardStream() = %#v", result)
}
if recorder.Body.String() != strings.Join(test.frames, "") {
t.Fatalf("response body = %q", recorder.Body.String())
}
})
}
}
func TestExecutionForwarderIgnoresOpenAIDoneForUsageCapture(t *testing.T) {
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
for _, event := range []execution.StreamEvent{
{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
},
{
Sequence: 2, Kind: execution.StreamEventData,
Data: []byte("data: {\"choices\":[],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":6}}\n\n"),
},
{
Sequence: 3, Kind: execution.StreamEventData,
Data: []byte("data: [DONE]\n\n"),
},
} {
if err := sink(event); err != nil {
t.Fatalf("stream sink: %v", err)
}
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
forwarder := NewExecutionForwarder(executor)
input := executionForwardInput()
input.ObserveUsage = true
result := forwarder.ForwardStream(context.Background(), input, httptest.NewRecorder())
if result.Err != nil || result.Usage.State != usage.StateComplete ||
result.Usage.Tokens != (usage.Tokens{UncachedInput: 100, Output: 6}) ||
result.Usage.Diagnostics.Has(usage.DiagnosticInvalidPayload) {
t.Fatalf("ForwardStream() result = %#v", result)
}
if failures := forwarder.usageCapture.failureTotal.Load(); failures != 0 {
t.Fatalf("usage capture failures = %d, want 0", failures)
}
}
func TestExecutionForwarderClassifiesOpenAIResponsesStreamLifecycle(t *testing.T) {
t.Parallel()
const (
deltaEvent = "event: response.output_text.delta\n" +
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"hi\"}\n\n"
completedEvent = "event: response.completed\n" +
"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{\"input_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":3},\"output_tokens\":2,\"total_tokens\":10}}}\n\n"
incompleteEvent = "event: response.incomplete\n" +
"data: {\"type\":\"response.incomplete\",\"response\":{\"usage\":{\"input_tokens\":8,\"input_tokens_details\":{\"cached_tokens\":3},\"output_tokens\":2,\"total_tokens\":10}}}\n\n"
failedEvent = "event: response.failed\n" +
"data: {\"type\":\"response.failed\",\"error\":{\"message\":\"capacity exhausted\"},\"response\":{}}\n\n"
)
completedCRLF := strings.ReplaceAll(completedEvent, "\n", "\r\n")
completedCRLFPrefix := completedCRLF[:len(completedCRLF)-1]
tests := []struct {
name string
chunks []string
cancel bool
terminalError *execution.ErrorEvidence
wantEnd StreamEndReason
wantSummary string
wantUsageState usage.State
wantTokens usage.Tokens
providerError bool
}{
{
name: "completed terminal remains successful after client cancellation",
chunks: []string{
deltaEvent,
completedEvent[:len(completedEvent)-7],
completedEvent[len(completedEvent)-7:],
},
cancel: true,
terminalError: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "request context canceled",
},
wantEnd: StreamEndCleanEOF,
wantUsageState: usage.StateComplete,
wantTokens: usage.Tokens{
UncachedInput: 5, CacheRead: 3, Output: 2,
},
},
{
name: "split terminal CRLF remains canceled before final line feed is forwarded",
chunks: []string{completedCRLFPrefix},
cancel: true,
terminalError: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "request context canceled",
},
wantEnd: StreamEndClientCanceled,
wantSummary: fixedErrorSummary("client_canceled"),
wantUsageState: usage.StateComplete,
wantTokens: usage.Tokens{
UncachedInput: 5, CacheRead: 3, Output: 2,
},
},
{
name: "split terminal CRLF succeeds after final line feed is forwarded",
chunks: []string{completedCRLFPrefix, "\n"},
cancel: true,
terminalError: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "request context canceled",
},
wantEnd: StreamEndCleanEOF,
wantUsageState: usage.StateComplete,
wantTokens: usage.Tokens{
UncachedInput: 5, CacheRead: 3, Output: 2,
},
},
{
name: "client cancellation before terminal remains canceled",
chunks: []string{deltaEvent},
cancel: true,
terminalError: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "request context canceled",
},
wantEnd: StreamEndClientCanceled,
wantSummary: fixedErrorSummary("client_canceled"),
wantUsageState: usage.StateMissing,
},
{
name: "clean transport EOF without required terminal is protocol error",
chunks: []string{deltaEvent},
wantEnd: StreamEndUpstreamProtocolError,
wantSummary: fixedErrorSummary("upstream_protocol_error"),
wantUsageState: usage.StateMissing,
},
{
name: "incomplete terminal remains incomplete after client cancellation",
chunks: []string{incompleteEvent},
cancel: true,
terminalError: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "request context canceled",
},
wantEnd: StreamEndProviderIncomplete,
wantSummary: fixedErrorSummary("upstream_response_incomplete"),
wantUsageState: usage.StateComplete,
wantTokens: usage.Tokens{
UncachedInput: 5, CacheRead: 3, Output: 2,
},
},
{
name: "failed terminal remains provider error after client cancellation",
chunks: []string{failedEvent},
cancel: true,
terminalError: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "request context canceled",
},
wantSummary: "capacity exhausted",
wantUsageState: usage.StateMissing,
providerError: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
requestContext, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
for index, chunk := range test.chunks {
if err := sink(execution.StreamEvent{
Sequence: uint64(index + 2), Kind: execution.StreamEventData,
Data: []byte(chunk),
}); err != nil {
t.Fatalf("data sink %d: %v", index+1, err)
}
}
if test.cancel {
cancel()
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
Error: test.terminalError,
}
}}
input := responsesExecutionForwardInput()
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(requestContext, input, recorder)
if test.providerError {
if result.Committed || !result.ProviderErrorBeforeCommit ||
result.ErrorSummary != test.wantSummary || recorder.Body.Len() != 0 {
t.Fatalf("ForwardStream() result = %#v, body=%q", result, recorder.Body.String())
}
} else {
if !result.Committed || result.Stream.EndReason != test.wantEnd ||
result.Stream.ErrorSummary != test.wantSummary || result.Usage.State != test.wantUsageState ||
result.Usage.Tokens != test.wantTokens {
t.Fatalf("ForwardStream() result = %#v", result)
}
if got := recorder.Body.String(); got != strings.Join(test.chunks, "") {
t.Fatalf("response body = %q, want %q", got, strings.Join(test.chunks, ""))
}
}
})
}
}
func TestExecutionForwarderNormalizesSplitDowngradedResponsesStoreStream(t *testing.T) {
const stream = "event: response.created\n" +
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_1\",\"store\":true}}\n\n" +
"event: response.output_text.delta\n" +
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"ok\"}\n\n" +
"event: response.completed\n" +
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"store\":true,\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n"
chunks := []string{stream[:37], stream[37:111], stream[111 : len(stream)-9], stream[len(stream)-9:]}
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
spec execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if !bytes.Contains(spec.Body, []byte(`"store":false`)) || bytes.Contains(spec.Body, []byte(`"store":true`)) {
t.Fatalf("upstream request = %s", spec.Body)
}
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatal(err)
}
for index, chunk := range chunks {
if err := sink(execution.StreamEvent{
Sequence: uint64(index + 2), Kind: execution.StreamEventData, Data: []byte(chunk),
}); err != nil {
t.Fatal(err)
}
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.Request.Body = []byte(`{"model":"public","input":"hello","store":true,"stream":true}`)
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(context.Background(), input, recorder)
body := recorder.Body.String()
if result.Err != nil || result.Stream.EndReason != StreamEndCleanEOF ||
strings.Contains(body, `"store":true`) || strings.Count(body, `"store":false`) != 2 ||
!strings.Contains(body, `"type":"response.output_text.delta","delta":"ok"`) {
t.Fatalf("ForwardStream() = %#v, body=%q", result, body)
}
}
func TestExecutionForwarderRejectsDowngradedResponsesStoreStreamEncodingFailure(t *testing.T) {
response := `{"` + strings.Repeat("\u2028", maxResponsesStoreRewriteGrowthBytes/3+1) +
`":0,"id":"resp_1","store":true}`
stream := "event: response.created\n" +
"data: {\"type\":\"response.created\",\"response\":" + response + "}\n\n" +
"event: response.completed\n" +
"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"store\":true}}\n\n"
var sinkErr error
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
sinkErr = sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData, Data: []byte(stream),
})
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.Request.Body = []byte(`{"model":"public","input":"hello","store":true,"stream":true}`)
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(context.Background(), input, recorder)
if !errors.Is(sinkErr, ErrUpstreamProtocol) || !errors.Is(result.Err, ErrUpstreamProtocol) ||
result.Committed || recorder.Body.Len() != 0 {
t.Fatalf(
"ForwardStream() = %#v, sinkErr=%v, body=%q",
result,
sinkErr,
recorder.Body.String(),
)
}
}
func TestExecutionForwarderDoesNotReportStreamReadyForFirstProviderError(t *testing.T) {
t.Parallel()
const failedEvent = "event: response.failed\n" +
"data: {\"type\":\"response.failed\",\"error\":{\"message\":\"capacity exhausted\"},\"response\":{}}\n\n"
readyObserved := false
firstResponseObserved := false
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData, Data: []byte(failedEvent),
}); err != nil {
t.Fatalf("data sink: %v", err)
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIResponses()
input.OnStreamReady = func() { readyObserved = true }
input.OnFirstResponse = func() { firstResponseObserved = true }
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), input, recorder,
)
if result.Err != nil || result.Committed || !result.ProviderErrorBeforeCommit {
t.Fatalf("ForwardStream() result = %#v", result)
}
if readyObserved {
t.Fatal("provider error was reported as a ready successful stream")
}
if !firstResponseObserved {
t.Fatal("provider error must still count as the first upstream response")
}
}
func TestExecutionForwarderPreservesFirstProviderErrorEvidence(t *testing.T) {
t.Parallel()
const failedEvent = "event: error\n" +
"data: {\"type\":\"error\",\"error\":{\"type\":\"rate_limit_error\",\"code\":\"rate_limit\",\"message\":\"Please try again later\"}}\n\n"
header := http.Header{
"Content-Type": {"text/event-stream"},
"Retry-After": {"3"},
}
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady,
StatusCode: http.StatusOK, Header: header,
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData, Data: []byte(failedEvent),
}); err != nil {
t.Fatalf("data sink: %v", err)
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: header,
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIResponses()
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), input, httptest.NewRecorder(),
)
if result.Committed || !result.ProviderErrorBeforeCommit || result.ExecutionError == nil {
t.Fatalf("ForwardStream() result = %#v", result)
}
if result.ExecutionError.Type != "rate_limit_error" ||
result.ExecutionError.Code != "rate_limit" ||
result.ExecutionError.Hint != execution.FailureHintRateLimited ||
result.ExecutionError.ReplaySafety != "" {
t.Fatalf("error evidence = %#v", result.ExecutionError)
}
now := time.Date(2026, time.July, 28, 10, 0, 0, 0, time.UTC)
decision := judgeUpstreamResult(result, now, health.DecisionContext{DefaultRateLimitCooldown: time.Minute})
if decision.Category != health.FailureCategoryRateLimited ||
decision.Effect != health.EffectCooldownCredential ||
!decision.CooldownUntil.Equal(now.Add(3*time.Second)) {
t.Fatalf("health decision = %#v", decision)
}
}
func TestExecutionForwarderClassifiesFirstCapacityErrorReplaySafety(t *testing.T) {
for _, test := range []struct {
name string
errorType string
errorCode string
existing execution.ReplaySafety
want execution.ReplaySafety
images bool
}{
{
name: "server overload", errorType: "service_unavailable_error",
errorCode: "server_is_overloaded", want: execution.ReplaySafetyRejectedBeforeProcessing,
},
{
name: "transient rate limit", errorType: "rate_limit_error",
errorCode: "rate_limit_exceeded", want: execution.ReplaySafetyRejectedBeforeProcessing,
},
{
name: "images explicit unknown remains unsafe", errorType: "service_unavailable_error",
errorCode: "server_is_overloaded", existing: execution.ReplaySafetyUnknown,
want: execution.ReplaySafetyUnknown, images: true,
},
} {
t.Run(test.name, func(t *testing.T) {
failedEvent := "event: error\n" +
"data: {\"type\":\"error\",\"error\":{\"type\":\"" + test.errorType + "\",\"code\":\"" + test.errorCode + "\",\"message\":\"try another credential\"}}\n\n"
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData, Data: []byte(failedEvent),
}); err != nil {
t.Fatalf("data sink: %v", err)
}
result := execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}
if test.existing != "" {
result.Error = &execution.ErrorEvidence{
Kind: execution.ErrorKindProvider, Summary: "explicit replay safety evidence",
ReplaySafety: test.existing,
}
}
return result
}}
input := executionForwardInput()
if test.images {
input.Dialect = dialect.NewOpenAIImages()
input.ClientProtocol = protocol.OpenAIImages
input.Operation = execution.OperationImagesGenerate
input.Request.Path = "/v1/images/generations"
} else {
input.Dialect = dialect.NewOpenAIResponses()
}
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), input, httptest.NewRecorder(),
)
if result.Committed || !result.ProviderErrorBeforeCommit || result.ExecutionError == nil ||
result.ExecutionError.Type != test.errorType || result.ExecutionError.Code != test.errorCode ||
result.ExecutionError.ReplaySafety != test.want {
t.Fatalf("ForwardStream() result = %#v", result)
}
if test.images {
decision := judgeUpstreamResult(result, time.Now(), health.DecisionContext{
Method: http.MethodPost, Operation: execution.OperationImagesGenerate,
})
if decision.Retry != health.RetryNone {
t.Fatalf("JudgeExecution() = %#v", decision)
}
}
})
}
}
func TestExecutionForwarderDoesNotPromoteCapacityErrorAfterCommit(t *testing.T) {
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
emit := func(event execution.StreamEvent) {
if err := sink(event); err != nil {
t.Fatalf("stream sink: %v", err)
}
}
emit(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
})
emit(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData,
Data: []byte("event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_1\",\"status\":\"in_progress\",\"output\":[]}}\n\n"),
})
emit(execution.StreamEvent{
Sequence: 3, Kind: execution.StreamEventData,
Data: []byte("event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"service_unavailable_error\",\"code\":\"server_is_overloaded\",\"message\":\"too late\"}}\n\n"),
})
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIResponses()
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), input, httptest.NewRecorder(),
)
if !result.Committed || result.ProviderErrorBeforeCommit ||
result.ExecutionError != nil && result.ExecutionError.ReplaySafety != "" {
t.Fatalf("ForwardStream() result = %#v", result)
}
}
func TestExecutionForwarderRejectsStreamingEOFWithoutProtocolTerminal(t *testing.T) {
t.Parallel()
tests := []struct {
name string
dialect dialect.Dialect
data string
}{
{
name: "OpenAI Chat", dialect: dialect.NewOpenAI(),
data: "data: {\"id\":\"chat_1\",\"object\":\"chat.completion.chunk\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"},\"finish_reason\":null}]}\n\n",
},
{
name: "Anthropic", dialect: dialect.NewAnthropic(),
data: "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hi\"}}\n\n",
},
{
name: "Gemini", dialect: dialect.NewGemini(),
data: "data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"hi\"}],\"role\":\"model\"},\"index\":0}]}\n\n",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData, Data: []byte(test.data),
}); err != nil {
t.Fatalf("data sink: %v", err)
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = test.dialect
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), input, httptest.NewRecorder(),
)
if !result.Committed || result.Err == nil ||
result.Stream.EndReason != StreamEndUpstreamProtocolError {
t.Fatalf("ForwardStream() result = %#v", result)
}
})
}
}
func TestExecutionForwarderFreezesObservationAfterOpenAIResponsesTerminal(t *testing.T) {
t.Parallel()
const completedEvent = "event: response.completed\n" +
"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{\"input_tokens\":8,\"output_tokens\":2,\"total_tokens\":10}}}\n\n"
const trailingEvent = "event: error\n" +
"data: {\"type\":\"error\",\"error\":{\"message\":\"late failure\"},\"response\":{\"usage\":{\"input_tokens\":80,\"output_tokens\":20,\"total_tokens\":100}}}\n\n"
tests := []struct {
name string
chunks []string
}{
{name: "separate chunks", chunks: []string{completedEvent, trailingEvent}},
{name: "same chunk", chunks: []string{completedEvent + trailingEvent}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
var sinkErr error
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
for index, chunk := range test.chunks {
sinkErr = sink(execution.StreamEvent{
Sequence: uint64(index + 2), Kind: execution.StreamEventData,
Data: []byte(chunk),
})
if sinkErr != nil {
break
}
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
forwarder := NewExecutionForwarder(executor)
recorder := httptest.NewRecorder()
result := forwarder.ForwardStream(
context.Background(), responsesExecutionForwardInput(), recorder,
)
if sinkErr != nil || result.Err != nil || !result.Committed ||
result.Stream.EndReason != StreamEndCleanEOF || result.Stream.ErrorSummary != "" {
t.Fatalf("sink error/result = %v / %#v", sinkErr, result)
}
if result.Usage.State != usage.StateComplete ||
result.Usage.Tokens != (usage.Tokens{UncachedInput: 8, Output: 2}) {
t.Fatalf("usage = %#v", result.Usage)
}
if failures := forwarder.usageCapture.failureTotal.Load(); failures != 0 {
t.Fatalf("usage capture failures = %d, want 0", failures)
}
if got, want := recorder.Body.String(), completedEvent+trailingEvent; got != want {
t.Fatalf("response body = %q, want %q", got, want)
}
})
}
}
func TestExecutionForwarderPassesDataAfterTerminalAcrossDialects(t *testing.T) {
t.Parallel()
const trailingEvent = "data: {\"metadata\":{\"cost\":\"0\"}}\n\n"
tests := []struct {
name string
dialect dialect.Dialect
protocol protocol.Protocol
terminal string
}{
{
name: "OpenAI Chat", dialect: dialect.NewOpenAI(),
protocol: protocol.OpenAICompletions,
terminal: "data: [DONE]\n\n",
},
{
name: "Anthropic", dialect: dialect.NewAnthropic(),
protocol: protocol.Anthropic,
terminal: "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
},
{
name: "Gemini", dialect: dialect.NewGemini(),
protocol: protocol.Gemini,
terminal: "data: {\"candidates\":[{\"index\":0,\"finishReason\":\"STOP\"}]}\n\n",
},
{
name: "OpenAI Images", dialect: dialect.NewOpenAIImages(),
protocol: protocol.OpenAIImages,
terminal: "event: image_generation.completed\ndata: {\"type\":\"image_generation.completed\"}\n\n",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
var sinkErr error
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
sinkErr = sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData,
Data: []byte(test.terminal + trailingEvent),
})
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}
}}
input := executionForwardInput()
input.Dialect = test.dialect
input.ClientProtocol = test.protocol
if test.protocol == protocol.OpenAIImages {
input.Operation = execution.OperationImagesGenerate
input.Request.Path = "/v1/images/generations"
input.Request.RawQuery = ""
input.Request.Body = []byte(`{"model":"public","stream":true}`)
}
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), input, recorder,
)
if sinkErr != nil || result.Err != nil || !result.Committed ||
result.Stream.EndReason != StreamEndCleanEOF {
t.Fatalf("sink error/result = %v / %#v", sinkErr, result)
}
if got, want := recorder.Body.String(), test.terminal+trailingEvent; got != want {
t.Fatalf("response body = %q, want %q", got, want)
}
})
}
}
func TestExecutionForwarderRequiresSuccessfulFlushBeforeAcceptingResponsesTerminal(t *testing.T) {
t.Parallel()
const completedEvent = "event: response.completed\n" +
"data: {\"type\":\"response.completed\",\"response\":{}}\n\n"
flushErr := errors.New("downstream flush failed")
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{
Sequence: 2, Kind: execution.StreamEventData, Data: []byte(completedEvent),
}); !errors.Is(err, flushErr) {
t.Fatalf("terminal sink error = %v, want %v", err, flushErr)
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"text/event-stream"}},
Error: &execution.ErrorEvidence{
Kind: execution.ErrorKindCanceled, Summary: "stream sink stopped",
},
}
}}
writer := &executionFlushFailureWriter{
header: make(http.Header),
err: flushErr,
}
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), responsesExecutionForwardInput(), writer,
)
if !result.Committed || !errors.Is(result.Err, flushErr) ||
result.Stream.EndReason != StreamEndDownstreamWriteFailure {
t.Fatalf("ForwardStream() = %#v", result)
}
}
func TestExecutionForwarderBuffersRejectedStreamWithoutCommitting(t *testing.T) {
t.Parallel()
evidence := execution.ErrorEvidence{
Kind: execution.ErrorKindHTTP, StatusCode: http.StatusUnauthorized,
Summary: "upstream rejected request",
}
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
if err := sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady,
StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": {"application/json"}},
}); err != nil {
t.Fatalf("ready sink: %v", err)
}
if err := sink(execution.StreamEvent{Sequence: 2, Kind: execution.StreamEventData, Data: []byte(`{"error":"invalid"}`)}); err != nil {
t.Fatalf("data sink: %v", err)
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": {"application/json"}},
Error: &evidence,
}
}}
recorder := httptest.NewRecorder()
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), executionForwardInput(), recorder,
)
if result.Committed || recorder.Code != http.StatusOK || recorder.Body.Len() != 0 ||
result.StatusCode != http.StatusUnauthorized || string(result.Body) != `{"error":"invalid"}` ||
result.ExecutionError == nil {
t.Fatalf("ForwardStream() result=%#v response=%d %q", result, recorder.Code, recorder.Body.String())
}
}
func TestExecutionForwarderDoesNotRetryOrCommitAfterDownstreamFailure(t *testing.T) {
t.Parallel()
downstreamErr := errors.New("downstream closed")
executor := fakeExecutionExecutor{stream: func(
_ context.Context,
_ execution.AttemptSpec,
sink execution.StreamSink,
) execution.StreamResult {
_ = sink(execution.StreamEvent{
Sequence: 1, Kind: execution.StreamEventReady, StatusCode: http.StatusOK,
Header: make(http.Header),
})
if err := sink(execution.StreamEvent{Sequence: 2, Kind: execution.StreamEventData, Data: []byte("data: {\"id\":\"chat_1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"one\"},\"finish_reason\":null}]}\n\n")}); err == nil {
t.Fatal("data sink unexpectedly succeeded")
}
return execution.StreamResult{
DispatchState: execution.DispatchMaybeSent, ResponseStarted: true,
StatusCode: http.StatusOK, Header: make(http.Header),
Error: &execution.ErrorEvidence{Kind: execution.ErrorKindCanceled, Summary: "stream sink stopped"},
}
}}
writer := &failingExecutionResponseWriter{header: make(http.Header), err: downstreamErr}
result := NewExecutionForwarder(executor).ForwardStream(
context.Background(), executionForwardInput(), writer,
)
if !result.Committed || !errors.Is(result.Err, downstreamErr) || result.Stream.EndReason != StreamEndDownstreamWriteFailure {
t.Fatalf("ForwardStream() = %#v", result)
}
}
type failingExecutionResponseWriter struct {
header http.Header
err error
}
func (writer *failingExecutionResponseWriter) Header() http.Header { return writer.header }
func (*failingExecutionResponseWriter) WriteHeader(int) {}
func (writer *failingExecutionResponseWriter) Write([]byte) (int, error) {
return 0, writer.err
}
func (*failingExecutionResponseWriter) FlushError() error { return nil }
type executionFlushFailureWriter struct {
header http.Header
err error
}
func (writer *executionFlushFailureWriter) Header() http.Header { return writer.header }
func (*executionFlushFailureWriter) WriteHeader(int) {}
func (*executionFlushFailureWriter) Write(body []byte) (int, error) {
return len(body), nil
}
func (writer *executionFlushFailureWriter) FlushError() error { return writer.err }
func executionForwardInput() ForwardInput {
return ForwardInput{
Dialect: dialect.NewOpenAI(),
RequestID: "request-1",
AttemptID: "attempt-1",
AttemptSequence: 2,
ClientProtocol: protocol.OpenAICompletions,
Operation: execution.OperationChatCompletion,
ChannelID: "openai",
RouteMode: execution.RouteNative,
RouteRequirement: execution.RouteRequirementNative,
TargetConfig: []byte(`{}`),
APIKey: "secret",
Credential: execution.NewCredentialSnapshot(
7, 2, 3, []byte(`{"api_key":"secret"}`),
),
Request: &dialect.ParsedRequest{
Method: http.MethodPost,
Path: "/v1/chat/completions",
RawQuery: "trace=kept",
Header: http.Header{
"Authorization": {"Bearer downstream"},
"X-Remove": {"remove"},
"X-Test": {"kept"},
},
Body: []byte(`{"model":"public"}`),
},
ExternalModel: "public",
UpstreamModelID: "upstream",
Group: state.GroupView{
Timeouts: state.TimeoutConfig{
FirstByte: time.Second,
Request: time.Second, StreamIdle: time.Second,
},
HeaderRules: state.HeaderRules{
Set: map[string]string{"X-Template": "Bearer ${API_KEY}"},
Remove: []string{"X-Remove"},
},
},
}
}
func TestNewExecutionAttemptSpecKeepsContinuityPrivate(t *testing.T) {
input := executionForwardInput()
input.ContinuityKey = "tenant-scoped-hmac"
spec, err := newExecutionAttemptSpec(input)
if err != nil {
t.Fatal(err)
}
if spec.ContinuityKey != input.ContinuityKey {
t.Fatalf("ContinuityKey = %q, want %q", spec.ContinuityKey, input.ContinuityKey)
}
raw, err := json.Marshal(spec)
if err != nil {
t.Fatal(err)
}
if bytes.Contains(raw, []byte(input.ContinuityKey)) {
t.Fatalf("AttemptSpec JSON exposed continuity key: %s", raw)
}
}
func TestOtherChannelsRetainIdentityHeaderRules(t *testing.T) {
for _, test := range []struct {
name string
rules state.HeaderRules
wantUA string
wantVersion string
}{
{name: "client headers", wantUA: "client/1.0", wantVersion: "1.0"},
{
name: "configured headers",
rules: state.HeaderRules{Set: map[string]string{"User-Agent": "configured/2.0", "Version": "2.0"}},
wantUA: "configured/2.0", wantVersion: "2.0",
},
{name: "removed headers", rules: state.HeaderRules{Remove: []string{"User-Agent", "Version"}}},
} {
t.Run(test.name, func(t *testing.T) {
input := executionForwardInput()
input.Request.Header.Set("User-Agent", "client/1.0")
input.Request.Header.Set("Version", "1.0")
input.Group.HeaderRules = test.rules
spec, err := newExecutionAttemptSpec(input)
if err != nil {
t.Fatal(err)
}
if spec.Header.Get("User-Agent") != test.wantUA || spec.Header.Get("Version") != test.wantVersion {
t.Fatalf("non-Codex headers changed: UA=%q Version=%q", spec.Header.Get("User-Agent"), spec.Header.Get("Version"))
}
})
}
}
func TestNewExecutionAttemptSpecCarriesResponsesStoreDowngrade(t *testing.T) {
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.Request.Body = []byte(`{"model":"public","input":"hello","store":true}`)
spec, err := newExecutionAttemptSpec(input)
if err != nil {
t.Fatal(err)
}
if !spec.ResponsesStoreDowngraded || !bytes.Contains(spec.Body, []byte(`"store":false`)) ||
bytes.Contains(spec.Body, []byte(`"store":true`)) {
t.Fatalf("AttemptSpec = %#v, body=%s", spec, spec.Body)
}
}
func TestNewExecutionAttemptSpecInvalidatesRepresentationHeadersAfterStoreRewrite(t *testing.T) {
input := responsesExecutionForwardInput()
input.RouteRequirement = execution.RouteRequirementAny
input.ResponsesStoreDowngraded = true
input.Request.Body = []byte(`{"model":"public","input":"hello","store":true}`)
input.Group.HeaderRules = state.HeaderRules{Set: map[string]string{
"ETag": `"stale"`,
"Digest": "sha-256=stale",
"Content-MD5": "stale-md5",
"Content-Range": "bytes 0-1/2",
"Content-Digest": "sha-256=:c3RhbGU=:",
"Repr-Digest": "sha-256=:c3RhbGU=:",
"Signature": "stale-signature",
"Signature-Input": "stale-signature-input",
"X-Upstream": "kept",
}}
spec, err := newExecutionAttemptSpec(input)
if err != nil {
t.Fatal(err)
}
for _, name := range representationMetadataHeaderNames {
if values := headerFieldValues(spec.Header, name); len(values) > 0 {
t.Errorf("%s = %#v, want absent", name, values)
}
}
if got := spec.Header.Get("X-Upstream"); got != "kept" {
t.Fatalf("X-Upstream = %q, want kept", got)
}
}
func responsesExecutionForwardInput() ForwardInput {
input := executionForwardInput()
input.Dialect = dialect.NewOpenAIResponses()
input.ObserveUsage = true
input.ClientProtocol = protocol.OpenAIResponses
input.Operation = execution.OperationResponsesCreate
input.Request.Path = "/v1/responses"
input.Request.RawQuery = ""
input.Request.Body = []byte(`{"model":"public","stream":true}`)
return input
}
func TestStreamErrorFailureHintDoesNotTreatGenericForbiddenAsInvalidCredential(t *testing.T) {
tests := []struct {
name string
status int
values []string
want execution.FailureHint
}{
{
name: "model marker wins over forbidden",
status: http.StatusForbidden,
values: []string{"model_not_found"},
want: execution.FailureHintModelUnavailable,
},
{
name: "generic forbidden permission",
status: http.StatusForbidden,
values: []string{"permission_denied"},
},
{
name: "payment required",
status: http.StatusPaymentRequired,
values: []string{"billing disabled"},
},
{
name: "explicit invalid key",
status: http.StatusForbidden,
values: []string{"API key not valid"},
want: execution.FailureHintInvalidCredential,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := streamErrorFailureHint(test.status, test.values...); got != test.want {
t.Fatalf("streamErrorFailureHint() = %q, want %q", got, test.want)
}
})
}
}