fix(grok): allow passive Codex image tool declarations

This commit is contained in:
superman2003 committed 2026-07-16 00:16:21 +08:00
1 parent eb2b8632de
commit 410ea8490c
6 files changed
+316 -7

No files matched your search

@@ -85,7 +85,7 @@ func (h *GatewayHandler) Responses(c *gin.Context) {
setOpsRequestContext(c, reqModel, reqStream)
setOpsEndpointContext(c, "", int16(service.RequestTypeFromLegacy(reqStream, false)))
requestCtx := c.Request.Context()
if service.IsImageGenerationIntent("/v1/responses", reqModel, body) {
if service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, body, openAICompatibleRequestPlatform(apiKey)) {
requestCtx = service.WithOpenAIImageGenerationIntent(requestCtx)
}
@@ -272,7 +272,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) {
return
}
imageIntent := service.IsImageGenerationIntent("/v1/responses", reqModel, body)
imageIntent := service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, body, openAICompatibleRequestPlatform(apiKey))
if imageIntent && !service.GroupAllowsImageGeneration(apiKey.Group) {
h.errorResponse(c, http.StatusForbidden, "permission_error", service.ImageGenerationPermissionMessage())
return
@@ -1461,7 +1461,7 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) {
return
}
if service.IsImageGenerationIntent("/v1/responses", reqModel, firstMessage) && !service.GroupAllowsImageGeneration(apiKey.Group) {
if service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, firstMessage, openAICompatibleRequestPlatform(apiKey)) && !service.GroupAllowsImageGeneration(apiKey.Group) {
closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, service.ImageGenerationPermissionMessage())
return
}
@@ -0,0 +1,101 @@
package handler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/config"
middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func TestOpenAIGatewayHandlerResponses_GrokPassiveImageToolDeclarationBypassesPermissionGate(t *testing.T) {
body := `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`
rec := runOpenAIResponsesImagePermissionGateTest(t, service.PlatformGrok, body)
require.NotEqual(t, http.StatusForbidden, rec.Code)
require.NotContains(t, rec.Body.String(), service.ImageGenerationPermissionMessage())
}
func TestOpenAIGatewayHandlerResponses_GrokResponsesLiteImageToolDeclarationBypassesPermissionGate(t *testing.T) {
body := `{"model":"grok-4.5","tool_choice":"auto","input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]},{"type":"message","role":"user","content":"write code"}]}`
rec := runOpenAIResponsesImagePermissionGateTest(t, service.PlatformGrok, body)
require.NotEqual(t, http.StatusForbidden, rec.Code)
require.NotContains(t, rec.Body.String(), service.ImageGenerationPermissionMessage())
}
func TestOpenAIGatewayHandlerResponses_ImagePermissionHardSignalsStillRejected(t *testing.T) {
passiveNamespace := `{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`
tests := []struct {
name string
platform string
body string
}{
{
name: "OpenAI keeps declaration semantics",
platform: service.PlatformOpenAI,
body: passiveNamespace,
},
{
name: "Grok native image_generation declaration",
platform: service.PlatformGrok,
body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"input":"draw"}`,
},
{
name: "Grok explicit image_gen tool choice",
platform: service.PlatformGrok,
body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen"}],"tool_choice":{"type":"namespace","name":"image_gen"},"input":"draw"}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rec := runOpenAIResponsesImagePermissionGateTest(t, tt.platform, tt.body)
require.Equal(t, http.StatusForbidden, rec.Code)
require.Contains(t, rec.Body.String(), service.ImageGenerationPermissionMessage())
})
}
}
func runOpenAIResponsesImagePermissionGateTest(t *testing.T, platform string, body string) *httptest.ResponseRecorder {
t.Helper()
gin.SetMode(gin.TestMode)
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
c.Request.Header.Set("Content-Type", "application/json")
groupID := int64(6301)
userID := int64(6302)
c.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{
ID: 6303,
GroupID: &groupID,
Group: &service.Group{
ID: groupID,
Platform: platform,
AllowImageGeneration: false,
},
User: &service.User{ID: userID, Status: service.StatusActive},
})
c.Set(string(middleware2.ContextKeyUser), middleware2.AuthSubject{UserID: userID, Concurrency: 1})
h := &OpenAIGatewayHandler{
gatewayService: &service.OpenAIGatewayService{},
billingCacheService: service.NewBillingCacheService(nil, nil, nil, nil, nil, nil, &config.Config{RunMode: config.RunModeSimple}, nil),
apiKeyService: &service.APIKeyService{},
concurrencyHelper: &ConcurrencyHelper{concurrencyService: service.NewConcurrencyService(
&helperConcurrencyCacheStub{userSeq: []bool{true}},
)},
cfg: &config.Config{},
imageLimiter: &imageConcurrencyLimiter{},
}
h.Responses(c)
return rec
}
@@ -79,6 +79,56 @@ func IsImageGenerationIntent(endpoint string, requestedModel string, body []byte
return imageIntent
}
// IsImageGenerationIntentForPlatform applies platform-specific intent rules.
//
// Codex advertises the image_gen namespace on ordinary Responses requests so
// that it is available if the model needs it. Grok strips namespace and
// Responses Lite additional_tools declarations before forwarding, so those
// declarations alone must not turn every Codex request into an image request.
// Native image_generation tools, explicit image selection and image models
// remain image intent. Other platforms retain the original declaration rule.
func IsImageGenerationIntentForPlatform(endpoint string, requestedModel string, body []byte, platform string) bool {
if !strings.EqualFold(strings.TrimSpace(platform), PlatformGrok) {
return IsImageGenerationIntent(endpoint, requestedModel, body)
}
return isExplicitGrokImageGenerationIntent(endpoint, requestedModel, body)
}
func isExplicitGrokImageGenerationIntent(endpoint string, requestedModel string, body []byte) bool {
if IsImageGenerationEndpoint(endpoint) || isOpenAIImageGenerationModel(requestedModel) {
return true
}
if len(body) == 0 || !gjson.ValidBytes(body) {
return false
}
var modelSeen, toolsSeen, toolChoiceSeen bool
imageIntent := false
parseRawJSONView(body).ForEach(func(key, value gjson.Result) bool {
switch key.Str {
case "model":
if !modelSeen {
modelSeen = true
imageIntent = isOpenAIImageGenerationModel(strings.TrimSpace(value.String()))
}
case "tools":
if !toolsSeen {
toolsSeen = true
// Grok removes namespace catalogs before forwarding. Native
// image_generation remains an explicit capability request.
imageIntent = openAIJSONToolsContainNativeImageGeneration(value)
}
case "tool_choice":
if !toolChoiceSeen {
toolChoiceSeen = true
imageIntent = openAIJSONToolChoiceSelectsExplicitImageGeneration(value)
}
}
return !imageIntent && (!modelSeen || !toolsSeen || !toolChoiceSeen)
})
return imageIntent
}
// IsImageGenerationIntentMap is the map-backed variant used after service-side request mutation.
func IsImageGenerationIntentMap(endpoint string, requestedModel string, reqBody map[string]any) bool {
if IsImageGenerationEndpoint(endpoint) {
@@ -140,6 +190,18 @@ func openAIJSONToolsContainImageGeneration(tools gjson.Result) bool {
return found
}
func openAIJSONToolsContainNativeImageGeneration(tools gjson.Result) bool {
if !tools.IsArray() {
return false
}
found := false
tools.ForEach(func(_, item gjson.Result) bool {
found = isOpenAIImageGenerationType(openAIJSONString(item.Get("type")))
return !found
})
return found
}
func isOpenAIImageGenerationType(value string) bool {
return strings.TrimSpace(value) == "image_generation"
}
@@ -148,9 +210,8 @@ func isOpenAIImageGenNamespaceName(value string) bool {
return strings.TrimSpace(value) == "image_gen"
}
// isImageGenNamespaceTool detects the Codex namespace-style image generation
// tool declaration: { "type": "namespace", "name": "image_gen", ... }.
// Codex /image uses this instead of the flat { "type": "image_generation" }.
// isImageGenNamespaceTool detects the namespace advertised by Codex's built-in
// image-generation extension instead of a hosted image_generation tool.
func isImageGenNamespaceTool(tool gjson.Result) bool {
return openAIJSONString(tool.Get("type")) == "namespace" &&
isOpenAIImageGenNamespaceName(openAIJSONString(tool.Get("name")))
@@ -235,6 +296,45 @@ func openAIJSONToolChoiceSelectsImageGeneration(choice gjson.Result) bool {
return false
}
func openAIJSONToolChoiceSelectsExplicitImageGeneration(choice gjson.Result) bool {
if openAIJSONToolChoiceSelectsImageGeneration(choice) {
return true
}
if !choice.IsObject() {
return false
}
if tool := choice.Get("tool"); tool.IsObject() && openAIJSONToolChoiceSelectsExplicitImageGeneration(tool) {
return true
}
if isOpenAIImageGenFunctionReference(
openAIJSONString(choice.Get("namespace")),
openAIJSONString(choice.Get("name")),
) {
return true
}
if fn := choice.Get("function"); fn.IsObject() {
return isOpenAIImageGenFunctionReference(
openAIJSONString(fn.Get("namespace")),
openAIJSONString(fn.Get("name")),
)
}
return false
}
func isOpenAIImageGenFunctionReference(namespace string, name string) bool {
namespace = strings.TrimSpace(namespace)
name = strings.TrimSpace(name)
if namespace == "image_gen" && name == "imagegen" {
return true
}
switch name {
case "image_gen.imagegen", "image_gen__imagegen":
return true
default:
return false
}
}
func openAIAnyToolChoiceSelectsImageGeneration(choice any) bool {
switch v := choice.(type) {
case string:
@@ -0,0 +1,108 @@
package service
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestIsImageGenerationIntentForPlatform_GrokCodexDeclarations(t *testing.T) {
tests := []struct {
name string
body string
want bool
}{
{
name: "top-level image_gen namespace is passive",
body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`,
},
{
name: "responses lite additional_tools image_gen is passive",
body: `{"model":"grok-4.5","tool_choice":"auto","input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]},{"type":"message","role":"user","content":"write code"}]}`,
},
{
name: "legacy adapter flattened image_gen function declaration is passive",
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"input":"write code"}`,
},
{
name: "native image generation declaration remains explicit",
body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"input":"draw a cat"}`,
want: true,
},
{
name: "native image generation tool choice is explicit",
body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"tool_choice":{"type":"image_generation"},"input":"draw a cat"}`,
want: true,
},
{
name: "image_gen namespace tool choice is explicit",
body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen"}],"tool_choice":{"type":"namespace","name":"image_gen"},"input":"draw a cat"}`,
want: true,
},
{
name: "flattened image_gen function tool choice is explicit",
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"tool_choice":{"type":"function","name":"image_gen.imagegen"},"input":"draw a cat"}`,
want: true,
},
{
name: "wrapped image_gen function tool choice is explicit",
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"tool_choice":{"tool":{"type":"function","name":"image_gen.imagegen"}},"input":"draw a cat"}`,
want: true,
},
{
name: "vision input is not image generation",
body: `{"model":"grok-4.5","input":[{"type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}]}`,
},
{
name: "ordinary function declaration is not image generation",
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"lookup"}],"tool_choice":"auto","input":"write code"}`,
},
{
name: "image_gen function call history is not current intent",
body: `{"model":"grok-4.5","input":[{"type":"function_call","namespace":"image_gen","name":"imagegen","arguments":"{}"}]}`,
},
{
name: "image generation call history is not current intent",
body: `{"model":"grok-4.5","input":[{"type":"image_generation_call","id":"ig_1"}]}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.want, IsImageGenerationIntentForPlatform(
openAIResponsesEndpoint,
"grok-4.5",
[]byte(tt.body),
PlatformGrok,
))
})
}
}
func TestIsImageGenerationIntentForPlatform_GrokPreservesHardSignals(t *testing.T) {
require.True(t, IsImageGenerationIntentForPlatform(
"/v1/images/generations",
"grok-4.5",
[]byte(`{"input":"draw"}`),
PlatformGrok,
))
require.True(t, IsImageGenerationIntentForPlatform(
openAIResponsesEndpoint,
"gpt-image-2",
[]byte(`{"input":"draw"}`),
PlatformGrok,
))
require.True(t, IsImageGenerationIntentForPlatform(
openAIResponsesEndpoint,
"grok-4.5",
[]byte(`{"model":"gpt-image-2","input":"draw"}`),
PlatformGrok,
))
}
func TestIsImageGenerationIntentForPlatform_OtherPlatformsKeepDeclarationSemantics(t *testing.T) {
body := []byte(`{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"input":"write code"}`)
require.True(t, IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, "gpt-5.5", body, PlatformOpenAI))
require.True(t, IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, "gpt-5.5", body, PlatformAnthropic))
}
@@ -305,7 +305,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
normalized = stripped
logOpenAIWSModeInfo("ingress_ws_codex_spark_image_tool_stripped account_id=%d", account.ID)
}
imageIntent := IsImageGenerationIntent(openAIResponsesEndpoint, originalModel, normalized)
imageIntent := IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, originalModel, normalized, account.Platform)
if imageIntent && !imageGenerationAllowed {
return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, ImageGenerationPermissionMessage(), nil)
}