fix(grok): allow passive Codex image tool declarations
This commit is contained in:
1 parent
eb2b8632de
commit
410ea8490c
6 files changed
+316
-7
No files matched your search
@@ -85,7 +85,7 @@ func (h *GatewayHandler) Responses(c *gin.Context) {
|
||||
setOpsRequestContext(c, reqModel, reqStream)
|
||||
setOpsEndpointContext(c, "", int16(service.RequestTypeFromLegacy(reqStream, false)))
|
||||
requestCtx := c.Request.Context()
|
||||
if service.IsImageGenerationIntent("/v1/responses", reqModel, body) {
|
||||
if service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, body, openAICompatibleRequestPlatform(apiKey)) {
|
||||
requestCtx = service.WithOpenAIImageGenerationIntent(requestCtx)
|
||||
}
|
||||
|
||||
|
||||
@@ -272,7 +272,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
imageIntent := service.IsImageGenerationIntent("/v1/responses", reqModel, body)
|
||||
imageIntent := service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, body, openAICompatibleRequestPlatform(apiKey))
|
||||
if imageIntent && !service.GroupAllowsImageGeneration(apiKey.Group) {
|
||||
h.errorResponse(c, http.StatusForbidden, "permission_error", service.ImageGenerationPermissionMessage())
|
||||
return
|
||||
@@ -1461,7 +1461,7 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if service.IsImageGenerationIntent("/v1/responses", reqModel, firstMessage) && !service.GroupAllowsImageGeneration(apiKey.Group) {
|
||||
if service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, firstMessage, openAICompatibleRequestPlatform(apiKey)) && !service.GroupAllowsImageGeneration(apiKey.Group) {
|
||||
closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, service.ImageGenerationPermissionMessage())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestOpenAIGatewayHandlerResponses_GrokPassiveImageToolDeclarationBypassesPermissionGate(t *testing.T) {
|
||||
body := `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`
|
||||
rec := runOpenAIResponsesImagePermissionGateTest(t, service.PlatformGrok, body)
|
||||
|
||||
require.NotEqual(t, http.StatusForbidden, rec.Code)
|
||||
require.NotContains(t, rec.Body.String(), service.ImageGenerationPermissionMessage())
|
||||
}
|
||||
|
||||
func TestOpenAIGatewayHandlerResponses_GrokResponsesLiteImageToolDeclarationBypassesPermissionGate(t *testing.T) {
|
||||
body := `{"model":"grok-4.5","tool_choice":"auto","input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]},{"type":"message","role":"user","content":"write code"}]}`
|
||||
rec := runOpenAIResponsesImagePermissionGateTest(t, service.PlatformGrok, body)
|
||||
|
||||
require.NotEqual(t, http.StatusForbidden, rec.Code)
|
||||
require.NotContains(t, rec.Body.String(), service.ImageGenerationPermissionMessage())
|
||||
}
|
||||
|
||||
func TestOpenAIGatewayHandlerResponses_ImagePermissionHardSignalsStillRejected(t *testing.T) {
|
||||
passiveNamespace := `{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`
|
||||
tests := []struct {
|
||||
name string
|
||||
platform string
|
||||
body string
|
||||
}{
|
||||
{
|
||||
name: "OpenAI keeps declaration semantics",
|
||||
platform: service.PlatformOpenAI,
|
||||
body: passiveNamespace,
|
||||
},
|
||||
{
|
||||
name: "Grok native image_generation declaration",
|
||||
platform: service.PlatformGrok,
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"input":"draw"}`,
|
||||
},
|
||||
{
|
||||
name: "Grok explicit image_gen tool choice",
|
||||
platform: service.PlatformGrok,
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen"}],"tool_choice":{"type":"namespace","name":"image_gen"},"input":"draw"}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rec := runOpenAIResponsesImagePermissionGateTest(t, tt.platform, tt.body)
|
||||
|
||||
require.Equal(t, http.StatusForbidden, rec.Code)
|
||||
require.Contains(t, rec.Body.String(), service.ImageGenerationPermissionMessage())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runOpenAIResponsesImagePermissionGateTest(t *testing.T, platform string, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body))
|
||||
c.Request.Header.Set("Content-Type", "application/json")
|
||||
|
||||
groupID := int64(6301)
|
||||
userID := int64(6302)
|
||||
c.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{
|
||||
ID: 6303,
|
||||
GroupID: &groupID,
|
||||
Group: &service.Group{
|
||||
ID: groupID,
|
||||
Platform: platform,
|
||||
AllowImageGeneration: false,
|
||||
},
|
||||
User: &service.User{ID: userID, Status: service.StatusActive},
|
||||
})
|
||||
c.Set(string(middleware2.ContextKeyUser), middleware2.AuthSubject{UserID: userID, Concurrency: 1})
|
||||
|
||||
h := &OpenAIGatewayHandler{
|
||||
gatewayService: &service.OpenAIGatewayService{},
|
||||
billingCacheService: service.NewBillingCacheService(nil, nil, nil, nil, nil, nil, &config.Config{RunMode: config.RunModeSimple}, nil),
|
||||
apiKeyService: &service.APIKeyService{},
|
||||
concurrencyHelper: &ConcurrencyHelper{concurrencyService: service.NewConcurrencyService(
|
||||
&helperConcurrencyCacheStub{userSeq: []bool{true}},
|
||||
)},
|
||||
cfg: &config.Config{},
|
||||
imageLimiter: &imageConcurrencyLimiter{},
|
||||
}
|
||||
|
||||
h.Responses(c)
|
||||
return rec
|
||||
}
|
||||
@@ -79,6 +79,56 @@ func IsImageGenerationIntent(endpoint string, requestedModel string, body []byte
|
||||
return imageIntent
|
||||
}
|
||||
|
||||
// IsImageGenerationIntentForPlatform applies platform-specific intent rules.
|
||||
//
|
||||
// Codex advertises the image_gen namespace on ordinary Responses requests so
|
||||
// that it is available if the model needs it. Grok strips namespace and
|
||||
// Responses Lite additional_tools declarations before forwarding, so those
|
||||
// declarations alone must not turn every Codex request into an image request.
|
||||
// Native image_generation tools, explicit image selection and image models
|
||||
// remain image intent. Other platforms retain the original declaration rule.
|
||||
func IsImageGenerationIntentForPlatform(endpoint string, requestedModel string, body []byte, platform string) bool {
|
||||
if !strings.EqualFold(strings.TrimSpace(platform), PlatformGrok) {
|
||||
return IsImageGenerationIntent(endpoint, requestedModel, body)
|
||||
}
|
||||
return isExplicitGrokImageGenerationIntent(endpoint, requestedModel, body)
|
||||
}
|
||||
|
||||
func isExplicitGrokImageGenerationIntent(endpoint string, requestedModel string, body []byte) bool {
|
||||
if IsImageGenerationEndpoint(endpoint) || isOpenAIImageGenerationModel(requestedModel) {
|
||||
return true
|
||||
}
|
||||
if len(body) == 0 || !gjson.ValidBytes(body) {
|
||||
return false
|
||||
}
|
||||
|
||||
var modelSeen, toolsSeen, toolChoiceSeen bool
|
||||
imageIntent := false
|
||||
parseRawJSONView(body).ForEach(func(key, value gjson.Result) bool {
|
||||
switch key.Str {
|
||||
case "model":
|
||||
if !modelSeen {
|
||||
modelSeen = true
|
||||
imageIntent = isOpenAIImageGenerationModel(strings.TrimSpace(value.String()))
|
||||
}
|
||||
case "tools":
|
||||
if !toolsSeen {
|
||||
toolsSeen = true
|
||||
// Grok removes namespace catalogs before forwarding. Native
|
||||
// image_generation remains an explicit capability request.
|
||||
imageIntent = openAIJSONToolsContainNativeImageGeneration(value)
|
||||
}
|
||||
case "tool_choice":
|
||||
if !toolChoiceSeen {
|
||||
toolChoiceSeen = true
|
||||
imageIntent = openAIJSONToolChoiceSelectsExplicitImageGeneration(value)
|
||||
}
|
||||
}
|
||||
return !imageIntent && (!modelSeen || !toolsSeen || !toolChoiceSeen)
|
||||
})
|
||||
return imageIntent
|
||||
}
|
||||
|
||||
// IsImageGenerationIntentMap is the map-backed variant used after service-side request mutation.
|
||||
func IsImageGenerationIntentMap(endpoint string, requestedModel string, reqBody map[string]any) bool {
|
||||
if IsImageGenerationEndpoint(endpoint) {
|
||||
@@ -140,6 +190,18 @@ func openAIJSONToolsContainImageGeneration(tools gjson.Result) bool {
|
||||
return found
|
||||
}
|
||||
|
||||
func openAIJSONToolsContainNativeImageGeneration(tools gjson.Result) bool {
|
||||
if !tools.IsArray() {
|
||||
return false
|
||||
}
|
||||
found := false
|
||||
tools.ForEach(func(_, item gjson.Result) bool {
|
||||
found = isOpenAIImageGenerationType(openAIJSONString(item.Get("type")))
|
||||
return !found
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
func isOpenAIImageGenerationType(value string) bool {
|
||||
return strings.TrimSpace(value) == "image_generation"
|
||||
}
|
||||
@@ -148,9 +210,8 @@ func isOpenAIImageGenNamespaceName(value string) bool {
|
||||
return strings.TrimSpace(value) == "image_gen"
|
||||
}
|
||||
|
||||
// isImageGenNamespaceTool detects the Codex namespace-style image generation
|
||||
// tool declaration: { "type": "namespace", "name": "image_gen", ... }.
|
||||
// Codex /image uses this instead of the flat { "type": "image_generation" }.
|
||||
// isImageGenNamespaceTool detects the namespace advertised by Codex's built-in
|
||||
// image-generation extension instead of a hosted image_generation tool.
|
||||
func isImageGenNamespaceTool(tool gjson.Result) bool {
|
||||
return openAIJSONString(tool.Get("type")) == "namespace" &&
|
||||
isOpenAIImageGenNamespaceName(openAIJSONString(tool.Get("name")))
|
||||
@@ -235,6 +296,45 @@ func openAIJSONToolChoiceSelectsImageGeneration(choice gjson.Result) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func openAIJSONToolChoiceSelectsExplicitImageGeneration(choice gjson.Result) bool {
|
||||
if openAIJSONToolChoiceSelectsImageGeneration(choice) {
|
||||
return true
|
||||
}
|
||||
if !choice.IsObject() {
|
||||
return false
|
||||
}
|
||||
if tool := choice.Get("tool"); tool.IsObject() && openAIJSONToolChoiceSelectsExplicitImageGeneration(tool) {
|
||||
return true
|
||||
}
|
||||
if isOpenAIImageGenFunctionReference(
|
||||
openAIJSONString(choice.Get("namespace")),
|
||||
openAIJSONString(choice.Get("name")),
|
||||
) {
|
||||
return true
|
||||
}
|
||||
if fn := choice.Get("function"); fn.IsObject() {
|
||||
return isOpenAIImageGenFunctionReference(
|
||||
openAIJSONString(fn.Get("namespace")),
|
||||
openAIJSONString(fn.Get("name")),
|
||||
)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isOpenAIImageGenFunctionReference(namespace string, name string) bool {
|
||||
namespace = strings.TrimSpace(namespace)
|
||||
name = strings.TrimSpace(name)
|
||||
if namespace == "image_gen" && name == "imagegen" {
|
||||
return true
|
||||
}
|
||||
switch name {
|
||||
case "image_gen.imagegen", "image_gen__imagegen":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func openAIAnyToolChoiceSelectsImageGeneration(choice any) bool {
|
||||
switch v := choice.(type) {
|
||||
case string:
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsImageGenerationIntentForPlatform_GrokCodexDeclarations(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "top-level image_gen namespace is passive",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`,
|
||||
},
|
||||
{
|
||||
name: "responses lite additional_tools image_gen is passive",
|
||||
body: `{"model":"grok-4.5","tool_choice":"auto","input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]},{"type":"message","role":"user","content":"write code"}]}`,
|
||||
},
|
||||
{
|
||||
name: "legacy adapter flattened image_gen function declaration is passive",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"input":"write code"}`,
|
||||
},
|
||||
{
|
||||
name: "native image generation declaration remains explicit",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"input":"draw a cat"}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "native image generation tool choice is explicit",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"tool_choice":{"type":"image_generation"},"input":"draw a cat"}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "image_gen namespace tool choice is explicit",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen"}],"tool_choice":{"type":"namespace","name":"image_gen"},"input":"draw a cat"}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "flattened image_gen function tool choice is explicit",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"tool_choice":{"type":"function","name":"image_gen.imagegen"},"input":"draw a cat"}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "wrapped image_gen function tool choice is explicit",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"tool_choice":{"tool":{"type":"function","name":"image_gen.imagegen"}},"input":"draw a cat"}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "vision input is not image generation",
|
||||
body: `{"model":"grok-4.5","input":[{"type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}]}`,
|
||||
},
|
||||
{
|
||||
name: "ordinary function declaration is not image generation",
|
||||
body: `{"model":"grok-4.5","tools":[{"type":"function","name":"lookup"}],"tool_choice":"auto","input":"write code"}`,
|
||||
},
|
||||
{
|
||||
name: "image_gen function call history is not current intent",
|
||||
body: `{"model":"grok-4.5","input":[{"type":"function_call","namespace":"image_gen","name":"imagegen","arguments":"{}"}]}`,
|
||||
},
|
||||
{
|
||||
name: "image generation call history is not current intent",
|
||||
body: `{"model":"grok-4.5","input":[{"type":"image_generation_call","id":"ig_1"}]}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, IsImageGenerationIntentForPlatform(
|
||||
openAIResponsesEndpoint,
|
||||
"grok-4.5",
|
||||
[]byte(tt.body),
|
||||
PlatformGrok,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsImageGenerationIntentForPlatform_GrokPreservesHardSignals(t *testing.T) {
|
||||
require.True(t, IsImageGenerationIntentForPlatform(
|
||||
"/v1/images/generations",
|
||||
"grok-4.5",
|
||||
[]byte(`{"input":"draw"}`),
|
||||
PlatformGrok,
|
||||
))
|
||||
require.True(t, IsImageGenerationIntentForPlatform(
|
||||
openAIResponsesEndpoint,
|
||||
"gpt-image-2",
|
||||
[]byte(`{"input":"draw"}`),
|
||||
PlatformGrok,
|
||||
))
|
||||
require.True(t, IsImageGenerationIntentForPlatform(
|
||||
openAIResponsesEndpoint,
|
||||
"grok-4.5",
|
||||
[]byte(`{"model":"gpt-image-2","input":"draw"}`),
|
||||
PlatformGrok,
|
||||
))
|
||||
}
|
||||
|
||||
func TestIsImageGenerationIntentForPlatform_OtherPlatformsKeepDeclarationSemantics(t *testing.T) {
|
||||
body := []byte(`{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"input":"write code"}`)
|
||||
|
||||
require.True(t, IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, "gpt-5.5", body, PlatformOpenAI))
|
||||
require.True(t, IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, "gpt-5.5", body, PlatformAnthropic))
|
||||
}
|
||||
@@ -305,7 +305,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
|
||||
normalized = stripped
|
||||
logOpenAIWSModeInfo("ingress_ws_codex_spark_image_tool_stripped account_id=%d", account.ID)
|
||||
}
|
||||
imageIntent := IsImageGenerationIntent(openAIResponsesEndpoint, originalModel, normalized)
|
||||
imageIntent := IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, originalModel, normalized, account.Platform)
|
||||
if imageIntent && !imageGenerationAllowed {
|
||||
return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, ImageGenerationPermissionMessage(), nil)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user