diff --git a/src/agent/agent.ts b/src/agent/agent.ts index 25195cc..564ddb0 100644 --- a/src/agent/agent.ts +++ b/src/agent/agent.ts @@ -46,8 +46,10 @@ import { } from "./compaction"; import { DelegationManager } from "./delegations"; import { containsEncryptedReasoning, sanitizeModelMessages } from "./reasoning"; +import { buildVisionUserMessages } from "./vision-input"; const MAX_TOOL_ROUNDS = 400; +const VISION_MODEL = "grok-4-1-fast-reasoning"; interface AgentOptions { persistSession?: boolean; @@ -256,12 +258,15 @@ function buildSubagentPrompt( subagents?: CustomSubagentConfig[], ): string { const isExplore = request.agent === "explore"; + const isVision = request.agent === "vision"; const mode: AgentMode = isExplore ? "ask" : "agent"; const role = custom ? `You are the custom sub-agent "${custom.name}". You can investigate, edit files, and run commands unless the delegated task says otherwise.` : request.agent === "explore" ? "You are the Explore sub-agent. You are read-only and focus on fast codebase research." - : "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work."; + : isVision + ? "You are the Vision sub-agent." + : "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work."; const rules = isExplore ? [ @@ -269,11 +274,13 @@ function buildSubagentPrompt( "Prefer `read_file` and search commands over broad shell exploration.", "Return concise findings for the parent agent.", ] - : [ - "Work only on the delegated task below.", - "Use tools directly instead of narrating your intent.", - "Return a concise summary for the parent agent with key outcomes and any open risks.", - ]; + : isVision + ? ["Validate the image."] + : [ + "Work only on the delegated task below.", + "Use tools directly instead of narrating your intent.", + "Return a concise summary for the parent agent with key outcomes and any open risks.", + ]; const instructionLines = custom?.instruction.trim() ? ["", "SUB-AGENT INSTRUCTIONS:", custom.instruction.trim()] : []; @@ -550,11 +557,12 @@ export class Agent { const agentKey = String(request.agent); const isExplore = agentKey === "explore"; const isGeneral = agentKey === "general"; + const isVision = agentKey === "vision"; const subagents = loadValidSubAgents(); - const custom = !isExplore && !isGeneral ? findCustomSubagent(agentKey, subagents) : undefined; + const custom = !isExplore && !isGeneral && !isVision ? findCustomSubagent(agentKey, subagents) : undefined; - if (!isExplore && !isGeneral && !custom) { - const message = `Unknown sub-agent "${agentKey}". Use general, explore, or a configured name from ~/.grok/user-settings.json.`; + if (!isExplore && !isGeneral && !isVision && !custom) { + const message = `Unknown sub-agent "${agentKey}". Use general, explore, vision, or a configured name from ~/.grok/user-settings.json.`; return { success: false, output: message, @@ -574,8 +582,12 @@ export class Agent { let lastActivity = initialDetail; let childTools: ToolSet = childBaseTools; let closeMcp: (() => Promise) | undefined; - const childModelId = normalizeModelId(isExplore ? DEFAULT_MODEL : custom ? custom.model : this.modelId); - const childRuntime = resolveModelRuntime(provider, childModelId); + const childModelId = normalizeModelId( + isVision ? VISION_MODEL : isExplore ? DEFAULT_MODEL : custom ? custom.model : this.modelId, + ); + const childRuntime = isVision + ? { ...resolveModelRuntime(provider, childModelId), model: provider.responses(childModelId) } + : resolveModelRuntime(provider, childModelId); const childSystem = applyModelConstraints( buildSubagentPrompt(request, childBash.getCwd(), custom ?? null, subagents), childRuntime.modelId, @@ -594,10 +606,14 @@ export class Agent { } } + const childMessages = isVision + ? await buildVisionUserMessages(request.prompt, childBash.getCwd(), signal) + : [{ role: "user" as const, content: request.prompt }]; + const result = streamText({ model: childRuntime.model, system: childSystem, - messages: [{ role: "user", content: request.prompt }], + messages: childMessages, tools: childRuntime.modelInfo?.supportsClientTools === false ? {} : childTools, stopWhen: stepCountIs(Math.min(this.maxToolRounds, isExplore ? 60 : 120)), maxRetries: 0, diff --git a/src/agent/vision-input.test.ts b/src/agent/vision-input.test.ts new file mode 100644 index 0000000..7b7138e --- /dev/null +++ b/src/agent/vision-input.test.ts @@ -0,0 +1,39 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { buildVisionUserMessages } from "./vision-input"; + +describe("buildVisionUserMessages", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "grok-vision-")); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it("builds a multimodal user message when the prompt contains a local image path", async () => { + const imagePath = path.join(tempDir, "screen.png"); + fs.writeFileSync(imagePath, Buffer.from([1, 2, 3, 4])); + + const messages = await buildVisionUserMessages(`Validate the image at ${imagePath}`, tempDir); + + expect(messages).toHaveLength(1); + expect(messages[0]?.role).toBe("user"); + expect(Array.isArray(messages[0]?.content)).toBe(true); + + const content = messages[0]?.content as Array>; + expect(content[0]).toMatchObject({ + type: "file", + mediaType: "image/png", + }); + expect(content[0]?.data).toBeInstanceOf(Uint8Array); + expect(content[1]).toMatchObject({ + type: "text", + text: `Validate the image at ${imagePath}`, + }); + }); +}); diff --git a/src/agent/vision-input.ts b/src/agent/vision-input.ts new file mode 100644 index 0000000..5a80b47 --- /dev/null +++ b/src/agent/vision-input.ts @@ -0,0 +1,118 @@ +import type { ModelMessage } from "ai"; +import { existsSync, readFileSync } from "fs"; +import { extname, isAbsolute, resolve } from "path"; + +export async function buildVisionUserMessages( + prompt: string, + cwd: string, + abortSignal?: AbortSignal, +): Promise { + const imageSources = extractImageSourcesFromPrompt(prompt); + if (imageSources.length === 0) { + return [{ role: "user", content: prompt }]; + } + + const content: Array<{ type: "file"; data: Uint8Array | URL; mediaType: string } | { type: "text"; text: string }> = + []; + + for (const source of imageSources) { + const resolved = await resolveVisionImageSource(source, cwd, abortSignal).catch(() => null); + if (!resolved) continue; + content.push({ + type: "file", + data: resolved.sourceUrl ? new URL(resolved.sourceUrl) : resolved.data, + mediaType: resolved.mediaType, + }); + } + + if (content.length === 0) { + return [{ role: "user", content: prompt }]; + } + + content.push({ type: "text", text: prompt }); + return [{ role: "user", content }]; +} + +function extractImageSourcesFromPrompt(prompt: string): string[] { + const matches = prompt.match(/((?:file:\/\/|https?:\/\/|\/)[^\n]*?\.(?:png|jpe?g))/gi) ?? []; + const seen = new Set(); + const sources: string[] = []; + + for (const match of matches) { + const normalized = normalizeImageSourceText(match); + if (!normalized || seen.has(normalized)) continue; + seen.add(normalized); + sources.push(normalized); + } + + return sources; +} + +async function resolveVisionImageSource( + source: string, + cwd: string, + abortSignal?: AbortSignal, +): Promise<{ data: Uint8Array; mediaType: string; sourceUrl?: string }> { + if (isHttpUrl(source)) { + const response = await fetch(source, { signal: abortSignal }); + if (!response.ok) { + throw new Error(`Source image download failed: ${response.status} ${response.statusText}`); + } + const data = new Uint8Array(await response.arrayBuffer()); + return { + data, + mediaType: response.headers.get("content-type") || guessImageMediaType(source), + sourceUrl: source, + }; + } + + const localPath = source.startsWith("file://") ? decodeFileUrl(source) : resolveLocalImagePath(source, cwd); + if (!existsSync(localPath)) { + throw new Error(`Source image not found: ${source}`); + } + + return { + data: readFileSync(localPath), + mediaType: guessImageMediaType(localPath), + }; +} + +function normalizeImageSourceText(value: string): string { + return value + .trim() + .replace(/^["']|["']$/g, "") + .replace(/\\([ "'()[\]{}])/g, "$1"); +} + +function isHttpUrl(value: string): boolean { + try { + const parsed = new URL(value); + return parsed.protocol === "http:" || parsed.protocol === "https:"; + } catch { + return false; + } +} + +function decodeFileUrl(value: string): string { + try { + return decodeURIComponent(new URL(value).pathname); + } catch { + return value; + } +} + +function resolveLocalImagePath(source: string, cwd: string): string { + return isAbsolute(source) ? source : resolve(cwd, source); +} + +function guessImageMediaType(pathLike: string): string { + switch (extname(pathLike).toLowerCase()) { + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".png": + return "image/png"; + default: + return "image/png"; + } +} diff --git a/src/grok/tools.ts b/src/grok/tools.ts index 94794db..a9f5570 100644 --- a/src/grok/tools.ts +++ b/src/grok/tools.ts @@ -228,21 +228,21 @@ export function createTools( if (options.runTask) { const customNames = (options.subagents ?? loadValidSubAgents()).map((agent) => agent.name); - const taskAgentEnum = ["general", "explore", ...customNames] as [string, ...string[]]; + const taskAgentEnum = ["general", "explore", "vision", ...customNames] as [string, ...string[]]; const customHint = customNames.length > 0 ? ` You may also use these user-defined sub-agents by exact name: ${customNames.join(", ")}.` : ""; tools.task = tool({ - description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, and code quality work instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution and \`explore\` for fast read-only investigation.${customHint} Provide a short description plus a detailed prompt for the child agent.`, + description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, and code quality work instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution, \`explore\` for fast read-only investigation, and \`vision\` for image validation.${customHint} Provide a short description plus a detailed prompt for the child agent.`, inputSchema: z.object({ agent: z .enum(taskAgentEnum) .default("general") .describe( customNames.length > 0 - ? "Built-in general or explore, or a configured custom sub-agent name from user settings" + ? "Built-in general, explore, or vision, or a configured custom sub-agent name from user settings" : "Which sub-agent to use", ), description: z.string().describe("A short label for the delegated task, such as 'Deep code quality analysis'"), diff --git a/src/types/index.ts b/src/types/index.ts index e253557..cbcf34b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -29,7 +29,7 @@ export interface Plan { questions?: PlanQuestion[]; } -export type BuiltinSubagentId = "general" | "explore"; +export type BuiltinSubagentId = "general" | "explore" | "vision"; export interface TaskRequest { agent: BuiltinSubagentId | string; diff --git a/src/ui/app.tsx b/src/ui/app.tsx index 3155b49..f39921b 100644 --- a/src/ui/app.tsx +++ b/src/ui/app.tsx @@ -3444,6 +3444,7 @@ function InlineTool({ t, pending: _pending, children }: { t: Theme; pending: boo function formatSubagentName(agent: string): string { if (agent === "general") return "General"; if (agent === "explore") return "Explore"; + if (agent === "vision") return "Vision"; return agent || "Sub-agent"; } diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 364dda0..6de5d4b 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -43,7 +43,7 @@ export interface CustomSubagentConfig { instruction: string; } -const RESERVED_SUBAGENT_NAMES = new Set(["general", "explore"]); +const RESERVED_SUBAGENT_NAMES = new Set(["general", "explore", "vision"]); export function isReservedSubagentName(name: string): boolean { return RESERVED_SUBAGENT_NAMES.has(name.trim().toLowerCase()); diff --git a/src/utils/subagents-settings.test.ts b/src/utils/subagents-settings.test.ts index ba8a7ff..a80499e 100644 --- a/src/utils/subagents-settings.test.ts +++ b/src/utils/subagents-settings.test.ts @@ -33,6 +33,7 @@ describe("parseSubAgentsRawList", () => { parseSubAgentsRawList([ { name: "general", model: "grok-4-1-fast-reasoning", instruction: "x" }, { name: "Explore", model: "grok-4-1-fast-reasoning", instruction: "x" }, + { name: "vision", model: "grok-4-1-fast-reasoning", instruction: "x" }, { name: "", model: "grok-4-1-fast-reasoning", instruction: "x" }, { name: " ", model: "grok-4-1-fast-reasoning", instruction: "x" }, ]),