feat: add built-in Vision sub-agent (#209)

Add a default Vision sub-agent and wire its image-validation requests through the xAI Responses runtime using the fast reasoning model.

src/agent/agent.ts

@@ -46,8 +46,10 @@ import {
 } from "./compaction";
 import { DelegationManager } from "./delegations";
 import { containsEncryptedReasoning, sanitizeModelMessages } from "./reasoning";
+import { buildVisionUserMessages } from "./vision-input";
 
 const MAX_TOOL_ROUNDS = 400;
+const VISION_MODEL = "grok-4-1-fast-reasoning";
 
 interface AgentOptions {
   persistSession?: boolean;
@@ -256,12 +258,15 @@ function buildSubagentPrompt(
   subagents?: CustomSubagentConfig[],
 ): string {
   const isExplore = request.agent === "explore";
+  const isVision = request.agent === "vision";
   const mode: AgentMode = isExplore ? "ask" : "agent";
   const role = custom
     ? `You are the custom sub-agent "${custom.name}". You can investigate, edit files, and run commands unless the delegated task says otherwise.`
     : request.agent === "explore"
       ? "You are the Explore sub-agent. You are read-only and focus on fast codebase research."
-      : "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work.";
+      : isVision
+        ? "You are the Vision sub-agent."
+        : "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work.";
 
   const rules = isExplore
     ? [
@@ -269,11 +274,13 @@ function buildSubagentPrompt(
         "Prefer `read_file` and search commands over broad shell exploration.",
         "Return concise findings for the parent agent.",
       ]
-    : [
-        "Work only on the delegated task below.",
-        "Use tools directly instead of narrating your intent.",
-        "Return a concise summary for the parent agent with key outcomes and any open risks.",
-      ];
+    : isVision
+      ? ["Validate the image."]
+      : [
+          "Work only on the delegated task below.",
+          "Use tools directly instead of narrating your intent.",
+          "Return a concise summary for the parent agent with key outcomes and any open risks.",
+        ];
 
   const instructionLines = custom?.instruction.trim() ? ["", "SUB-AGENT INSTRUCTIONS:", custom.instruction.trim()] : [];
 
@@ -550,11 +557,12 @@ export class Agent {
     const agentKey = String(request.agent);
     const isExplore = agentKey === "explore";
     const isGeneral = agentKey === "general";
+    const isVision = agentKey === "vision";
     const subagents = loadValidSubAgents();
-    const custom = !isExplore && !isGeneral ? findCustomSubagent(agentKey, subagents) : undefined;
+    const custom = !isExplore && !isGeneral && !isVision ? findCustomSubagent(agentKey, subagents) : undefined;
 
-    if (!isExplore && !isGeneral && !custom) {
-      const message = `Unknown sub-agent "${agentKey}". Use general, explore, or a configured name from ~/.grok/user-settings.json.`;
+    if (!isExplore && !isGeneral && !isVision && !custom) {
+      const message = `Unknown sub-agent "${agentKey}". Use general, explore, vision, or a configured name from ~/.grok/user-settings.json.`;
       return {
         success: false,
         output: message,
@@ -574,8 +582,12 @@ export class Agent {
     let lastActivity = initialDetail;
     let childTools: ToolSet = childBaseTools;
     let closeMcp: (() => Promise<void>) | undefined;
-    const childModelId = normalizeModelId(isExplore ? DEFAULT_MODEL : custom ? custom.model : this.modelId);
-    const childRuntime = resolveModelRuntime(provider, childModelId);
+    const childModelId = normalizeModelId(
+      isVision ? VISION_MODEL : isExplore ? DEFAULT_MODEL : custom ? custom.model : this.modelId,
+    );
+    const childRuntime = isVision
+      ? { ...resolveModelRuntime(provider, childModelId), model: provider.responses(childModelId) }
+      : resolveModelRuntime(provider, childModelId);
     const childSystem = applyModelConstraints(
       buildSubagentPrompt(request, childBash.getCwd(), custom ?? null, subagents),
       childRuntime.modelId,
@@ -594,10 +606,14 @@ export class Agent {
         }
       }
 
+      const childMessages = isVision
+        ? await buildVisionUserMessages(request.prompt, childBash.getCwd(), signal)
+        : [{ role: "user" as const, content: request.prompt }];
+
       const result = streamText({
         model: childRuntime.model,
         system: childSystem,
-        messages: [{ role: "user", content: request.prompt }],
+        messages: childMessages,
         tools: childRuntime.modelInfo?.supportsClientTools === false ? {} : childTools,
         stopWhen: stepCountIs(Math.min(this.maxToolRounds, isExplore ? 60 : 120)),
         maxRetries: 0,

src/agent/vision-input.test.ts

@@ -0,0 +1,39 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { buildVisionUserMessages } from "./vision-input";
+
+describe("buildVisionUserMessages", () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "grok-vision-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it("builds a multimodal user message when the prompt contains a local image path", async () => {
+    const imagePath = path.join(tempDir, "screen.png");
+    fs.writeFileSync(imagePath, Buffer.from([1, 2, 3, 4]));
+
+    const messages = await buildVisionUserMessages(`Validate the image at ${imagePath}`, tempDir);
+
+    expect(messages).toHaveLength(1);
+    expect(messages[0]?.role).toBe("user");
+    expect(Array.isArray(messages[0]?.content)).toBe(true);
+
+    const content = messages[0]?.content as Array<Record<string, unknown>>;
+    expect(content[0]).toMatchObject({
+      type: "file",
+      mediaType: "image/png",
+    });
+    expect(content[0]?.data).toBeInstanceOf(Uint8Array);
+    expect(content[1]).toMatchObject({
+      type: "text",
+      text: `Validate the image at ${imagePath}`,
+    });
+  });
+});

src/agent/vision-input.ts

@@ -0,0 +1,118 @@
+import type { ModelMessage } from "ai";
+import { existsSync, readFileSync } from "fs";
+import { extname, isAbsolute, resolve } from "path";
+
+export async function buildVisionUserMessages(
+  prompt: string,
+  cwd: string,
+  abortSignal?: AbortSignal,
+): Promise<ModelMessage[]> {
+  const imageSources = extractImageSourcesFromPrompt(prompt);
+  if (imageSources.length === 0) {
+    return [{ role: "user", content: prompt }];
+  }
+
+  const content: Array<{ type: "file"; data: Uint8Array | URL; mediaType: string } | { type: "text"; text: string }> =
+    [];
+
+  for (const source of imageSources) {
+    const resolved = await resolveVisionImageSource(source, cwd, abortSignal).catch(() => null);
+    if (!resolved) continue;
+    content.push({
+      type: "file",
+      data: resolved.sourceUrl ? new URL(resolved.sourceUrl) : resolved.data,
+      mediaType: resolved.mediaType,
+    });
+  }
+
+  if (content.length === 0) {
+    return [{ role: "user", content: prompt }];
+  }
+
+  content.push({ type: "text", text: prompt });
+  return [{ role: "user", content }];
+}
+
+function extractImageSourcesFromPrompt(prompt: string): string[] {
+  const matches = prompt.match(/((?:file:\/\/|https?:\/\/|\/)[^\n]*?\.(?:png|jpe?g))/gi) ?? [];
+  const seen = new Set<string>();
+  const sources: string[] = [];
+
+  for (const match of matches) {
+    const normalized = normalizeImageSourceText(match);
+    if (!normalized || seen.has(normalized)) continue;
+    seen.add(normalized);
+    sources.push(normalized);
+  }
+
+  return sources;
+}
+
+async function resolveVisionImageSource(
+  source: string,
+  cwd: string,
+  abortSignal?: AbortSignal,
+): Promise<{ data: Uint8Array; mediaType: string; sourceUrl?: string }> {
+  if (isHttpUrl(source)) {
+    const response = await fetch(source, { signal: abortSignal });
+    if (!response.ok) {
+      throw new Error(`Source image download failed: ${response.status} ${response.statusText}`);
+    }
+    const data = new Uint8Array(await response.arrayBuffer());
+    return {
+      data,
+      mediaType: response.headers.get("content-type") || guessImageMediaType(source),
+      sourceUrl: source,
+    };
+  }
+
+  const localPath = source.startsWith("file://") ? decodeFileUrl(source) : resolveLocalImagePath(source, cwd);
+  if (!existsSync(localPath)) {
+    throw new Error(`Source image not found: ${source}`);
+  }
+
+  return {
+    data: readFileSync(localPath),
+    mediaType: guessImageMediaType(localPath),
+  };
+}
+
+function normalizeImageSourceText(value: string): string {
+  return value
+    .trim()
+    .replace(/^["']|["']$/g, "")
+    .replace(/\\([ "'()[\]{}])/g, "$1");
+}
+
+function isHttpUrl(value: string): boolean {
+  try {
+    const parsed = new URL(value);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+
+function decodeFileUrl(value: string): string {
+  try {
+    return decodeURIComponent(new URL(value).pathname);
+  } catch {
+    return value;
+  }
+}
+
+function resolveLocalImagePath(source: string, cwd: string): string {
+  return isAbsolute(source) ? source : resolve(cwd, source);
+}
+
+function guessImageMediaType(pathLike: string): string {
+  switch (extname(pathLike).toLowerCase()) {
+    case ".jpg":
+    case ".jpeg":
+      return "image/jpeg";
+    case ".png":
+      return "image/png";
+    default:
+      return "image/png";
+  }
+}

src/grok/tools.ts

@@ -228,21 +228,21 @@ export function createTools(
 
   if (options.runTask) {
     const customNames = (options.subagents ?? loadValidSubAgents()).map((agent) => agent.name);
-    const taskAgentEnum = ["general", "explore", ...customNames] as [string, ...string[]];
+    const taskAgentEnum = ["general", "explore", "vision", ...customNames] as [string, ...string[]];
     const customHint =
       customNames.length > 0
         ? ` You may also use these user-defined sub-agents by exact name: ${customNames.join(", ")}.`
         : "";
 
     tools.task = tool({
-      description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, and code quality work instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution and \`explore\` for fast read-only investigation.${customHint} Provide a short description plus a detailed prompt for the child agent.`,
+      description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, and code quality work instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution, \`explore\` for fast read-only investigation, and \`vision\` for image validation.${customHint} Provide a short description plus a detailed prompt for the child agent.`,
       inputSchema: z.object({
         agent: z
           .enum(taskAgentEnum)
           .default("general")
           .describe(
             customNames.length > 0
-              ? "Built-in general or explore, or a configured custom sub-agent name from user settings"
+              ? "Built-in general, explore, or vision, or a configured custom sub-agent name from user settings"
               : "Which sub-agent to use",
           ),
         description: z.string().describe("A short label for the delegated task, such as 'Deep code quality analysis'"),

src/types/index.ts

@@ -29,7 +29,7 @@ export interface Plan {
   questions?: PlanQuestion[];
 }
 
-export type BuiltinSubagentId = "general" | "explore";
+export type BuiltinSubagentId = "general" | "explore" | "vision";
 
 export interface TaskRequest {
   agent: BuiltinSubagentId | string;

src/ui/app.tsx

@@ -3444,6 +3444,7 @@ function InlineTool({ t, pending: _pending, children }: { t: Theme; pending: boo
 function formatSubagentName(agent: string): string {
   if (agent === "general") return "General";
   if (agent === "explore") return "Explore";
+  if (agent === "vision") return "Vision";
   return agent || "Sub-agent";
 }
 

src/utils/settings.ts

@@ -43,7 +43,7 @@ export interface CustomSubagentConfig {
   instruction: string;
 }
 
-const RESERVED_SUBAGENT_NAMES = new Set(["general", "explore"]);
+const RESERVED_SUBAGENT_NAMES = new Set(["general", "explore", "vision"]);
 
 export function isReservedSubagentName(name: string): boolean {
   return RESERVED_SUBAGENT_NAMES.has(name.trim().toLowerCase());

src/utils/subagents-settings.test.ts

@@ -33,6 +33,7 @@ describe("parseSubAgentsRawList", () => {
       parseSubAgentsRawList([
         { name: "general", model: "grok-4-1-fast-reasoning", instruction: "x" },
         { name: "Explore", model: "grok-4-1-fast-reasoning", instruction: "x" },
+        { name: "vision", model: "grok-4-1-fast-reasoning", instruction: "x" },
         { name: "", model: "grok-4-1-fast-reasoning", instruction: "x" },
         { name: "  ", model: "grok-4-1-fast-reasoning", instruction: "x" },
       ]),