diff --git a/CHANGELOG.md b/CHANGELOG.md index 355bc29..bff3ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,18 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added -- Built-in `telegram-bridge` command for headless Telegram bridge entrypoint with shared approval persistence (#211) -- Add CLI update checker (#223) -- Add verify workflow +## [1.1.3] - 2026-04-01 -### Fixed -- Issue with schedule modal (#226) -- Host-side browser smoke tests, skill inheritance, and screenshot paths +### Added +- @-mention file autocomplete (#236) + +## [1.1.2] - 2026-04-01 + +### Added +- Switch computer sub-agent to agent-desktop (#233) + +### Removed +- Tracked telegram pair code from repo (#234) + +## [1.1.1] - 2026-04-01 + +### Added +- Verify workflow with sandboxed testing and browser smoke checks (#228) +- Batch mode for headless Grok CLI runs (#231) + +## [1.1.0] - 2026-03-26 + +### Added +- CLI update checker (#223) ### Changed - Replace commit scan with PR security scan (#224) +### Fixed +- Issue with schedule modal (#226) + ## [1.0.0-rc7] - 2026-03-26 ### Added diff --git a/package.json b/package.json index 7d684ca..7a9417a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grok-dev", - "version": "1.1.3", + "version": "1.1.4", "description": "An open-source AI coding agent powered by Grok, built with Bun and OpenTUI.", "type": "module", "main": "dist/index.js", diff --git a/src/agent/agent.ts b/src/agent/agent.ts index cf8f7a2..9f6b3df 100644 --- a/src/agent/agent.ts +++ b/src/agent/agent.ts @@ -55,7 +55,8 @@ import { type SandboxSettings, } from "../utils/settings"; import { discoverSkills, formatSkillsForPrompt } from "../utils/skills"; -import { buildVerifyDetectPrompt, normalizeVerifyRecipe } from "../verify/entrypoint"; +import { buildVerifyDetectPrompt, normalizeVerifyRecipe, prepareVerifySandbox } from "../verify/entrypoint"; +import { runVerifyOrchestration } from "../verify/orchestrator"; import { type CompactionSettings, createCompactionSummaryMessage, @@ -322,6 +323,7 @@ function buildSubagentPrompt( const isVision = request.agent === "vision"; const isVerify = request.agent === "verify"; const isVerifyDetect = request.agent === "verify-detect"; + const isVerifyManifest = request.agent === "verify-manifest"; const isComputer = request.agent === "computer"; const mode: AgentMode = isExplore || isVerifyDetect ? "ask" : "agent"; const role = custom @@ -332,11 +334,13 @@ function buildSubagentPrompt( ? "You are the Vision sub-agent." : isVerifyDetect ? "You are the Verify Detect sub-agent. You inspect a repository to produce a structured verification recipe. You are read-only." - : isVerify - ? "You are the Verify sub-agent. You specialize in sandbox-aware local verification using builds, tests, app boot checks, and optional browser smoke tests." - : isComputer - ? "You are the Computer sub-agent. You specialize in host desktop automation using accessibility snapshots, semantic element refs, screenshots, and careful mouse and keyboard actions." - : "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work."; + : isVerifyManifest + ? "You are the Verify Manifest sub-agent. You inspect a repository and create or update .grok/environment.json so verification can run reproducibly." + : isVerify + ? "You are the Verify sub-agent. You specialize in sandbox-aware local verification using builds, tests, app boot checks, and optional browser smoke tests." + : isComputer + ? "You are the Computer sub-agent. You specialize in host desktop automation using accessibility snapshots, semantic element refs, screenshots, and careful mouse and keyboard actions." + : "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work."; const rules = isExplore ? [ @@ -350,31 +354,76 @@ function buildSubagentPrompt( "Read config files, package manifests, scripts, and source layout to understand the project.", "Return ONLY a valid JSON object with the VerifyRecipe schema. No markdown, no prose, no explanation outside the JSON.", ] - : isVision - ? ["Validate the image."] - : isComputer - ? [ - "Operate carefully on the HOST desktop, not inside the shell sandbox.", - "Start with `computer_snapshot` when possible. It returns stable refs like @e1 that remain valid until the next snapshot.", - "Prefer accessibility refs over coordinates. Use `computer_click`, `computer_type`, `computer_scroll`, and `computer_get` with refs from the latest snapshot.", - "After any meaningful UI transition, launch, dialog open, or menu change, take another `computer_snapshot` before reusing old refs.", - "Use `computer_launch`, `computer_list_windows`, `computer_focus_window`, and `computer_wait` to manage apps and window state.", - "Use `computer_press` for shortcuts like Enter or cmd+k. Use `computer_screenshot` only for visual confirmation or when the accessibility tree is insufficient.", - "If `agent-desktop` is unavailable, permissions are missing, refs go stale, or the state is ambiguous, stop and return the blocker clearly to the parent agent.", - "Do not perform destructive or high-risk desktop actions unless the delegated task explicitly requires them.", - ] - : isVerify + : isVerifyManifest + ? [ + "Focus on creating or updating .grok/environment.json as the primary verification contract for this repository.", + "Read package.json and key config files to understand the project, then write .grok/environment.json.", + "Prefer editing only .grok/environment.json unless the delegated task explicitly requires something else.", + "", + "SANDBOX ENVIRONMENT (Shuru):", + "- OS: Debian GNU/Linux 13 (trixie)", + "- Architecture: aarch64 (ARM64)", + "- Pre-installed: NOTHING. No node, npm, npx, bun, python3, pip, go, cargo, java, or any runtime.", + "- Only basic system tools exist (sh, apt-get, curl, etc).", + "- Network access is available during bootstrap and install.", + "- The workspace is mounted at /workspace.", + "", + "MANIFEST REQUIREMENTS:", + "- bootstrapCommands: MUST install every runtime and build tool the project needs from scratch via apt-get or curl.", + "- For Node.js/Next.js/Vite/etc: `apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm`", + "- For Bun projects: also `curl -fsSL https://bun.sh/install | bash` and shellInitCommands with BUN_INSTALL/PATH exports.", + "- For Python: `apt-get update && apt-get install -y python3 python3-pip python3-venv ca-certificates git`", + "- For Go: `apt-get update && apt-get install -y golang ca-certificates git`", + "- For Rust: `apt-get update && apt-get install -y curl ca-certificates git build-essential && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y`", + "- installCommands: The package install command (npm install, pip install, etc).", + "- buildCommands: Build commands if applicable.", + "- testCommands: Test/lint commands if applicable.", + "- startCommand + startPort: How to start the app for smoke testing.", + "- smokeKind: 'http' if the app has a web UI, 'cli' for CLI tools, 'none' otherwise.", + "- Do NOT leave bootstrapCommands empty. The sandbox has nothing.", + "", + "Return a concise summary of what you wrote and why.", + ] + : isVision + ? ["Validate the image."] + : isComputer ? [ - "Focus on verification first. Do not make durable source edits unless the delegated task explicitly asks for fixes.", - "Prefer the smallest meaningful set of validation commands and explain any environment blockers clearly.", - "IMPORTANT: When the recipe includes a smoke target URL and a forwarded port, you MUST attempt browser smoke testing using agent-browser via the bash tool. The agent-browser command runs on the HOST, not inside the sandbox. It will work even in sandbox mode. Do not skip it or assume it is unavailable. Just run the command.", - "Return a concise structured verification report for the parent agent.", + "Operate carefully on the HOST desktop, not inside the shell sandbox.", + "Start with `computer_snapshot` when possible. It returns stable refs like @e1 that remain valid until the next snapshot.", + "Prefer accessibility refs over coordinates. Use `computer_click`, `computer_type`, `computer_scroll`, and `computer_get` with refs from the latest snapshot.", + "After any meaningful UI transition, launch, dialog open, or menu change, take another `computer_snapshot` before reusing old refs.", + "Use `computer_launch`, `computer_list_windows`, `computer_focus_window`, and `computer_wait` to manage apps and window state.", + "Use `computer_press` for shortcuts like Enter or cmd+k. Use `computer_screenshot` only for visual confirmation or when the accessibility tree is insufficient.", + "If `agent-desktop` is unavailable, permissions are missing, refs go stale, or the state is ambiguous, stop and return the blocker clearly to the parent agent.", + "Do not perform destructive or high-risk desktop actions unless the delegated task explicitly requires them.", ] - : [ - "Work only on the delegated task below.", - "Use tools directly instead of narrating your intent.", - "Return a concise summary for the parent agent with key outcomes and any open risks.", - ]; + : isVerify + ? [ + "You are a QA engineer. Your job is to prove the app works end-to-end, not just that it builds.", + "Do not make durable source edits unless the delegated task explicitly asks for fixes.", + "", + "MANDATORY VERIFICATION STEPS (do ALL of these in order):", + "1. Install dependencies (run installCommands from the recipe).", + "2. Build the project (run buildCommands from the recipe).", + "3. Run tests/lint if available (run testCommands from the recipe).", + "4. Start the app (run startCommand from the recipe in the background).", + "5. Wait for the app to be ready (curl readiness check or agent-browser wait).", + "6. Run browser smoke tests like a real human QA tester:", + " - Open the app in the browser, record a video, take screenshots.", + " - Navigate the app: click links, buttons, menus. Verify pages load.", + " - Check for JavaScript console errors.", + " - Spend 3-5 interactions testing the critical path.", + "7. Stop recording, close browser, then stop the dev server.", + "", + "Do NOT stop after build/lint. Starting the app and testing it in the browser is the most important part.", + "agent-browser commands run on the HOST, not inside the sandbox. They WILL work. Do not skip them.", + "Return a concise verification report. Keep it compact but always include Evidence with artifact file paths.", + ] + : [ + "Work only on the delegated task below.", + "Use tools directly instead of narrating your intent.", + "Return a concise summary for the parent agent with key outcomes and any open risks.", + ]; const instructionLines = custom?.instruction.trim() ? ["", "SUB-AGENT INSTRUCTIONS:", custom.instruction.trim()] : []; @@ -952,15 +1001,25 @@ export class Agent { const isVision = agentKey === "vision"; const isVerify = agentKey === "verify"; const isVerifyDetect = agentKey === "verify-detect"; + const isVerifyManifest = agentKey === "verify-manifest"; const isComputer = agentKey === "computer"; const subagents = loadValidSubAgents(); const custom = - !isExplore && !isGeneral && !isVision && !isVerify && !isVerifyDetect && !isComputer + !isExplore && !isGeneral && !isVision && !isVerify && !isVerifyDetect && !isVerifyManifest && !isComputer ? findCustomSubagent(agentKey, subagents) : undefined; - if (!isExplore && !isGeneral && !isVision && !isVerify && !isVerifyDetect && !isComputer && !custom) { - const message = `Unknown sub-agent "${agentKey}". Use general, explore, vision, verify, computer, or a configured name from ~/.grok/user-settings.json.`; + if ( + !isExplore && + !isGeneral && + !isVision && + !isVerify && + !isVerifyDetect && + !isVerifyManifest && + !isComputer && + !custom + ) { + const message = `Unknown sub-agent "${agentKey}". Use general, explore, vision, verify, verify-detect, verify-manifest, computer, or a configured name from ~/.grok/user-settings.json.`; return { success: false, output: message, @@ -976,10 +1035,22 @@ export class Agent { const verifySandboxOverrides: SandboxSettings = isVerify ? { allowNet: true, allowedHosts: undefined, allowEphemeralInstall: true, hostBrowserCommandsOnHost: true } : {}; + let verifyPreparedSettings: SandboxSettings | null = null; + let verifyPreparedRecipe: VerifyRecipe | null = null; + if (isVerify) { + const prepared = await prepareVerifySandbox( + this.bash.getCwd(), + { ...this.bash.getSandboxSettings(), ...verifySandboxOverrides }, + undefined, + onActivity, + ); + verifyPreparedSettings = prepared.sandboxSettings; + verifyPreparedRecipe = prepared.profile.recipe; + } const childBash = new BashTool(this.bash.getCwd(), { sandboxMode: isVerify ? "shuru" : this.bash.getSandboxMode(), sandboxSettings: isVerify - ? { ...this.bash.getSandboxSettings(), ...verifySandboxOverrides } + ? (verifyPreparedSettings ?? { ...this.bash.getSandboxSettings(), ...verifySandboxOverrides }) : this.bash.getSandboxSettings(), }); const childBaseTools = createTools(childBash, provider, childMode); @@ -987,11 +1058,13 @@ export class Agent { ? "Scanning the codebase" : isVerifyDetect ? "Detecting verification recipe" - : isVerify - ? "Preparing verification pass" - : isComputer - ? "Preparing computer control pass" - : "Planning delegated work"; + : isVerifyManifest + ? "Creating verification manifest" + : isVerify + ? "Preparing verification pass" + : isComputer + ? "Preparing computer control pass" + : "Planning delegated work"; let assistantText = ""; let lastActivity = initialDetail; let childTools: ToolSet = childBaseTools; @@ -1047,9 +1120,14 @@ export class Agent { } } + const childPrompt = + isVerify && verifyPreparedRecipe + ? `${request.prompt}\n\nPrepared verify recipe JSON (use this as the primary execution recipe and keep .grok/environment.json aligned with it if present):\n${JSON.stringify(verifyPreparedRecipe, null, 2)}` + : request.prompt; + const childMessages = isVision ? await buildVisionUserMessages(request.prompt, childBash.getCwd(), signal) - : [{ role: "user" as const, content: request.prompt }]; + : [{ role: "user" as const, content: childPrompt }]; if (this.batchApi) { return await this.runTaskRequestBatch({ @@ -1781,6 +1859,31 @@ export class Agent { return null; } } + + async runVerify(onProgress?: (detail: string) => void, abortSignal?: AbortSignal): Promise { + this.abortController = new AbortController(); + const signal = abortSignal ?? this.abortController.signal; + const userModelMessage: ModelMessage = { role: "user", content: "/verify" }; + this.messages.push(userModelMessage); + this.messageSeqs.push(null); + + try { + await this.consumeBackgroundNotifications(); + const result = await runVerifyOrchestration(this, { onProgress, abortSignal: signal }); + const assistantText = result.output || result.error || "Verification completed."; + this.appendCompletedTurn(userModelMessage, [{ role: "assistant", content: assistantText }]); + return result; + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + const failureText = signal.aborted ? "Verification aborted." : `Verification failed: ${msg}`; + this.appendCompletedTurn(userModelMessage, [{ role: "assistant", content: failureText }]); + return { success: false, output: failureText }; + } finally { + if (this.abortController?.signal === signal) { + this.abortController = null; + } + } + } } interface ExecutedBatchTool { diff --git a/src/grok/tools.test.ts b/src/grok/tools.test.ts index 5884017..69949e4 100644 --- a/src/grok/tools.test.ts +++ b/src/grok/tools.test.ts @@ -110,6 +110,66 @@ describe("schedule daemon tools", () => { expect(result).toEqual({ success: true, output: "verified" }); }); + it("routes verify-detect task requests through the task tool", async () => { + const runTask = vi.fn(async () => ({ success: true, output: "recipe" })); + const tools = createTools(new BashTool("/tmp"), {} as never, "agent", { + runTask, + subagents: [], + }) as Record Promise; description?: string }>; + + const taskTool = tools.task; + expect(taskTool.description).toContain("`verify-detect`"); + + const result = (await taskTool.execute( + { + agent: "verify-detect", + description: "Detect verification recipe", + prompt: "Inspect the repository and return a VerifyRecipe JSON object.", + }, + { abortSignal: undefined }, + )) as { success: boolean; output: string }; + + expect(runTask).toHaveBeenCalledWith( + { + agent: "verify-detect", + description: "Detect verification recipe", + prompt: "Inspect the repository and return a VerifyRecipe JSON object.", + }, + undefined, + ); + expect(result).toEqual({ success: true, output: "recipe" }); + }); + + it("routes verify-manifest task requests through the task tool", async () => { + const runTask = vi.fn(async () => ({ success: true, output: "manifest written" })); + const tools = createTools(new BashTool("/tmp"), {} as never, "agent", { + runTask, + subagents: [], + }) as Record Promise; description?: string }>; + + const taskTool = tools.task; + expect(taskTool.description).toContain("`verify-manifest`"); + + const result = (await taskTool.execute( + { + agent: "verify-manifest", + description: "Create verify manifest", + prompt: "Inspect the repository and write .grok/environment.json.", + }, + { abortSignal: undefined }, + )) as { success: boolean; output: string }; + + expect(runTask).toHaveBeenCalledWith( + { + agent: "verify-manifest", + description: "Create verify manifest", + prompt: "Inspect the repository and write .grok/environment.json.", + }, + undefined, + ); + expect(result).toEqual({ success: true, output: "manifest written" }); + }); + it("exposes computer tools and routes computer task requests", async () => { const runTask = vi.fn(async () => ({ success: true, output: "computer-ready" })); const tools = createTools(new BashTool("/tmp"), {} as never, "agent", { diff --git a/src/grok/tools.ts b/src/grok/tools.ts index 1430547..dd7b010 100644 --- a/src/grok/tools.ts +++ b/src/grok/tools.ts @@ -259,24 +259,30 @@ export function createTools( if (options.runTask) { const customNames = (options.subagents ?? loadValidSubAgents()).map((agent) => agent.name); - const taskAgentEnum = ["general", "explore", "vision", "verify", "computer", ...customNames] as [ - string, - ...string[], - ]; + const taskAgentEnum = [ + "general", + "explore", + "vision", + "verify", + "verify-detect", + "verify-manifest", + "computer", + ...customNames, + ] as [string, ...string[]]; const customHint = customNames.length > 0 ? ` You may also use these user-defined sub-agents by exact name: ${customNames.join(", ")}.` : ""; tools.task = tool({ - description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, code quality work, verification, and computer-use flows instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution, \`explore\` for fast read-only investigation, \`vision\` for image validation, \`verify\` for sandbox-aware build, test, and smoke validation, and \`computer\` for host desktop screenshot/input workflows.${customHint} Provide a short description plus a detailed prompt for the child agent.`, + description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, code quality work, verification, and computer-use flows instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution, \`explore\` for fast read-only investigation, \`vision\` for image validation, \`verify\` for sandbox-aware build, test, and smoke validation, \`verify-detect\` for read-only verification recipe detection, \`verify-manifest\` to create or update a verification manifest, and \`computer\` for host desktop screenshot/input workflows.${customHint} Provide a short description plus a detailed prompt for the child agent.`, inputSchema: z.object({ agent: z .enum(taskAgentEnum) .default("general") .describe( customNames.length > 0 - ? "Built-in general, explore, vision, verify, or computer, or a configured custom sub-agent name from user settings" + ? "Built-in general, explore, vision, verify, verify-detect, verify-manifest, or computer, or a configured custom sub-agent name from user settings" : "Which sub-agent to use", ), description: z.string().describe("A short label for the delegated task, such as 'Deep code quality analysis'"), diff --git a/src/index.ts b/src/index.ts index f76cb9a..23da5ba 100755 --- a/src/index.ts +++ b/src/index.ts @@ -27,7 +27,7 @@ import { saveUserSettings, } from "./utils/settings"; import { runUpdate } from "./utils/update-checker"; -import { getVerifyCliError, VERIFY_PROMPT } from "./verify/entrypoint"; +import { buildVerifyPrompt, getVerifyCliError } from "./verify/entrypoint"; dotenv.config(); @@ -309,7 +309,7 @@ program } await runHeadless( - VERIFY_PROMPT, + buildVerifyPrompt(process.cwd()), requireApiKey(config.apiKey), config.baseURL, config.model, diff --git a/src/types/index.ts b/src/types/index.ts index 1b0665a..e1c116a 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -29,7 +29,14 @@ export interface Plan { questions?: PlanQuestion[]; } -export type BuiltinSubagentId = "general" | "explore" | "vision" | "verify" | "verify-detect" | "computer"; +export type BuiltinSubagentId = + | "general" + | "explore" + | "vision" + | "verify" + | "verify-detect" + | "verify-manifest" + | "computer"; export interface TaskRequest { agent: BuiltinSubagentId | string; @@ -104,6 +111,58 @@ export interface VerifyRecipe { notes: string[]; } +export interface VerifyEnvironmentManifest { + ecosystem?: string; + appKind?: string; + appLabel?: string; + shellInit?: string[] | string; + shellInitCommands?: string[] | string; + bootstrap?: string[] | string; + bootstrapCommands?: string[] | string; + install?: string[] | string; + installCommands?: string[] | string; + build?: string[] | string; + buildCommands?: string[] | string; + test?: string[] | string; + testCommands?: string[] | string; + start?: string; + startCommand?: string; + startPort?: string; + smokeKind?: "http" | "cli" | "none"; + smokeTarget?: string; + evidence?: string[] | string; + notes?: string[] | string; + sandbox?: { + allowNet?: boolean; + allowedHosts?: string[]; + ports?: string[]; + cpus?: number; + memory?: number; + diskSize?: number; + secrets?: Array<{ name: string; fromEnv: string; hosts: string[] }>; + from?: string; + verifyBaseFrom?: string; + guestWorkdir?: string; + syncHostWorkspace?: boolean; + shellInit?: string[]; + hostBrowserCommandsOnHost?: boolean; + }; + recipe?: Partial & Record; +} + +export interface VerifyRetryStrategy { + id: string; + when: string; + reason: string; + commands: string[]; +} + +export interface VerifyArtifact { + kind: "log" | "screenshot" | "video"; + path: string; + description: string; +} + export interface ToolResult { success: boolean; output?: string; diff --git a/src/ui/app.tsx b/src/ui/app.tsx index d2bda7a..c2d9750 100644 --- a/src/ui/app.tsx +++ b/src/ui/app.tsx @@ -56,7 +56,7 @@ import { import { discoverSkills, formatSkillsForChat } from "../utils/skills"; import { formatSubagentName } from "../utils/subagent-display"; import { checkForUpdate, runUpdate, type UpdateCheckResult } from "../utils/update-checker"; -import { VERIFY_PROMPT } from "../verify/entrypoint"; +import { buildVerifyPrompt } from "../verify/entrypoint"; import { buildSubagentBrowseRows, SUBAGENT_EDITOR_FIELDS, @@ -2088,7 +2088,7 @@ export function App({ agent, startupConfig, initialMessage, onExit }: AppProps) return true; } if (c === "/verify") { - processMessage(VERIFY_PROMPT); + processMessage(buildVerifyPrompt(agent.getCwd())); return true; } if (c === "/commit-push") { @@ -2117,6 +2117,7 @@ export function App({ agent, startupConfig, initialMessage, onExit }: AppProps) return false; }, [ + agent, handleExit, openAgentsModal, openMcpModal, @@ -2184,7 +2185,7 @@ export function App({ agent, startupConfig, initialMessage, onExit }: AppProps) processMessage(REVIEW_PROMPT); break; case "verify": - processMessage(VERIFY_PROMPT); + processMessage(buildVerifyPrompt(agent.getCwd())); break; case "commit-push": processMessage(COMMIT_PUSH_PROMPT); diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 8951f5c..7f6e501 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -89,7 +89,15 @@ export interface CustomSubagentConfig { instruction: string; } -const RESERVED_SUBAGENT_NAMES = new Set(["general", "explore", "vision", "verify", "verify-detect", "computer"]); +const RESERVED_SUBAGENT_NAMES = new Set([ + "general", + "explore", + "vision", + "verify", + "verify-detect", + "verify-manifest", + "computer", +]); export function isReservedSubagentName(name: string): boolean { return RESERVED_SUBAGENT_NAMES.has(name.trim().toLowerCase()); @@ -306,6 +314,15 @@ export function normalizeSandboxSettings(raw: unknown): SandboxSettings { if (secrets.length > 0) result.secrets = secrets; } if (typeof raw.from === "string" && raw.from.trim()) result.from = raw.from.trim(); + if (typeof raw.verifyBaseFrom === "string" && raw.verifyBaseFrom.trim()) + result.verifyBaseFrom = raw.verifyBaseFrom.trim(); + if (typeof raw.allowEphemeralInstall === "boolean") result.allowEphemeralInstall = raw.allowEphemeralInstall; + if (typeof raw.syncHostWorkspace === "boolean") result.syncHostWorkspace = raw.syncHostWorkspace; + if (typeof raw.guestWorkdir === "string" && raw.guestWorkdir.trim()) result.guestWorkdir = raw.guestWorkdir.trim(); + if (Array.isArray(raw.shellInit)) { + const shellInit = raw.shellInit.filter((line): line is string => typeof line === "string" && line.trim() !== ""); + if (shellInit.length > 0) result.shellInit = shellInit; + } if (typeof raw.hostBrowserCommandsOnHost === "boolean") result.hostBrowserCommandsOnHost = raw.hostBrowserCommandsOnHost; diff --git a/src/utils/subagent-display.test.ts b/src/utils/subagent-display.test.ts index 867b9ea..523cde8 100644 --- a/src/utils/subagent-display.test.ts +++ b/src/utils/subagent-display.test.ts @@ -8,6 +8,7 @@ describe("formatSubagentName", () => { expect(formatSubagentName("vision")).toBe("Vision"); expect(formatSubagentName("verify")).toBe("Verify"); expect(formatSubagentName("verify-detect")).toBe("Verify Detect"); + expect(formatSubagentName("verify-manifest")).toBe("Verify Manifest"); expect(formatSubagentName("computer")).toBe("Computer"); }); diff --git a/src/utils/subagent-display.ts b/src/utils/subagent-display.ts index 1907ff7..644746e 100644 --- a/src/utils/subagent-display.ts +++ b/src/utils/subagent-display.ts @@ -4,6 +4,7 @@ export function formatSubagentName(agent: string): string { if (agent === "vision") return "Vision"; if (agent === "verify") return "Verify"; if (agent === "verify-detect") return "Verify Detect"; + if (agent === "verify-manifest") return "Verify Manifest"; if (agent === "computer") return "Computer"; if (!agent) return "Sub-agent"; return agent.charAt(0).toUpperCase() + agent.slice(1); diff --git a/src/verify/checkpoint.test.ts b/src/verify/checkpoint.test.ts index 482761c..4342239 100644 --- a/src/verify/checkpoint.test.ts +++ b/src/verify/checkpoint.test.ts @@ -3,15 +3,37 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -vi.mock("child_process", () => ({ - execFile: vi.fn(), -})); +vi.mock("child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + execFile: vi.fn(), + spawn: vi.fn(() => { + const { EventEmitter } = require("events"); + const { Readable } = require("stream"); + const child = new EventEmitter(); + child.stdout = new Readable({ + read() { + this.push(null); + }, + }); + child.stderr = new Readable({ + read() { + this.push(null); + }, + }); + setTimeout(() => child.emit("close", 0), 0); + return child; + }), + }; +}); -import { execFile } from "child_process"; +import { execFile, spawn } from "child_process"; import { ensureVerifyCheckpoint, getVerifyCheckpointName } from "./checkpoint"; import { inferVerifyProjectProfile } from "./entrypoint"; const execFileMock = vi.mocked(execFile); +const spawnMock = vi.mocked(spawn); const tempDirs: string[] = []; function makeTempDir(prefix: string): string { @@ -22,6 +44,24 @@ function makeTempDir(prefix: string): string { afterEach(() => { execFileMock.mockReset(); + spawnMock.mockReset(); + spawnMock.mockImplementation(() => { + const { EventEmitter } = require("events"); + const { Readable } = require("stream"); + const child = new EventEmitter(); + child.stdout = new Readable({ + read() { + this.push(null); + }, + }); + child.stderr = new Readable({ + read() { + this.push(null); + }, + }); + setTimeout(() => child.emit("close", 0), 0); + return child; + }); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -46,13 +86,13 @@ describe("verify checkpoints", () => { ); fs.writeFileSync(path.join(dir, "package-lock.json"), ""); - execFileMock.mockImplementation((command, args, _options, callback) => { + execFileMock.mockImplementation((_command, args, _options, callback) => { const cb = callback as (error: Error | null, stdout: string, stderr: string) => void; if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "list") { cb(null, "", ""); return {} as never; } - cb(null, "created", ""); + cb(null, "", ""); return {} as never; }); @@ -62,10 +102,11 @@ describe("verify checkpoints", () => { expect(result.created).toBe(true); expect(result.checkpointName).toMatch(/^verify-nextjs-/); expect(result.guestWorkdir).toBe("/grok/verify/worktree"); - expect(execFileMock).toHaveBeenCalledTimes(2); - const createCall = execFileMock.mock.calls[1]; - expect(createCall[0]).toBe("shuru"); - const createArgs = createCall[1] as string[]; + expect(execFileMock).toHaveBeenCalledTimes(1); + expect(spawnMock).toHaveBeenCalledTimes(1); + const spawnArgs = spawnMock.mock.calls[0]; + expect(spawnArgs[0]).toBe("shuru"); + const createArgs = spawnArgs[1] as string[]; expect(createArgs.slice(0, 3)).toEqual(["checkpoint", "create", result.checkpointName!]); expect(createArgs.join(" ")).toContain("export DEBIAN_FRONTEND=noninteractive"); }); @@ -80,7 +121,7 @@ describe("verify checkpoints", () => { const profile = inferVerifyProjectProfile(dir); const checkpointName = getVerifyCheckpointName(dir, profile.recipe); - execFileMock.mockImplementation((command, args, _options, callback) => { + execFileMock.mockImplementation((_command, args, _options, callback) => { const cb = callback as (error: Error | null, stdout: string, stderr: string) => void; if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "list") { cb(null, `${checkpointName}\n`, ""); @@ -103,16 +144,12 @@ describe("verify checkpoints", () => { const profile = inferVerifyProjectProfile(dir); const checkpointName = getVerifyCheckpointName(dir, profile.recipe); - execFileMock.mockImplementation((command, args, _options, callback) => { + execFileMock.mockImplementation((_command, args, _options, callback) => { const cb = callback as (error: Error | null, stdout: string, stderr: string) => void; if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "list") { cb(null, "", ""); return {} as never; } - if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "create") { - cb(new Error("bun: not found"), "", "bun: not found"); - return {} as never; - } if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "delete") { cb(null, "", ""); return {} as never; @@ -121,10 +158,29 @@ describe("verify checkpoints", () => { return {} as never; }); + spawnMock.mockImplementation(() => { + const { EventEmitter } = require("events"); + const { Readable } = require("stream"); + const child = new EventEmitter(); + child.stdout = new Readable({ + read() { + this.push(null); + }, + }); + child.stderr = new Readable({ + read() { + this.push("bun: not found\n"); + this.push(null); + }, + }); + setTimeout(() => child.emit("close", 1), 0); + return child; + }); + await expect(ensureVerifyCheckpoint(dir, profile, profile.sandboxSettings)).rejects.toThrow( `Verify checkpoint bootstrap failed for "${checkpointName}"`, ); - expect(execFileMock).toHaveBeenCalledTimes(3); - expect(execFileMock.mock.calls[2]?.[1]).toEqual(["checkpoint", "delete", checkpointName]); + expect(execFileMock).toHaveBeenCalledTimes(2); + expect(execFileMock.mock.calls[1]?.[1]).toEqual(["checkpoint", "delete", checkpointName]); }); }); diff --git a/src/verify/checkpoint.ts b/src/verify/checkpoint.ts index c51c1ca..a878ea3 100644 --- a/src/verify/checkpoint.ts +++ b/src/verify/checkpoint.ts @@ -1,10 +1,10 @@ -import { execFile } from "child_process"; +import { execFile, spawn } from "child_process"; import { createHash } from "crypto"; import * as fs from "fs"; import * as path from "path"; import type { VerifyRecipe } from "../types/index"; import type { SandboxSettings } from "../utils/settings"; -import type { VerifyProjectProfile } from "./entrypoint"; +import type { VerifyProjectProfile } from "./recipes"; const DEFAULT_VERIFY_GUEST_WORKDIR = "/grok/verify/worktree"; @@ -25,6 +25,51 @@ function execFileAsync(command: string, args: string[], cwd: string): Promise<{ }); } +function spawnWithProgress( + command: string, + args: string[], + cwd: string, + onLine?: (line: string) => void, +): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: { ...process.env, FORCE_COLOR: "0" }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stdout += text; + if (onLine) { + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (trimmed) onLine(trimmed); + } + } + }); + child.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderr += text; + if (onLine) { + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (trimmed) onLine(trimmed); + } + } + }); + child.on("error", (err) => reject(err)); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error((stderr || stdout || `Process exited with code ${code}`).trim())); + } else { + resolve({ stdout, stderr }); + } + }); + }); +} + function stableRecipePayload(recipe: VerifyRecipe): string { return JSON.stringify({ ecosystem: recipe.ecosystem, @@ -168,6 +213,7 @@ export async function ensureVerifyCheckpoint( cwd: string, profile: VerifyProjectProfile, settings: SandboxSettings, + onProgress?: (detail: string) => void, ): Promise { if (profile.recipe.installCommands.length === 0) { return { created: false }; @@ -177,8 +223,9 @@ export async function ensureVerifyCheckpoint( const checkpoints: string[] = await listCheckpoints(cwd).catch((): string[] => []); if (!checkpoints.includes(checkpointName)) { const args = buildCheckpointCreateArgs(cwd, checkpointName, DEFAULT_VERIFY_GUEST_WORKDIR, profile.recipe, settings); + onProgress?.(`Creating checkpoint: ${checkpointName}`); try { - await execFileAsync("shuru", args, cwd); + await spawnWithProgress("shuru", args, cwd, onProgress); } catch (error) { await deleteCheckpoint(cwd, checkpointName).catch(() => {}); const message = error instanceof Error ? error.message : String(error); @@ -187,5 +234,6 @@ export async function ensureVerifyCheckpoint( return { checkpointName, guestWorkdir: DEFAULT_VERIFY_GUEST_WORKDIR, created: true }; } + onProgress?.(`Reusing checkpoint: ${checkpointName}`); return { checkpointName, guestWorkdir: DEFAULT_VERIFY_GUEST_WORKDIR, created: false }; } diff --git a/src/verify/entrypoint.test.ts b/src/verify/entrypoint.test.ts index 4dae01e..4755b60 100644 --- a/src/verify/entrypoint.test.ts +++ b/src/verify/entrypoint.test.ts @@ -5,12 +5,12 @@ import { afterEach, describe, expect, it } from "vitest"; import type { VerifyRecipe } from "../types/index"; import { buildVerifyDetectPrompt, + buildVerifyPrompt, buildVerifyTaskPrompt, createVerifyRuntimeConfig, getVerifyCliError, inferVerifyProjectProfile, inferVerifySmokeUrl, - VERIFY_PROMPT, } from "./entrypoint"; const tempDirs: string[] = []; @@ -56,12 +56,15 @@ describe("verify entrypoint helpers", () => { expect(profile.packageManager).toBe("npm"); expect(profile.sandboxSettings.ports).toEqual(["3000:3000"]); expect(profile.availableScripts).toEqual(["dev", "build", "lint"]); - expect(profile.recipe.bootstrapCommands).toEqual([]); + expect(profile.recipe.bootstrapCommands).toEqual([ + "apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm", + ]); + expect(profile.recipe.shellInitCommands).toEqual(["export DEBIAN_FRONTEND=noninteractive"]); expect(profile.recipe.startCommand).toBe("npm run dev"); expect(profile.recipe.testCommands).toContain("npm run lint"); }); - it("detects bun as package manager without hardcoded bootstrap", () => { + it("detects bun as package manager and bootstraps bun plus node tooling for web apps", () => { const dir = makeTempDir("grok-verify-bun-"); fs.writeFileSync( path.join(dir, "package.json"), @@ -71,7 +74,17 @@ describe("verify entrypoint helpers", () => { const profile = inferVerifyProjectProfile(dir); expect(profile.packageManager).toBe("bun"); - expect(profile.recipe.bootstrapCommands).toEqual([]); + expect(profile.recipe.bootstrapCommands).toEqual([ + "apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm", + "curl -fsSL https://bun.sh/install | bash", + ]); + expect(profile.recipe.shellInitCommands).toEqual([ + "export DEBIAN_FRONTEND=noninteractive", + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable in test assertion + 'export BUN_INSTALL="${HOME}/.bun"', + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable in test assertion + 'export PATH="${BUN_INSTALL}/bin:$PATH"', + ]); expect(profile.recipe.installCommands).toContain("bun install"); }); @@ -188,16 +201,20 @@ describe("verify entrypoint helpers", () => { expect(prompt).toContain("Bootstrap commands:"); expect(prompt).toContain("Install commands:"); expect(prompt).toContain("MUST run browser smoke tests"); - expect(prompt).toContain("--screenshot-dir .grok/verify-artifacts"); - expect(prompt).toContain("record start .grok/verify-artifacts/verify-smoke.webm"); + expect(prompt).toContain("record start"); expect(prompt).toContain("record stop"); - expect(prompt).toContain("Browser Checks"); + expect(prompt).toContain("Results"); expect(prompt).toContain("Evidence"); - expect(prompt).toContain("Recipe"); - expect(prompt).toContain("ephemeral dependency installs are allowed inside the sandbox"); - expect(prompt).toContain("prefer chaining install + build/test/start"); + expect(prompt).toContain("Blockers"); + expect(prompt).toContain("Ephemeral installs are allowed"); + expect(prompt).toContain("Probe the sandbox for runtimes"); + expect(prompt).toContain("Only install what is missing"); + expect(prompt).toContain("MANDATORY workflow"); + expect(prompt).toContain("Phase 4"); + expect(prompt).toContain("QA tester"); expect(prompt).toContain(".grok/verify-artifacts"); - expect(prompt).toContain("markdown links"); + expect(prompt).toContain("Keep the report compact"); + expect(prompt).toContain("Evidence is mandatory even on failure"); }); it("builds a detector prompt that requests JSON output", () => { @@ -207,6 +224,7 @@ describe("verify entrypoint helpers", () => { expect(prompt).toContain("Return ONLY valid JSON"); expect(prompt).toContain('"bootstrapCommands": string[]'); expect(prompt).toContain("Fallback hints from static detection"); + expect(prompt).toContain("probes for runtimes/tools first"); }); it("explains ambiguous browser setup when multiple ports exist", () => { @@ -217,11 +235,24 @@ describe("verify entrypoint helpers", () => { expect(prompt).toContain("Skip browser checks unless the user clearly identifies"); }); - it("VERIFY_PROMPT drives both sub-agents", () => { - expect(VERIFY_PROMPT).toContain("verify-detect"); - expect(VERIFY_PROMPT).toContain("Step 1"); - expect(VERIFY_PROMPT).toContain("Step 2"); - expect(VERIFY_PROMPT).toContain("task"); + it("buildVerifyPrompt drives both sub-agents with a draft recipe", () => { + const dir = makeTempDir("grok-verify-prompt-dynamic-"); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2), + ); + fs.writeFileSync(path.join(dir, "package-lock.json"), ""); + + const prompt = buildVerifyPrompt(dir); + expect(prompt).toContain("verify-manifest"); + expect(prompt).toContain("Step 1"); + expect(prompt).toContain("Step 2"); + expect(prompt).toContain("task"); + expect(prompt).toContain(".grok/environment.json"); + expect(prompt).toContain('"ecosystem"'); + expect(prompt).toContain('"bootstrapCommands"'); + expect(prompt).toContain("apt-get"); + expect(prompt).toContain("nodejs npm"); }); it("uses an override recipe when creating the runtime config", () => { diff --git a/src/verify/entrypoint.ts b/src/verify/entrypoint.ts index 0ae45b4..5cfc4be 100644 --- a/src/verify/entrypoint.ts +++ b/src/verify/entrypoint.ts @@ -1,68 +1,22 @@ -import * as fs from "fs"; -import * as path from "path"; import type { TaskRequest, VerifyRecipe } from "../types/index"; -import { mergeSandboxSettings, type SandboxSettings } from "../utils/settings"; +import type { SandboxSettings } from "../utils/settings"; +import { ensureVerifyCheckpoint, type PreparedVerifyCheckpoint } from "./checkpoint"; +import { loadVerifyEnvironment } from "./environment"; +import { buildBrowserGuidance, buildEvidenceGuidance, buildReadinessGuidance } from "./evidence"; +import { + defaultShellInit, + detectPackageManager, + getNodeWebBootstrapCommands, + getNodeWebShellInitCommands, + inferVerifyProjectProfile, + normalizeVerifyAppKind, + type VerifyProjectProfile, +} from "./recipes"; +import { buildRetryGuidance } from "./retry"; export const VERIFY_SUBAGENT_ID = "verify"; export const VERIFY_TASK_DESCRIPTION = "Run local verification"; -type VerifyAppKind = - | "nextjs" - | "vite" - | "astro" - | "sveltekit" - | "remix" - | "cra" - | "node" - | "django" - | "python" - | "go" - | "rust" - | "maven" - | "gradle" - | "make" - | "unknown"; - -function normalizeVerifyAppKind(value: string): VerifyAppKind { - return ( - [ - "nextjs", - "vite", - "astro", - "sveltekit", - "remix", - "cra", - "node", - "django", - "python", - "go", - "rust", - "maven", - "gradle", - "make", - "unknown", - ] as const - ).includes(value as VerifyAppKind) - ? (value as VerifyAppKind) - : "unknown"; -} - -interface PackageJsonLike { - dependencies?: Record; - devDependencies?: Record; - scripts?: Record; -} - -export interface VerifyProjectProfile { - appKind: VerifyAppKind; - appLabel: string; - packageManager: string | null; - availableScripts: string[]; - hasNodeModules: boolean; - sandboxSettings: SandboxSettings; - recipe: VerifyRecipe; -} - export interface VerifyRuntimeConfig { sandboxMode: "shuru"; sandboxSettings: SandboxSettings; @@ -71,468 +25,16 @@ export interface VerifyRuntimeConfig { checkpointCreated?: boolean; } -function fileExists(cwd: string, file: string): boolean { - return fs.existsSync(path.join(cwd, file)); -} - -function readTextFile(cwd: string, file: string): string | null { - try { - return fs.readFileSync(path.join(cwd, file), "utf8"); - } catch { - return null; - } -} - -function parseHostPort(mapping: string): string | null { - const match = mapping.trim().match(/^(\d+):(\d+)$/); - return match ? match[1] : null; -} - -function readPackageJson(cwd: string): PackageJsonLike | null { - const raw = readTextFile(cwd, "package.json"); - if (!raw) return null; - try { - return JSON.parse(raw) as PackageJsonLike; - } catch { - return null; - } -} - -function detectPackageManager(cwd: string): string | null { - const candidates: Array<[string, string]> = [ - ["pnpm-lock.yaml", "pnpm"], - ["bun.lock", "bun"], - ["bun.lockb", "bun"], - ["yarn.lock", "yarn"], - ["package-lock.json", "npm"], - ["uv.lock", "uv"], - ["poetry.lock", "poetry"], - ["Pipfile.lock", "pipenv"], - ]; - - for (const [file, manager] of candidates) { - if (fileExists(cwd, file)) { - return manager; - } - } - - return null; +export interface PreparedVerifySandbox { + profile: VerifyProjectProfile; + sandboxSettings: SandboxSettings; + checkpoint?: PreparedVerifyCheckpoint; } function dedupe(values: Array): string[] { return [...new Set(values.map((v) => v?.trim()).filter((v): v is string => Boolean(v)))]; } -function defaultShellInit(): string[] { - return ["export DEBIAN_FRONTEND=noninteractive"]; -} - -function inferPortFromCommand(command: string | undefined): string | undefined { - if (!command) return undefined; - const flagMatch = command.match(/(?:--port|-p)\s+(\d{2,5})/); - if (flagMatch) return flagMatch[1]; - const envMatch = command.match(/\bPORT=(\d{2,5})\b/); - if (envMatch) return envMatch[1]; - return undefined; -} - -function parseTargetNames(raw: string): string[] { - return raw - .split(/\r?\n/) - .map((line) => line.match(/^([A-Za-z0-9_.-]+):(?:\s|$)/)?.[1]) - .filter((target): target is string => Boolean(target)); -} - -function detectMakeRecipe(cwd: string): VerifyRecipe | null { - const makefile = readTextFile(cwd, "Makefile"); - if (!makefile) return null; - const targets = parseTargetNames(makefile); - const has = (names: string[]) => names.find((name) => targets.includes(name)); - const install = has(["install", "setup", "bootstrap"]); - const build = has(["build", "compile"]); - const test = has(["test", "check"]); - const run = has(["run", "start", "serve", "dev"]); - - return { - ecosystem: "make", - appKind: "make", - appLabel: "Makefile-driven project", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: install ? [`make ${install}`] : [], - buildCommands: build ? [`make ${build}`] : [], - testCommands: test ? [`make ${test}`] : [], - startCommand: run ? `make ${run}` : undefined, - smokeKind: "none", - evidence: ["Detected Makefile", `Targets: ${targets.join(", ") || "(none)"}`], - notes: [], - }; -} - -function detectNodeRecipe(cwd: string, pkg: PackageJsonLike, packageManager: string | null): VerifyRecipe { - const scripts = pkg.scripts ?? {}; - const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) }; - let appKind: VerifyAppKind = "node"; - let appLabel = "Node.js app"; - let defaultPort: string | undefined; - - if (deps.next) { - appKind = "nextjs"; - appLabel = "Next.js"; - defaultPort = "3000"; - } else if (deps["@sveltejs/kit"]) { - appKind = "sveltekit"; - appLabel = "SvelteKit"; - defaultPort = "5173"; - } else if (deps.astro) { - appKind = "astro"; - appLabel = "Astro"; - defaultPort = "4321"; - } else if (deps["@remix-run/dev"] || deps["@remix-run/react"]) { - appKind = "remix"; - appLabel = "Remix"; - defaultPort = "3000"; - } else if (deps["react-scripts"]) { - appKind = "cra"; - appLabel = "Create React App"; - defaultPort = "3000"; - } else if (deps.vite) { - appKind = "vite"; - appLabel = "Vite"; - defaultPort = "5173"; - } - - const install = packageManager - ? packageManager === "pnpm" - ? "pnpm install" - : packageManager === "bun" - ? "bun install" - : packageManager === "yarn" - ? "yarn install" - : "npm install" - : undefined; - const startCommand = scripts.dev ?? scripts.start; - const startPort = inferPortFromCommand(startCommand) ?? defaultPort; - const smokeKind: VerifyRecipe["smokeKind"] = startCommand && startPort ? "http" : "none"; - - return { - ecosystem: "node", - appKind, - appLabel, - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: dedupe([install]), - buildCommands: dedupe( - [scripts.build, scripts.typecheck].map((script) => script && pickPackageScript(packageManager, scripts, script)), - ), - testCommands: dedupe( - ["test", "check", "lint"] - .filter((name) => scripts[name]) - .map((name) => pickPackageScript(packageManager, scripts, scripts[name]!)), - ), - startCommand: startCommand ? pickPackageScript(packageManager, scripts, startCommand) : undefined, - startPort, - smokeKind, - evidence: [`Detected package.json`, `Scripts: ${Object.keys(scripts).join(", ") || "(none)"}`], - notes: [], - }; -} - -function pickPackageScript(packageManager: string | null, scripts: Record, body: string): string { - const entry = Object.entries(scripts).find(([, scriptBody]) => scriptBody === body)?.[0]; - if (!entry) return body; - const runner = - packageManager === "pnpm" - ? "pnpm" - : packageManager === "bun" - ? "bun" - : packageManager === "yarn" - ? "yarn" - : "npm run"; - return runner === "yarn" ? `yarn ${entry}` : runner === "bun" ? `bun run ${entry}` : `${runner} ${entry}`; -} - -function detectPythonRecipe(cwd: string): VerifyRecipe | null { - const pyproject = readTextFile(cwd, "pyproject.toml"); - const requirements = readTextFile(cwd, "requirements.txt"); - const managePy = fileExists(cwd, "manage.py"); - if (!pyproject && !requirements && !managePy && !fileExists(cwd, "setup.py")) { - return null; - } - - const lower = `${pyproject ?? ""}\n${requirements ?? ""}`.toLowerCase(); - const packageManager = detectPackageManager(cwd); - const isDjango = managePy || lower.includes("django"); - const isFastApi = lower.includes("fastapi") || lower.includes("uvicorn"); - - let install = "pip install -r requirements.txt"; - if (packageManager === "uv") install = "uv sync"; - else if (packageManager === "poetry") install = "poetry install"; - else if (packageManager === "pipenv") install = "pipenv install"; - else if (pyproject && !requirements) install = "pip install -e ."; - - if (isDjango) { - return { - ecosystem: "python", - appKind: "django", - appLabel: "Django app", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [install], - buildCommands: [], - testCommands: ["python manage.py test"], - startCommand: "python manage.py runserver 0.0.0.0:8000", - startPort: "8000", - smokeKind: "http", - evidence: ["Detected manage.py", pyproject ? "Detected pyproject.toml" : undefined].filter(Boolean) as string[], - notes: [], - }; - } - - if (isFastApi) { - const appModule = fileExists(cwd, "main.py") ? "main:app" : fileExists(cwd, "app.py") ? "app:app" : "main:app"; - return { - ecosystem: "python", - appKind: "python", - appLabel: "Python web app", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [install], - buildCommands: [], - testCommands: fileExists(cwd, "tests") ? ["pytest"] : [], - startCommand: `uvicorn ${appModule} --host 0.0.0.0 --port 8000`, - startPort: "8000", - smokeKind: "http", - evidence: ["Detected Python project", "Detected FastAPI/Uvicorn dependency"], - notes: [], - }; - } - - return { - ecosystem: "python", - appKind: "python", - appLabel: "Python project", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [install], - buildCommands: [], - testCommands: fileExists(cwd, "tests") ? ["pytest"] : ["python -m unittest discover"], - smokeKind: "none", - evidence: ["Detected Python project"], - notes: [], - }; -} - -function detectGoRecipe(cwd: string): VerifyRecipe | null { - if (!fileExists(cwd, "go.mod")) return null; - return { - ecosystem: "go", - appKind: "go", - appLabel: "Go project", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [], - buildCommands: ["go build ./..."], - testCommands: ["go test ./..."], - startCommand: fileExists(cwd, "main.go") ? "go run ." : undefined, - smokeKind: "none", - evidence: ["Detected go.mod"], - notes: [], - }; -} - -function detectRustRecipe(cwd: string): VerifyRecipe | null { - if (!fileExists(cwd, "Cargo.toml")) return null; - return { - ecosystem: "rust", - appKind: "rust", - appLabel: "Rust project", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [], - buildCommands: ["cargo build"], - testCommands: ["cargo test"], - startCommand: fileExists(cwd, path.join("src", "main.rs")) ? "cargo run" : undefined, - smokeKind: "none", - evidence: ["Detected Cargo.toml"], - notes: [], - }; -} - -function detectJavaRecipe(cwd: string): VerifyRecipe | null { - if (fileExists(cwd, "pom.xml")) { - return { - ecosystem: "java", - appKind: "maven", - appLabel: "Maven project", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [], - buildCommands: ["mvn package"], - testCommands: ["mvn test"], - smokeKind: "none", - evidence: ["Detected pom.xml"], - notes: [], - }; - } - - if (fileExists(cwd, "build.gradle") || fileExists(cwd, "build.gradle.kts")) { - const gradle = fileExists(cwd, "gradlew") ? "./gradlew" : "gradle"; - return { - ecosystem: "java", - appKind: "gradle", - appLabel: "Gradle project", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [], - buildCommands: [`${gradle} build`], - testCommands: [`${gradle} test`], - smokeKind: "none", - evidence: ["Detected Gradle build file"], - notes: [], - }; - } - - return null; -} - -function detectFallbackRecipe(cwd: string): VerifyRecipe { - const makeRecipe = detectMakeRecipe(cwd); - if (makeRecipe) return makeRecipe; - return { - ecosystem: "unknown", - appKind: "unknown", - appLabel: "Unknown project type", - shellInitCommands: defaultShellInit(), - bootstrapCommands: [], - installCommands: [], - buildCommands: [], - testCommands: [], - smokeKind: "none", - evidence: ["No known app metadata detected"], - notes: ["The verify sub-agent should inspect the repo directly and derive commands from the codebase."], - }; -} - -function inferFallbackRecipe(cwd: string, pkg: PackageJsonLike | null, packageManager: string | null): VerifyRecipe { - if (pkg) return detectNodeRecipe(cwd, pkg, packageManager); - return ( - detectPythonRecipe(cwd) ?? - detectGoRecipe(cwd) ?? - detectRustRecipe(cwd) ?? - detectJavaRecipe(cwd) ?? - detectFallbackRecipe(cwd) - ); -} - -export function normalizeVerifyRecipe(value: unknown): VerifyRecipe | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const raw = value as Record; - const asStrings = (input: unknown): string[] => - Array.isArray(input) - ? input.filter((v): v is string => typeof v === "string" && v.trim() !== "").map((v) => v.trim()) - : []; - const ecosystem = typeof raw.ecosystem === "string" ? raw.ecosystem.trim() : ""; - const appKind = typeof raw.appKind === "string" ? raw.appKind.trim() : ""; - const appLabel = typeof raw.appLabel === "string" ? raw.appLabel.trim() : ""; - const smokeKind = - raw.smokeKind === "http" || raw.smokeKind === "cli" || raw.smokeKind === "none" ? raw.smokeKind : "none"; - if (!ecosystem || !appKind || !appLabel) return null; - return { - ecosystem, - appKind, - appLabel, - shellInitCommands: asStrings(raw.shellInitCommands), - bootstrapCommands: asStrings(raw.bootstrapCommands), - installCommands: asStrings(raw.installCommands), - buildCommands: asStrings(raw.buildCommands), - testCommands: asStrings(raw.testCommands), - startCommand: typeof raw.startCommand === "string" && raw.startCommand.trim() ? raw.startCommand.trim() : undefined, - startPort: typeof raw.startPort === "string" && raw.startPort.trim() ? raw.startPort.trim() : undefined, - smokeKind, - smokeTarget: typeof raw.smokeTarget === "string" && raw.smokeTarget.trim() ? raw.smokeTarget.trim() : undefined, - evidence: asStrings(raw.evidence), - notes: asStrings(raw.notes), - }; -} - -export function inferVerifySmokeUrl(settings?: SandboxSettings): string | null { - const ports = settings?.ports ?? []; - if (ports.length !== 1) { - return null; - } - - const hostPort = parseHostPort(ports[0]); - return hostPort ? `http://127.0.0.1:${hostPort}` : null; -} - -export function inferVerifyProjectProfile( - cwd: string, - baseSettings: SandboxSettings = {}, - recipeOverride?: VerifyRecipe | null, -): VerifyProjectProfile { - const pkg = readPackageJson(cwd); - const packageManager = detectPackageManager(cwd); - const recipe = recipeOverride ?? inferFallbackRecipe(cwd, pkg, packageManager); - const inferredDefaults: SandboxSettings = - recipe.smokeKind === "http" && recipe.startPort ? { ports: [`${recipe.startPort}:${recipe.startPort}`] } : {}; - const sandboxSettings = mergeSandboxSettings(inferredDefaults, baseSettings); - const smokeUrl = inferVerifySmokeUrl(sandboxSettings); - - const recipeWithRuntime: VerifyRecipe = { - ...recipe, - smokeTarget: recipe.smokeKind === "http" ? (smokeUrl ?? recipe.smokeTarget) : undefined, - }; - - if (!fs.existsSync(path.join(cwd, "node_modules")) && recipeWithRuntime.ecosystem === "node") { - recipeWithRuntime.notes = dedupe([ - ...recipeWithRuntime.notes, - "Host dependencies are not installed in node_modules. Verification may be limited unless a Shuru checkpoint already contains the needed runtime dependencies.", - ]); - } - - return { - appKind: normalizeVerifyAppKind(recipeWithRuntime.appKind), - appLabel: recipeWithRuntime.appLabel, - packageManager, - availableScripts: Object.keys(pkg?.scripts ?? {}), - hasNodeModules: fs.existsSync(path.join(cwd, "node_modules")), - sandboxSettings, - recipe: recipeWithRuntime, - }; -} - -function buildBrowserGuidance(profile: VerifyProjectProfile): string[] { - if (profile.recipe.smokeKind === "http" && profile.recipe.smokeTarget) { - return [ - `- REQUIRED: After the dev server is running, you MUST run browser smoke tests against ${profile.recipe.smokeTarget}.`, - "- The agent-browser command runs on the HOST, not inside the sandbox. It WILL work. Do not skip it or assume it is unavailable.", - "- Run this exact sequence using the bash tool:", - ` mkdir -p .grok/verify-artifacts && agent-browser record start .grok/verify-artifacts/verify-smoke.webm && agent-browser --screenshot-dir .grok/verify-artifacts open ${profile.recipe.smokeTarget} && agent-browser wait --load networkidle && agent-browser --screenshot-dir .grok/verify-artifacts screenshot && agent-browser get title && agent-browser record stop && agent-browser close`, - "- IMPORTANT: Use --screenshot-dir .grok/verify-artifacts to control where screenshots are saved. Do NOT pass a filename as a positional arg to the screenshot command.", - "- IMPORTANT: Use `agent-browser record start ` and `agent-browser record stop` to capture a video recording of the smoke test flow.", - "- If that command fails, report the exact error output. Do not preemptively skip browser checks.", - "- Include both the screenshot and video file paths from the output in the Evidence section.", - ]; - } - - if ((profile.sandboxSettings.ports?.length ?? 0) > 1) { - return [ - "- Multiple forwarded ports are configured, so browser smoke testing is ambiguous.", - "- Skip browser checks unless the user clearly identifies which forwarded localhost URL to verify.", - ]; - } - - if (profile.recipe.smokeKind === "cli") { - return ["- This project appears to need CLI-style runtime validation rather than browser smoke testing."]; - } - - return [ - "- No unambiguous forwarded localhost URL is configured, so browser smoke testing is optional and should usually be skipped.", - "- If the app can still be verified with bash-only checks, do that and explain why browser checks were skipped.", - ]; -} - function formatRecipeCommands(title: string, commands: string[]): string { return commands.length > 0 ? `- ${title}: ${commands.join(" ; ")}` : `- ${title}: (none inferred)`; } @@ -564,7 +66,9 @@ export function buildVerifyTaskPrompt( settings?: SandboxSettings, recipeOverride?: VerifyRecipe | null, ): string { - const profile = inferVerifyProjectProfile(cwd, settings, recipeOverride); + const manifest = recipeOverride ? null : loadVerifyEnvironment(cwd, settings); + const effectiveSettings = manifest?.sandboxSettings ?? settings; + const profile = inferVerifyProjectProfile(cwd, effectiveSettings, recipeOverride ?? manifest?.recipe ?? null); const checkpoint = profile.sandboxSettings.from?.trim(); const network = profile.sandboxSettings.allowNet ? profile.sandboxSettings.allowedHosts?.length @@ -581,6 +85,7 @@ export function buildVerifyTaskPrompt( "- Then execute that recipe inside the active Shuru sandbox and report the result.", "", "Detected project context and inferred recipe:", + ...(manifest ? [`- Verify environment manifest: ${manifest.path}.`] : []), ...buildProjectContextLines(profile), "", "Environment:", @@ -591,32 +96,47 @@ export function buildVerifyTaskPrompt( : "- No Shuru checkpoint is configured; use the current sandbox settings as-is.", "- Shuru runs are ephemeral in this version. Shell-side workspace edits do not persist back to the host.", "", - "Required workflow:", - "- Quickly inspect the code and config files to confirm or correct the inferred recipe before executing it.", - "- If the inferred recipe is wrong, say what you changed and why before proceeding.", - "- Prefer the recipe commands as the default execution plan.", - "- In verify mode, ephemeral dependency installs are allowed inside the sandbox.", - "- If the recipe needs install/setup work and no checkpoint already provides it, prefer chaining install + build/test/start in the same sandbox command so the installed dependencies remain available for that command.", - "- Use background bash only when a dev server or watcher must stay alive while you continue verifying.", - "- If an inferred install/setup step is blocked by sandbox persistence rules, explain the blocker clearly and continue with any valid non-setup checks.", - "- IMPORTANT: agent-browser commands run on the HOST, not inside the sandbox. They WILL work. Do not skip browser checks or assume agent-browser is unavailable. Just run the command via the bash tool.", - "- Always save a screenshot and include the screenshot file path in the final report so the user can verify visually.", + "MANDATORY workflow (do ALL steps in order, do NOT stop after build/lint):", + "", + "Phase 1 — Setup:", + "- Probe the sandbox for runtimes (`command -v node`, `command -v npm`, etc). Only install what is missing.", + "- Ephemeral installs are allowed. Chain install + build in the same sandbox command if no checkpoint provides deps.", + "", + "Phase 2 — Build and test:", + "- Run installCommands, buildCommands, and testCommands from the recipe.", + "", + "Phase 3 — Start the app (REQUIRED, do not skip):", + "- Start the app using startCommand from the recipe, running it in the background.", + "- Wait for the app to be ready: use a curl readiness loop or `agent-browser wait --load networkidle`.", + "- If the app fails to start, report the error but still attempt to capture evidence (logs, screenshots).", + "", + "Phase 4 — Browser QA testing (REQUIRED, do not skip):", + "- You are a QA tester. Open the app in the browser and test it like a human would.", + "- agent-browser commands run on the HOST, not the sandbox. They WILL work. Do not skip them.", + "- Record a video of the entire browser session.", + "- Navigate the app: click links, buttons, menus. Verify pages load correctly.", + "- Check for JavaScript console errors.", + "- Spend 3-5 interactions testing the critical path. Take screenshots after each.", + "- This is the most important phase. Build/lint passing means nothing if the app doesn't actually work.", + "", + "Phase 5 — Teardown:", + "- Stop recording, close browser, THEN stop the dev server.", + ...buildReadinessGuidance(profile), ...buildBrowserGuidance(profile), + ...buildRetryGuidance(profile), + ...buildEvidenceGuidance(), "", "Reporting requirements:", - "- Return a concise structured report with these sections:", + "- Return a concise structured report with these sections only:", " Summary", - " Recipe", - " Environment", - " Commands Run", - " Browser Checks", - " Evidence", " Results", - " Failures or Blockers", + " Evidence", + " Blockers", " Residual Risk", - "- The Recipe section must say what recipe you used and whether you changed the inferred one.", - "- If you captured screenshots or other browser artifacts, include their exact workspace-relative file paths in the Evidence section.", - "- Use markdown links for screenshot paths when practical, otherwise include the plain relative paths.", + "- Keep the report compact: prefer 1-3 short bullets per section, and do not paste large command logs unless they are essential blockers.", + "- The Summary must say what recipe/source of truth you used and whether you changed the inferred/default one.", + "- Evidence is mandatory even on failure. If you captured screenshots, video, or logs, include their exact workspace-relative file paths in the Evidence section.", + "- Use markdown links for artifact paths when practical, otherwise include the plain relative paths.", ].join("\n"); } @@ -633,15 +153,22 @@ export function createVerifyTaskRequest( } export function buildVerifyDetectPrompt(cwd: string, settings?: SandboxSettings): string { - const fallbackProfile = inferVerifyProjectProfile(cwd, settings); + const manifest = loadVerifyEnvironment(cwd, settings); + const fallbackProfile = inferVerifyProjectProfile( + cwd, + manifest?.sandboxSettings ?? settings, + manifest?.recipe ?? null, + ); return [ "Inspect this repository and produce a structured verification recipe.", "", "Your job:", "- Read the codebase, config files, and any relevant docs or AGENTS guidance.", + "- If environment.json or .grok/environment.json exists, treat it as the highest-priority source of truth and only fill in missing details.", "- Infer how the project should be installed, built, tested, and started.", "- Infer whether verification should use HTTP/browser smoke checks, CLI checks, or no runtime smoke step.", "- Prefer concrete commands that are likely to work in a fresh Debian Linux environment.", + "- Design the recipe so verification probes for runtimes/tools first and only installs missing dependencies or toolchains when necessary.", "- Use the fallback hints below only as clues, not as the final answer.", "", "IMPORTANT for shellInitCommands and bootstrapCommands:", @@ -653,6 +180,7 @@ export function buildVerifyDetectPrompt(cwd: string, settings?: SandboxSettings) '- Example: ["apt-get update && apt-get install -y curl unzip ca-certificates && curl -fsSL https://bun.sh/install | bash", "apt-get install -y nodejs npm"]', "", "Fallback hints from static detection:", + ...(manifest ? [`- Detected verify environment manifest: ${manifest.path}.`] : []), ...buildProjectContextLines(fallbackProfile), "", "Return ONLY valid JSON with this exact shape:", @@ -680,12 +208,104 @@ export function buildVerifyDetectPrompt(cwd: string, settings?: SandboxSettings) ].join("\n"); } +const NODE_ECOSYSTEMS = new Set(["node", "nodejs", "npm", "bun", "yarn", "pnpm"]); +const PYTHON_ECOSYSTEMS = new Set(["python", "django", "fastapi"]); +const GO_ECOSYSTEMS = new Set(["go", "golang"]); +const RUST_ECOSYSTEMS = new Set(["rust", "cargo"]); + +function inferBootstrapFromEcosystem( + ecosystem: string, + packageManager: string | null, +): { bootstrap: string[]; shellInit: string[] } { + const eco = ecosystem.toLowerCase(); + + if ( + NODE_ECOSYSTEMS.has(eco) || + eco.includes("node") || + eco.includes("next") || + eco.includes("react") || + eco.includes("vite") + ) { + const bootstrap = [ + "apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm", + ]; + const shellInit = [...defaultShellInit()]; + if (packageManager === "bun") { + bootstrap.push("curl -fsSL https://bun.sh/install | bash"); + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template + shellInit.push('export BUN_INSTALL="${HOME}/.bun"'); + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template + shellInit.push('export PATH="${BUN_INSTALL}/bin:$PATH"'); + } + return { bootstrap, shellInit }; + } + + if (PYTHON_ECOSYSTEMS.has(eco) || eco.includes("python") || eco.includes("django") || eco.includes("flask")) { + return { + bootstrap: ["apt-get update && apt-get install -y python3 python3-pip python3-venv ca-certificates git"], + shellInit: defaultShellInit(), + }; + } + + if (GO_ECOSYSTEMS.has(eco) || eco.includes("go")) { + return { + bootstrap: ["apt-get update && apt-get install -y golang ca-certificates git"], + shellInit: defaultShellInit(), + }; + } + + if (RUST_ECOSYSTEMS.has(eco) || eco.includes("rust")) { + return { + bootstrap: [ + "apt-get update && apt-get install -y curl ca-certificates git build-essential && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y", + ], + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template + shellInit: [...defaultShellInit(), 'export PATH="${HOME}/.cargo/bin:$PATH"'], + }; + } + + return { bootstrap: [], shellInit: [] }; +} + +function ensureBootstrapCommands(cwd: string, recipe: VerifyRecipe): VerifyRecipe { + if (recipe.bootstrapCommands.length > 0) return recipe; + if (recipe.installCommands.length === 0) return recipe; + + const packageManager = detectPackageManager(cwd); + + const appKind = normalizeVerifyAppKind(recipe.appKind); + const webBootstrap = getNodeWebBootstrapCommands(packageManager, appKind); + if (webBootstrap.length > 0) { + const webShellInit = getNodeWebShellInitCommands(packageManager, appKind); + return { + ...recipe, + bootstrapCommands: webBootstrap, + shellInitCommands: dedupe([...recipe.shellInitCommands, ...webShellInit]), + }; + } + + const { bootstrap, shellInit } = inferBootstrapFromEcosystem(recipe.ecosystem, packageManager); + if (bootstrap.length === 0) return recipe; + + return { + ...recipe, + bootstrapCommands: bootstrap, + shellInitCommands: dedupe([...recipe.shellInitCommands, ...shellInit]), + }; +} + export function createVerifyRuntimeConfig( cwd: string, baseSettings: SandboxSettings = {}, recipeOverride?: VerifyRecipe | null, ): VerifyRuntimeConfig { - const profile = inferVerifyProjectProfile(cwd, baseSettings, recipeOverride); + const manifest = recipeOverride ? null : loadVerifyEnvironment(cwd, baseSettings); + const profile = inferVerifyProjectProfile( + cwd, + manifest?.sandboxSettings ?? baseSettings, + recipeOverride ?? manifest?.recipe ?? null, + ); + profile.recipe = ensureBootstrapCommands(cwd, profile.recipe); const sandboxSettings = { ...profile.sandboxSettings, allowNet: true, @@ -693,7 +313,7 @@ export function createVerifyRuntimeConfig( allowEphemeralInstall: true, hostBrowserCommandsOnHost: true, }; - sandboxSettings.shellInit = profile.recipe.shellInitCommands; + sandboxSettings.shellInit = dedupe([...(sandboxSettings.shellInit ?? []), ...profile.recipe.shellInitCommands]); return { sandboxMode: "shuru", sandboxSettings, @@ -702,33 +322,101 @@ export function createVerifyRuntimeConfig( }; } -export const VERIFY_PROMPT = [ - "Verify this project locally. Follow these steps in order using the `task` tool:", - "", - "Step 1: Run the `verify-detect` sub-agent to inspect the repository and produce a verification recipe.", - "- agent: verify-detect", - '- description: "Detect verification recipe"', - "- The prompt should ask it to read config files, package manifests, scripts, and source layout, then return ONLY a JSON VerifyRecipe object.", - "- The JSON must include: ecosystem, appKind, appLabel, shellInitCommands, bootstrapCommands, installCommands, buildCommands, testCommands, startCommand, startPort, smokeKind, smokeTarget, evidence, notes.", - "", - "Step 2: Once you have the recipe JSON from step 1, run the `verify` sub-agent to execute it.", - "- agent: verify", - '- description: "Run local verification"', - "- You MUST include the complete recipe JSON from step 1 in the verify prompt.", - "- You MUST also include these execution instructions in the verify prompt:", - " - The verify agent runs inside a Shuru sandbox with full network access enabled.", - " - Ephemeral dependency installs are allowed inside the sandbox.", - " - agent-browser commands run on the HOST, not inside the sandbox. They WILL work.", - " - If the recipe has a startCommand and startPort, start the app in the background and run browser smoke tests with agent-browser against http://127.0.0.1:.", - " - Use `agent-browser record start .grok/verify-artifacts/verify-smoke.webm` before opening the page and `agent-browser record stop` after to capture video.", - " - Use `agent-browser --screenshot-dir .grok/verify-artifacts screenshot` to capture screenshots.", - " - Produce a structured verification report with: Summary, Recipe, Environment, Commands Run, Browser Checks, Evidence (with file paths), Results, Failures or Blockers, Residual Risk.", - "", - "Important:", - "- Do NOT perform verification work yourself. Delegate each step to the respective sub-agent using the `task` tool.", - "- Run step 1 first, then step 2. Do not skip or combine them.", - "- After step 2 completes, relay the verification report back to the user.", -].join("\n"); +export async function prepareVerifySandbox( + cwd: string, + baseSettings: SandboxSettings = {}, + recipeOverride?: VerifyRecipe | null, + onProgress?: (detail: string) => void, +): Promise { + const runtime = createVerifyRuntimeConfig(cwd, baseSettings, recipeOverride); + onProgress?.("Preparing verify checkpoint"); + const checkpoint = await ensureVerifyCheckpoint(cwd, runtime.profile, runtime.sandboxSettings, onProgress); + if (checkpoint.checkpointName) { + runtime.sandboxSettings.from = checkpoint.checkpointName; + if (checkpoint.guestWorkdir) { + runtime.sandboxSettings.guestWorkdir = checkpoint.guestWorkdir; + runtime.sandboxSettings.syncHostWorkspace = true; + } + onProgress?.( + checkpoint.created + ? `Created verify checkpoint: ${checkpoint.checkpointName}` + : `Using verify checkpoint: ${checkpoint.checkpointName}`, + ); + } else { + onProgress?.("No verify checkpoint needed"); + } + return { + profile: runtime.profile, + sandboxSettings: runtime.sandboxSettings, + checkpoint, + }; +} + +export function buildVerifyPrompt(cwd: string): string { + const profile = inferVerifyProjectProfile(cwd); + const recipe = ensureBootstrapCommands(cwd, profile.recipe); + const draftJson = JSON.stringify( + { + recipe: { + ecosystem: recipe.ecosystem, + appKind: recipe.appKind, + appLabel: recipe.appLabel, + shellInitCommands: recipe.shellInitCommands, + bootstrapCommands: recipe.bootstrapCommands, + installCommands: recipe.installCommands, + buildCommands: recipe.buildCommands, + testCommands: recipe.testCommands, + startCommand: recipe.startCommand, + startPort: recipe.startPort, + smokeKind: recipe.smokeKind, + smokeTarget: recipe.smokeTarget, + evidence: recipe.evidence, + notes: recipe.notes, + }, + }, + null, + 2, + ); + + return [ + "Verify this project locally. Follow these steps in order using the `task` tool:", + "", + "Step 1: Run the `verify-manifest` sub-agent to create or update `.grok/environment.json`.", + "- agent: verify-manifest", + '- description: "Create verify manifest"', + "- Include this draft manifest JSON directly in the prompt so the sub-agent can refine and write it quickly:", + "", + "```json", + draftJson, + "```", + "", + "- Tell the sub-agent: Review this draft recipe against package.json and key config files. Adjust if needed, then write .grok/environment.json. If the draft looks correct, write it as-is. Do not over-research.", + "- IMPORTANT: The Shuru sandbox is Debian 13 trixie on aarch64 (ARM64) with NOTHING pre-installed. bootstrapCommands MUST install every runtime from scratch (e.g. nodejs, npm, python3, go, cargo).", + "", + "Step 2: After the manifest is written, run the `verify` sub-agent.", + "- agent: verify", + '- description: "Run local verification"', + "- Tell it to use `.grok/environment.json` as the source of truth.", + "- Include these execution instructions:", + " - Runs inside a Shuru sandbox (Debian 13 trixie, aarch64/ARM64) with full network access.", + " - The sandbox has NOTHING pre-installed unless bootstrapCommands in the manifest already ran during checkpoint creation.", + " - Ephemeral installs allowed.", + " - Probe for runtimes first (`command -v node`, etc), only install what is missing.", + " - agent-browser runs on the HOST, not the sandbox. It WILL work.", + " - If recipe has startCommand + startPort, start app in background and run browser smoke tests.", + " - Use `agent-browser record start .grok/verify-artifacts/verify-smoke.webm` before opening the page.", + " - Use `agent-browser --screenshot-dir .grok/verify-artifacts screenshot` after the page loads.", + " - CRITICAL: Stop the recording (`agent-browser record stop`) and close the browser (`agent-browser close`) BEFORE stopping the dev server. The server must stay alive until all browser commands finish.", + " - Return a concise report: Summary, Results, Evidence (mandatory), Blockers, Residual Risk.", + "", + "Important:", + "- Do NOT perform verification work yourself. Delegate each step via `task`.", + "- Run step 1 first, then step 2.", + "- After step 2, relay the verification report to the user.", + ].join("\n"); +} + +export const VERIFY_PROMPT = "__DYNAMIC__"; export function getVerifyCliError(options: { hasPrompt?: boolean; hasMessageArgs?: boolean }): string | null { if (options.hasPrompt) { @@ -741,3 +429,11 @@ export function getVerifyCliError(options: { hasPrompt?: boolean; hasMessageArgs return null; } + +export { + defaultShellInit, + inferVerifyProjectProfile, + inferVerifySmokeUrl, + normalizeVerifyRecipe, + type VerifyProjectProfile, +} from "./recipes"; diff --git a/src/verify/environment.test.ts b/src/verify/environment.test.ts new file mode 100644 index 0000000..a1cce89 --- /dev/null +++ b/src/verify/environment.test.ts @@ -0,0 +1,119 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadVerifyEnvironment, saveVerifyEnvironment } from "./environment"; + +const tempDirs: string[] = []; + +function makeTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("verify environment manifest", () => { + it("loads a root environment.json manifest and normalizes sandbox settings", () => { + const dir = makeTempDir("grok-verify-env-"); + fs.writeFileSync( + path.join(dir, "environment.json"), + JSON.stringify( + { + ecosystem: "node", + appKind: "nextjs", + appLabel: "Demo app", + install: ["npm ci"], + build: ["npm run build"], + start: "npm run start", + startPort: "3000", + smokeKind: "http", + sandbox: { + from: "verify-node-web", + verifyBaseFrom: "verify-node-web-base", + shellInit: ["export FOO=bar"], + }, + }, + null, + 2, + ), + ); + + const loaded = loadVerifyEnvironment(dir, { allowNet: true }); + expect(loaded?.path).toBe(path.join(dir, "environment.json")); + expect(loaded?.recipe.installCommands).toEqual(["npm ci"]); + expect(loaded?.recipe.buildCommands).toEqual(["npm run build"]); + expect(loaded?.recipe.startCommand).toBe("npm run start"); + expect(loaded?.sandboxSettings.from).toBe("verify-node-web"); + expect(loaded?.sandboxSettings.verifyBaseFrom).toBe("verify-node-web-base"); + expect(loaded?.sandboxSettings.shellInit).toEqual(["export FOO=bar"]); + expect(loaded?.sandboxSettings.allowNet).toBe(true); + }); + + it("prefers .grok/environment.json over the root manifest", () => { + const dir = makeTempDir("grok-verify-env-precedence-"); + fs.writeFileSync( + path.join(dir, "environment.json"), + JSON.stringify({ + ecosystem: "node", + appKind: "vite", + appLabel: "Root", + install: ["npm install"], + smokeKind: "none", + }), + ); + fs.mkdirSync(path.join(dir, ".grok")); + fs.writeFileSync( + path.join(dir, ".grok", "environment.json"), + JSON.stringify({ + ecosystem: "node", + appKind: "nextjs", + appLabel: "Nested", + install: ["npm ci"], + smokeKind: "none", + }), + ); + + const loaded = loadVerifyEnvironment(dir); + expect(loaded?.path).toBe(path.join(dir, ".grok", "environment.json")); + expect(loaded?.recipe.appKind).toBe("nextjs"); + expect(loaded?.recipe.installCommands).toEqual(["npm ci"]); + }); + + it("writes a generated .grok/environment.json manifest", () => { + const dir = makeTempDir("grok-verify-env-write-"); + const written = saveVerifyEnvironment( + dir, + { + ecosystem: "node", + appKind: "nextjs", + appLabel: "Generated app", + shellInitCommands: ["export DEBIAN_FRONTEND=noninteractive"], + bootstrapCommands: ["apt-get update"], + installCommands: ["npm ci"], + buildCommands: ["npm run build"], + testCommands: ["npm run lint"], + startCommand: "npm run start", + startPort: "3000", + smokeKind: "http", + smokeTarget: "http://127.0.0.1:3000", + evidence: ["Detected package.json"], + notes: ["Generated by verify"], + }, + { verifyBaseFrom: "verify-node-web", ports: ["3000:3000"], allowEphemeralInstall: true }, + ); + + expect(written).toBe(path.join(dir, ".grok", "environment.json")); + const loaded = loadVerifyEnvironment(dir); + expect(loaded?.path).toBe(written); + expect(loaded?.recipe.installCommands).toEqual(["npm ci"]); + expect(loaded?.sandboxSettings.verifyBaseFrom).toBe("verify-node-web"); + expect(loaded?.sandboxSettings.ports).toEqual(["3000:3000"]); + expect(loaded?.sandboxSettings.allowEphemeralInstall).toBeUndefined(); + }); +}); diff --git a/src/verify/environment.ts b/src/verify/environment.ts new file mode 100644 index 0000000..afc1163 --- /dev/null +++ b/src/verify/environment.ts @@ -0,0 +1,115 @@ +import * as fs from "fs"; +import * as path from "path"; +import type { VerifyEnvironmentManifest, VerifyRecipe } from "../types/index"; +import { mergeSandboxSettings, normalizeSandboxSettings, type SandboxSettings } from "../utils/settings"; +import { normalizeVerifyRecipe } from "./recipes"; + +export interface LoadedVerifyEnvironment { + path: string; + recipe: VerifyRecipe; + sandboxSettings: SandboxSettings; +} + +const VERIFY_ENVIRONMENT_FILES = [".grok/environment.json", "environment.json"]; +const GENERATED_VERIFY_ENVIRONMENT = ".grok/environment.json"; + +function readJson(filePath: string): VerifyEnvironmentManifest | null { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")) as VerifyEnvironmentManifest; + } catch { + return null; + } +} + +function asStringArray(value: unknown): string[] { + if (typeof value === "string" && value.trim()) return [value.trim()]; + if (!Array.isArray(value)) return []; + return value + .filter((entry): entry is string => typeof entry === "string" && entry.trim() !== "") + .map((entry) => entry.trim()); +} + +function normalizeManifestRecipe(raw: VerifyEnvironmentManifest): VerifyRecipe | null { + const base = raw.recipe && typeof raw.recipe === "object" ? raw.recipe : raw; + return normalizeVerifyRecipe({ + ecosystem: base.ecosystem ?? raw.ecosystem ?? "custom", + appKind: base.appKind ?? raw.appKind ?? "unknown", + appLabel: base.appLabel ?? raw.appLabel ?? "Custom verify environment", + shellInitCommands: base.shellInitCommands ?? base.shellInit ?? raw.shellInitCommands ?? raw.shellInit ?? [], + bootstrapCommands: base.bootstrapCommands ?? base.bootstrap ?? raw.bootstrapCommands ?? raw.bootstrap ?? [], + installCommands: base.installCommands ?? base.install ?? raw.installCommands ?? raw.install ?? [], + buildCommands: base.buildCommands ?? base.build ?? raw.buildCommands ?? raw.build ?? [], + testCommands: base.testCommands ?? base.test ?? raw.testCommands ?? raw.test ?? [], + startCommand: base.startCommand ?? base.start ?? raw.startCommand ?? raw.start, + startPort: base.startPort ?? raw.startPort, + smokeKind: base.smokeKind ?? raw.smokeKind ?? "none", + smokeTarget: base.smokeTarget ?? raw.smokeTarget, + evidence: asStringArray(base.evidence ?? raw.evidence), + notes: asStringArray(base.notes ?? raw.notes), + }); +} + +export function loadVerifyEnvironment(cwd: string, baseSettings: SandboxSettings = {}): LoadedVerifyEnvironment | null { + for (const candidate of VERIFY_ENVIRONMENT_FILES) { + const filePath = path.join(cwd, candidate); + if (!fs.existsSync(filePath)) continue; + const manifest = readJson(filePath); + if (!manifest) continue; + const recipe = normalizeManifestRecipe(manifest); + if (!recipe) continue; + const sandboxSettings = mergeSandboxSettings(baseSettings, normalizeSandboxSettings(manifest.sandbox)); + return { + path: filePath, + recipe, + sandboxSettings, + }; + } + return null; +} + +function pickPersistentSandboxSettings(settings: SandboxSettings): SandboxSettings { + const persistent: SandboxSettings = {}; + if (settings.allowNet !== undefined) persistent.allowNet = settings.allowNet; + if (settings.allowedHosts?.length) persistent.allowedHosts = settings.allowedHosts; + if (settings.ports?.length) persistent.ports = settings.ports; + if (settings.cpus) persistent.cpus = settings.cpus; + if (settings.memory) persistent.memory = settings.memory; + if (settings.diskSize) persistent.diskSize = settings.diskSize; + if (settings.secrets?.length) persistent.secrets = settings.secrets; + if (settings.from) persistent.from = settings.from; + if (settings.verifyBaseFrom) persistent.verifyBaseFrom = settings.verifyBaseFrom; + if (settings.guestWorkdir) persistent.guestWorkdir = settings.guestWorkdir; + if (settings.syncHostWorkspace !== undefined) persistent.syncHostWorkspace = settings.syncHostWorkspace; + if (settings.shellInit?.length) persistent.shellInit = settings.shellInit; + return persistent; +} + +export function saveVerifyEnvironment( + cwd: string, + recipe: VerifyRecipe, + sandboxSettings: SandboxSettings = {}, +): string { + const filePath = path.join(cwd, GENERATED_VERIFY_ENVIRONMENT); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const payload: VerifyEnvironmentManifest = { + recipe: { + ecosystem: recipe.ecosystem, + appKind: recipe.appKind, + appLabel: recipe.appLabel, + shellInitCommands: recipe.shellInitCommands, + bootstrapCommands: recipe.bootstrapCommands, + installCommands: recipe.installCommands, + buildCommands: recipe.buildCommands, + testCommands: recipe.testCommands, + startCommand: recipe.startCommand, + startPort: recipe.startPort, + smokeKind: recipe.smokeKind, + smokeTarget: recipe.smokeTarget, + evidence: recipe.evidence, + notes: recipe.notes, + }, + sandbox: pickPersistentSandboxSettings(sandboxSettings), + }; + fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + return filePath; +} diff --git a/src/verify/evidence.ts b/src/verify/evidence.ts new file mode 100644 index 0000000..b103ffa --- /dev/null +++ b/src/verify/evidence.ts @@ -0,0 +1,104 @@ +import type { VerifyArtifact } from "../types/index"; +import type { VerifyProjectProfile } from "./recipes"; + +export const VERIFY_ARTIFACT_DIR = ".grok/verify-artifacts"; +export const VERIFY_SCREENSHOT_PATH = `${VERIFY_ARTIFACT_DIR}/verify-screenshot.png`; +export const VERIFY_VIDEO_PATH = `${VERIFY_ARTIFACT_DIR}/verify-smoke.webm`; +export const VERIFY_LOG_PATH = `${VERIFY_ARTIFACT_DIR}/app.log`; + +export function getVerifyArtifacts(): VerifyArtifact[] { + return [ + { kind: "log", path: VERIFY_LOG_PATH, description: "Captured app stdout/stderr from the verify run" }, + { + kind: "screenshot", + path: VERIFY_SCREENSHOT_PATH, + description: "Final browser screenshot from smoke verification", + }, + { kind: "video", path: VERIFY_VIDEO_PATH, description: "Recorded browser smoke flow" }, + ]; +} + +export function buildReadinessGuidance(profile: VerifyProjectProfile): string[] { + if (profile.recipe.smokeKind !== "http" || !profile.recipe.startCommand || !profile.recipe.startPort) { + return [ + "- If you start any process, capture stdout/stderr into a log file under .grok/verify-artifacts whenever practical.", + ]; + } + + const target = profile.recipe.smokeTarget ?? `http://127.0.0.1:${profile.recipe.startPort}`; + return [ + `- Start the app with logs redirected to ${VERIFY_LOG_PATH} so failures leave evidence.`, + `- Before running browser checks, wait for readiness at ${target}. Prefer a bounded curl loop or equivalent readiness check instead of guessing.`, + "- If readiness never succeeds, include the final log tail and exact readiness command output in the report.", + ]; +} + +export function buildBrowserGuidance(profile: VerifyProjectProfile): string[] { + if (profile.recipe.smokeKind === "http" && profile.recipe.smokeTarget) { + return [ + `- REQUIRED: After the dev server is running, you MUST run browser smoke tests against ${profile.recipe.smokeTarget}.`, + "- The agent-browser command runs on the HOST, not inside the sandbox. It WILL work. Do not skip it or assume it is unavailable.", + "- CRITICAL SEQUENCING: The dev server MUST stay running for the entire browser test. Do NOT stop the server until after recording is stopped and the browser is closed.", + "", + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable reference in prompt text + "- IMPORTANT: All artifact paths MUST be absolute. Use `$(pwd)/${VERIFY_ARTIFACT_DIR}` to construct them. Example: `$(pwd)/.grok/verify-artifacts/verify-smoke.webm`.", + "", + "- Browser smoke test procedure (run each step as a SEPARATE bash tool call):", + ` 1. mkdir -p ${VERIFY_ARTIFACT_DIR}`, + ` 2. agent-browser record start "$(pwd)/${VERIFY_VIDEO_PATH}"`, + " 3. sleep 1", + ` 4. agent-browser --screenshot-dir "$(pwd)/${VERIFY_ARTIFACT_DIR}" open ${profile.recipe.smokeTarget}`, + " 5. agent-browser wait --load networkidle", + " 6. sleep 2", + ` 7. agent-browser --screenshot-dir "$(pwd)/${VERIFY_ARTIFACT_DIR}" screenshot`, + " 8. agent-browser get title", + "", + "- INTERACTIVE TESTING (do this now, while recording is still running):", + " Act like a real user doing an integration test. The goal is to verify the app actually works, not just that it loads.", + " - Run each agent-browser command as a SEPARATE bash tool call. Do NOT chain them.", + " - After each interaction, run `sleep 2` so the recording captures the result.", + " - Use `agent-browser snapshot` to see the page structure and interactive elements.", + " - Click links, buttons, and navigation using `agent-browser click ` (refs from snapshot).", + " - After each click, `agent-browser wait --load networkidle && sleep 2`, then snapshot and screenshot.", + " - Check for console errors using `agent-browser console`.", + " - Test at least: home page loads with content, primary navigation works, no JS errors in console.", + " - If the app has forms, try filling one. If it has a login page, verify it renders.", + " - Spend 3-5 interactions exploring the critical path. Do not rush.", + "", + "- Teardown (after interactive testing, run each as a separate bash call):", + " 9. agent-browser record stop", + " 10. sleep 2", + ` 11. ls -la "$(pwd)/${VERIFY_VIDEO_PATH}" (verify the video file exists and has a non-zero size)`, + " 12. agent-browser close", + " 13. Only NOW stop the dev server (kill the background process).", + "", + `- Keep all screenshots and video under ${VERIFY_ARTIFACT_DIR}. Include their exact paths in the Evidence section.`, + "- If the video file does not exist after `record stop`, note that in Blockers.", + "- If any browser command fails, report the exact error output. Do not skip browser checks.", + ]; + } + + if ((profile.sandboxSettings.ports?.length ?? 0) > 1) { + return [ + "- Multiple forwarded ports are configured, so browser smoke testing is ambiguous.", + "- Skip browser checks unless the user clearly identifies which forwarded localhost URL to verify.", + ]; + } + + if (profile.recipe.smokeKind === "cli") { + return ["- This project appears to need CLI-style runtime validation rather than browser smoke testing."]; + } + + return [ + "- No unambiguous forwarded localhost URL is configured, so browser smoke testing is optional and should usually be skipped.", + "- If the app can still be verified with bash-only checks, do that and explain why browser checks were skipped.", + ]; +} + +export function buildEvidenceGuidance(): string[] { + const artifacts = getVerifyArtifacts(); + return [ + "- Always include exact artifact file paths in the Evidence section when they exist.", + ...artifacts.map((artifact) => `- Artifact target: ${artifact.path} (${artifact.description}).`), + ]; +} diff --git a/src/verify/orchestrator.test.ts b/src/verify/orchestrator.test.ts new file mode 100644 index 0000000..a03f18d --- /dev/null +++ b/src/verify/orchestrator.test.ts @@ -0,0 +1,159 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("./checkpoint", () => ({ + ensureVerifyCheckpoint: vi.fn(async () => ({ created: false })), +})); + +import { ensureVerifyCheckpoint } from "./checkpoint"; +import { prepareVerifyRun, runVerifyOrchestration } from "./orchestrator"; + +const ensureVerifyCheckpointMock = vi.mocked(ensureVerifyCheckpoint); +const tempDirs: string[] = []; + +function makeTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + ensureVerifyCheckpointMock.mockReset(); + ensureVerifyCheckpointMock.mockResolvedValue({ created: false }); + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("verify orchestrator", () => { + it("uses the environment manifest as the highest-priority recipe source", async () => { + const dir = makeTempDir("grok-verify-orch-manifest-"); + fs.writeFileSync( + path.join(dir, "environment.json"), + JSON.stringify( + { + ecosystem: "custom", + appKind: "node", + appLabel: "Manifest app", + install: ["npm ci"], + build: ["npm run build"], + smokeKind: "none", + }, + null, + 2, + ), + ); + + const agent = { + getCwd: () => dir, + getSandboxSettings: () => ({}), + setSandboxSettings: vi.fn(), + detectVerifyRecipe: vi.fn(async () => null), + runTaskRequest: vi.fn(async () => ({ success: true, output: "ok" })), + }; + + const prepared = await prepareVerifyRun(agent, {}); + expect(prepared.profile.recipe.ecosystem).toBe("custom"); + expect(prepared.profile.recipe.installCommands).toEqual(["npm ci"]); + expect(prepared.usedVerifyDetect).toBe(false); + expect(agent.detectVerifyRecipe).not.toHaveBeenCalled(); + }); + + it("creates .grok/environment.json from verify-detect when no manifest exists", async () => { + const dir = makeTempDir("grok-verify-orch-generate-"); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2), + ); + fs.writeFileSync(path.join(dir, "package-lock.json"), ""); + + const agent = { + getCwd: () => dir, + getSandboxSettings: () => ({}), + setSandboxSettings: vi.fn(), + detectVerifyRecipe: vi.fn(async () => ({ + ecosystem: "custom", + appKind: "node", + appLabel: "Agent recipe", + shellInitCommands: ["export FOO=bar"], + bootstrapCommands: ["echo bootstrap"], + installCommands: ["npm ci"], + buildCommands: ["npm run build"], + testCommands: [], + startCommand: "npm run start", + startPort: "3000", + smokeKind: "http" as const, + smokeTarget: "http://127.0.0.1:3000", + evidence: ["agent-detect"], + notes: ["generated"], + })), + runTaskRequest: vi.fn(async () => ({ success: true, output: "ok" })), + }; + + const prepared = await prepareVerifyRun(agent, {}); + expect(prepared.manifestPath).toBe(path.join(dir, ".grok", "environment.json")); + expect(fs.existsSync(prepared.manifestPath!)).toBe(true); + expect(prepared.usedVerifyDetect).toBe(true); + expect(prepared.profile.recipe.installCommands).toEqual(["npm ci"]); + expect(prepared.profile.recipe.evidence).toContain("agent-detect"); + }); + + it("does not create a manifest when verify-detect returns no recipe", async () => { + const dir = makeTempDir("grok-verify-orch-no-manifest-"); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2), + ); + fs.writeFileSync(path.join(dir, "package-lock.json"), ""); + + const agent = { + getCwd: () => dir, + getSandboxSettings: () => ({}), + setSandboxSettings: vi.fn(), + detectVerifyRecipe: vi.fn(async () => null), + runTaskRequest: vi.fn(async () => ({ success: true, output: "ok" })), + }; + + const prepared = await prepareVerifyRun(agent, {}); + expect(prepared.manifestPath).toBeUndefined(); + expect(fs.existsSync(path.join(dir, ".grok", "environment.json"))).toBe(false); + expect(prepared.usedVerifyDetect).toBe(false); + }); + + it("restores sandbox settings after running verification and wires checkpoint settings", async () => { + const dir = makeTempDir("grok-verify-orch-run-"); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2), + ); + fs.writeFileSync(path.join(dir, "package-lock.json"), ""); + + ensureVerifyCheckpointMock.mockResolvedValue({ + created: true, + checkpointName: "verify-next-demo", + guestWorkdir: "/grok/verify/worktree", + }); + + const originalSettings = { allowNet: false, from: "base" }; + const agent = { + getCwd: () => dir, + getSandboxSettings: vi.fn(() => originalSettings), + setSandboxSettings: vi.fn(), + detectVerifyRecipe: vi.fn(async () => null), + runTaskRequest: vi.fn(async () => ({ success: true, output: "verified" })), + }; + + const result = await runVerifyOrchestration(agent, {}); + expect(result.success).toBe(true); + expect(agent.setSandboxSettings).toHaveBeenCalledTimes(2); + expect(agent.setSandboxSettings.mock.calls[0]?.[0]).toMatchObject({ + from: "verify-next-demo", + guestWorkdir: "/grok/verify/worktree", + syncHostWorkspace: true, + allowNet: true, + }); + expect(agent.setSandboxSettings.mock.calls[1]?.[0]).toBe(originalSettings); + }); +}); diff --git a/src/verify/orchestrator.ts b/src/verify/orchestrator.ts new file mode 100644 index 0000000..e0a8572 --- /dev/null +++ b/src/verify/orchestrator.ts @@ -0,0 +1,129 @@ +import type { TaskRequest, ToolResult, VerifyRecipe } from "../types/index"; +import type { SandboxSettings } from "../utils/settings"; +import { ensureVerifyCheckpoint, type PreparedVerifyCheckpoint } from "./checkpoint"; +import { buildVerifyTaskPrompt } from "./entrypoint"; +import { loadVerifyEnvironment, saveVerifyEnvironment } from "./environment"; +import { inferVerifyProjectProfile, type VerifyProjectProfile } from "./recipes"; + +export interface VerifyAgentLike { + getCwd(): string; + getSandboxSettings(): SandboxSettings; + setSandboxSettings(settings: SandboxSettings): void; + detectVerifyRecipe(settings?: SandboxSettings, abortSignal?: AbortSignal): Promise; + runTaskRequest( + request: TaskRequest, + onActivity?: (detail: string) => void, + abortSignal?: AbortSignal, + ): Promise; +} + +export interface PreparedVerifyRun { + profile: VerifyProjectProfile; + sandboxSettings: SandboxSettings; + taskRequest: TaskRequest; + checkpoint?: PreparedVerifyCheckpoint; + manifestPath?: string; + usedVerifyDetect: boolean; +} + +export interface VerifyOrchestratorOptions { + onProgress?: (detail: string) => void; + abortSignal?: AbortSignal; +} + +function buildRuntimeSandboxSettings(profile: VerifyProjectProfile): SandboxSettings { + return { + ...profile.sandboxSettings, + allowNet: true, + allowedHosts: undefined, + allowEphemeralInstall: true, + hostBrowserCommandsOnHost: true, + shellInit: [...new Set([...(profile.sandboxSettings.shellInit ?? []), ...profile.recipe.shellInitCommands])], + }; +} + +export async function prepareVerifyRun( + agent: VerifyAgentLike, + options: VerifyOrchestratorOptions = {}, +): Promise { + const cwd = agent.getCwd(); + const baseSettings = agent.getSandboxSettings(); + const manifest = loadVerifyEnvironment(cwd, baseSettings); + + if (manifest) { + options.onProgress?.(`Loaded verify environment manifest: ${manifest.path}`); + } else { + options.onProgress?.("No verify environment manifest found; running verify-detect to generate one"); + } + + let profile = inferVerifyProjectProfile(cwd, manifest?.sandboxSettings ?? baseSettings, manifest?.recipe ?? null); + let usedVerifyDetect = false; + let manifestPath = manifest?.path; + + if (!manifest) { + const detectedRecipe = await agent.detectVerifyRecipe(baseSettings, options.abortSignal); + if (detectedRecipe) { + usedVerifyDetect = true; + profile = inferVerifyProjectProfile(cwd, baseSettings, detectedRecipe); + options.onProgress?.(`verify-detect selected recipe for ${profile.appLabel}`); + manifestPath = saveVerifyEnvironment(cwd, profile.recipe, profile.sandboxSettings); + options.onProgress?.(`Created verify environment manifest: ${manifestPath}`); + } else { + options.onProgress?.( + "verify-detect did not return a usable recipe; keeping deterministic fallback without writing a manifest", + ); + } + } + + const sandboxSettings = buildRuntimeSandboxSettings(profile); + options.onProgress?.("Preparing verify checkpoint"); + const checkpoint = await ensureVerifyCheckpoint(cwd, profile, sandboxSettings); + if (checkpoint.checkpointName) { + sandboxSettings.from = checkpoint.checkpointName; + if (checkpoint.guestWorkdir) { + sandboxSettings.guestWorkdir = checkpoint.guestWorkdir; + sandboxSettings.syncHostWorkspace = true; + } + options.onProgress?.( + checkpoint.created + ? `Created verify checkpoint: ${checkpoint.checkpointName}` + : `Using verify checkpoint: ${checkpoint.checkpointName}`, + ); + } else { + options.onProgress?.("No verify checkpoint needed for this recipe"); + } + + const taskRequest: TaskRequest = { + agent: "verify", + description: "Run local verification", + prompt: buildVerifyTaskPrompt(cwd, sandboxSettings, profile.recipe), + }; + + return { + profile, + sandboxSettings, + taskRequest, + checkpoint, + manifestPath, + usedVerifyDetect, + }; +} + +export async function runVerifyOrchestration( + agent: VerifyAgentLike, + options: VerifyOrchestratorOptions = {}, +): Promise { + const originalSandboxSettings = agent.getSandboxSettings(); + const prepared = await prepareVerifyRun(agent, options); + agent.setSandboxSettings(prepared.sandboxSettings); + try { + options.onProgress?.("Running verify sub-agent"); + const result = await agent.runTaskRequest(prepared.taskRequest, options.onProgress, options.abortSignal); + return { + ...result, + verifyRecipe: prepared.profile.recipe, + }; + } finally { + agent.setSandboxSettings(originalSandboxSettings); + } +} diff --git a/src/verify/recipes.ts b/src/verify/recipes.ts new file mode 100644 index 0000000..990469b --- /dev/null +++ b/src/verify/recipes.ts @@ -0,0 +1,516 @@ +import * as fs from "fs"; +import * as path from "path"; +import type { VerifyRecipe } from "../types/index"; +import { mergeSandboxSettings, type SandboxSettings } from "../utils/settings"; + +export type VerifyAppKind = + | "nextjs" + | "vite" + | "astro" + | "sveltekit" + | "remix" + | "cra" + | "node" + | "django" + | "python" + | "go" + | "rust" + | "maven" + | "gradle" + | "make" + | "unknown"; + +export interface VerifyProjectProfile { + appKind: VerifyAppKind; + appLabel: string; + packageManager: string | null; + availableScripts: string[]; + hasNodeModules: boolean; + sandboxSettings: SandboxSettings; + recipe: VerifyRecipe; +} + +interface PackageJsonLike { + dependencies?: Record; + devDependencies?: Record; + scripts?: Record; +} + +function fileExists(cwd: string, file: string): boolean { + return fs.existsSync(path.join(cwd, file)); +} + +function readTextFile(cwd: string, file: string): string | null { + try { + return fs.readFileSync(path.join(cwd, file), "utf8"); + } catch { + return null; + } +} + +function readPackageJson(cwd: string): PackageJsonLike | null { + const raw = readTextFile(cwd, "package.json"); + if (!raw) return null; + try { + return JSON.parse(raw) as PackageJsonLike; + } catch { + return null; + } +} + +export function detectPackageManager(cwd: string): string | null { + const candidates: Array<[string, string]> = [ + ["pnpm-lock.yaml", "pnpm"], + ["bun.lock", "bun"], + ["bun.lockb", "bun"], + ["yarn.lock", "yarn"], + ["package-lock.json", "npm"], + ["uv.lock", "uv"], + ["poetry.lock", "poetry"], + ["Pipfile.lock", "pipenv"], + ]; + + for (const [file, manager] of candidates) { + if (fileExists(cwd, file)) return manager; + } + + return null; +} + +function dedupe(values: Array): string[] { + return [...new Set(values.map((v) => v?.trim()).filter((v): v is string => Boolean(v)))]; +} + +export function defaultShellInit(): string[] { + return ["export DEBIAN_FRONTEND=noninteractive"]; +} + +const NODE_WEB_APP_KINDS = new Set(["nextjs", "vite", "astro", "sveltekit", "remix", "cra"]); + +export function getNodeWebShellInitCommands(packageManager: string | null, appKind: VerifyAppKind): string[] { + const commands = [...defaultShellInit()]; + if (!NODE_WEB_APP_KINDS.has(appKind)) { + return commands; + } + if (packageManager === "bun") { + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template + commands.push('export BUN_INSTALL="${HOME}/.bun"'); + // biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template + commands.push('export PATH="${BUN_INSTALL}/bin:$PATH"'); + } + return commands; +} + +export function getNodeWebBootstrapCommands(packageManager: string | null, appKind: VerifyAppKind): string[] { + if (!NODE_WEB_APP_KINDS.has(appKind)) { + return []; + } + const commands = [ + "apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm", + ]; + if (packageManager === "bun") { + commands.push("curl -fsSL https://bun.sh/install | bash"); + } + return commands; +} + +function parseHostPort(mapping: string): string | null { + const match = mapping.trim().match(/^(\d+):(\d+)$/); + return match ? match[1] : null; +} + +function inferPortFromCommand(command: string | undefined): string | undefined { + if (!command) return undefined; + const flagMatch = command.match(/(?:--port|-p)\s+(\d{2,5})/); + if (flagMatch) return flagMatch[1]; + const envMatch = command.match(/\bPORT=(\d{2,5})\b/); + if (envMatch) return envMatch[1]; + return undefined; +} + +function parseTargetNames(raw: string): string[] { + return raw + .split(/\r?\n/) + .map((line) => line.match(/^([A-Za-z0-9_.-]+):(?:\s|$)/)?.[1]) + .filter((target): target is string => Boolean(target)); +} + +export function normalizeVerifyAppKind(value: string): VerifyAppKind { + return ( + [ + "nextjs", + "vite", + "astro", + "sveltekit", + "remix", + "cra", + "node", + "django", + "python", + "go", + "rust", + "maven", + "gradle", + "make", + "unknown", + ] as const + ).includes(value as VerifyAppKind) + ? (value as VerifyAppKind) + : "unknown"; +} + +function pickPackageScript(packageManager: string | null, scripts: Record, body: string): string { + const entry = Object.entries(scripts).find(([, scriptBody]) => scriptBody === body)?.[0]; + if (!entry) return body; + const runner = + packageManager === "pnpm" + ? "pnpm" + : packageManager === "bun" + ? "bun" + : packageManager === "yarn" + ? "yarn" + : "npm run"; + return runner === "yarn" ? `yarn ${entry}` : runner === "bun" ? `bun run ${entry}` : `${runner} ${entry}`; +} + +function detectMakeRecipe(cwd: string): VerifyRecipe | null { + const makefile = readTextFile(cwd, "Makefile"); + if (!makefile) return null; + const targets = parseTargetNames(makefile); + const has = (names: string[]) => names.find((name) => targets.includes(name)); + const install = has(["install", "setup", "bootstrap"]); + const build = has(["build", "compile"]); + const test = has(["test", "check"]); + const run = has(["run", "start", "serve", "dev"]); + + return { + ecosystem: "make", + appKind: "make", + appLabel: "Makefile-driven project", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: install ? [`make ${install}`] : [], + buildCommands: build ? [`make ${build}`] : [], + testCommands: test ? [`make ${test}`] : [], + startCommand: run ? `make ${run}` : undefined, + smokeKind: "none", + evidence: ["Detected Makefile", `Targets: ${targets.join(", ") || "(none)"}`], + notes: [], + }; +} + +function detectNodeRecipe(_cwd: string, pkg: PackageJsonLike, packageManager: string | null): VerifyRecipe { + const scripts = pkg.scripts ?? {}; + const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) }; + let appKind: VerifyAppKind = "node"; + let appLabel = "Node.js app"; + let defaultPort: string | undefined; + + if (deps.next) { + appKind = "nextjs"; + appLabel = "Next.js"; + defaultPort = "3000"; + } else if (deps["@sveltejs/kit"]) { + appKind = "sveltekit"; + appLabel = "SvelteKit"; + defaultPort = "5173"; + } else if (deps.astro) { + appKind = "astro"; + appLabel = "Astro"; + defaultPort = "4321"; + } else if (deps["@remix-run/dev"] || deps["@remix-run/react"]) { + appKind = "remix"; + appLabel = "Remix"; + defaultPort = "3000"; + } else if (deps["react-scripts"]) { + appKind = "cra"; + appLabel = "Create React App"; + defaultPort = "3000"; + } else if (deps.vite) { + appKind = "vite"; + appLabel = "Vite"; + defaultPort = "5173"; + } + + const install = packageManager + ? packageManager === "pnpm" + ? "pnpm install" + : packageManager === "bun" + ? "bun install" + : packageManager === "yarn" + ? "yarn install" + : "npm install" + : undefined; + const startCommand = scripts.dev ?? scripts.start; + const startPort = inferPortFromCommand(startCommand) ?? defaultPort; + const smokeKind: VerifyRecipe["smokeKind"] = startCommand && startPort ? "http" : "none"; + + return { + ecosystem: "node", + appKind, + appLabel, + shellInitCommands: getNodeWebShellInitCommands(packageManager, appKind), + bootstrapCommands: getNodeWebBootstrapCommands(packageManager, appKind), + installCommands: dedupe([install]), + buildCommands: dedupe( + [scripts.build, scripts.typecheck].map((script) => script && pickPackageScript(packageManager, scripts, script)), + ), + testCommands: dedupe( + ["test", "check", "lint"] + .filter((name) => scripts[name]) + .map((name) => pickPackageScript(packageManager, scripts, scripts[name]!)), + ), + startCommand: startCommand ? pickPackageScript(packageManager, scripts, startCommand) : undefined, + startPort, + smokeKind, + evidence: ["Detected package.json", `Scripts: ${Object.keys(scripts).join(", ") || "(none)"}`], + notes: [], + }; +} + +function detectPythonRecipe(cwd: string): VerifyRecipe | null { + const pyproject = readTextFile(cwd, "pyproject.toml"); + const requirements = readTextFile(cwd, "requirements.txt"); + const managePy = fileExists(cwd, "manage.py"); + if (!pyproject && !requirements && !managePy && !fileExists(cwd, "setup.py")) { + return null; + } + + const lower = `${pyproject ?? ""}\n${requirements ?? ""}`.toLowerCase(); + const packageManager = detectPackageManager(cwd); + const isDjango = managePy || lower.includes("django"); + const isFastApi = lower.includes("fastapi") || lower.includes("uvicorn"); + + let install = "pip install -r requirements.txt"; + if (packageManager === "uv") install = "uv sync"; + else if (packageManager === "poetry") install = "poetry install"; + else if (packageManager === "pipenv") install = "pipenv install"; + else if (pyproject && !requirements) install = "pip install -e ."; + + if (isDjango) { + return { + ecosystem: "python", + appKind: "django", + appLabel: "Django app", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [install], + buildCommands: [], + testCommands: ["python manage.py test"], + startCommand: "python manage.py runserver 0.0.0.0:8000", + startPort: "8000", + smokeKind: "http", + evidence: ["Detected manage.py", pyproject ? "Detected pyproject.toml" : undefined].filter(Boolean) as string[], + notes: [], + }; + } + + if (isFastApi) { + const appModule = fileExists(cwd, "main.py") ? "main:app" : fileExists(cwd, "app.py") ? "app:app" : "main:app"; + return { + ecosystem: "python", + appKind: "python", + appLabel: "Python web app", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [install], + buildCommands: [], + testCommands: fileExists(cwd, "tests") ? ["pytest"] : [], + startCommand: `uvicorn ${appModule} --host 0.0.0.0 --port 8000`, + startPort: "8000", + smokeKind: "http", + evidence: ["Detected Python project", "Detected FastAPI/Uvicorn dependency"], + notes: [], + }; + } + + return { + ecosystem: "python", + appKind: "python", + appLabel: "Python project", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [install], + buildCommands: [], + testCommands: fileExists(cwd, "tests") ? ["pytest"] : ["python -m unittest discover"], + smokeKind: "none", + evidence: ["Detected Python project"], + notes: [], + }; +} + +function detectGoRecipe(cwd: string): VerifyRecipe | null { + if (!fileExists(cwd, "go.mod")) return null; + return { + ecosystem: "go", + appKind: "go", + appLabel: "Go project", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [], + buildCommands: ["go build ./..."], + testCommands: ["go test ./..."], + startCommand: fileExists(cwd, "main.go") ? "go run ." : undefined, + smokeKind: "none", + evidence: ["Detected go.mod"], + notes: [], + }; +} + +function detectRustRecipe(cwd: string): VerifyRecipe | null { + if (!fileExists(cwd, "Cargo.toml")) return null; + return { + ecosystem: "rust", + appKind: "rust", + appLabel: "Rust project", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [], + buildCommands: ["cargo build"], + testCommands: ["cargo test"], + startCommand: fileExists(cwd, path.join("src", "main.rs")) ? "cargo run" : undefined, + smokeKind: "none", + evidence: ["Detected Cargo.toml"], + notes: [], + }; +} + +function detectJavaRecipe(cwd: string): VerifyRecipe | null { + if (fileExists(cwd, "pom.xml")) { + return { + ecosystem: "java", + appKind: "maven", + appLabel: "Maven project", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [], + buildCommands: ["mvn package"], + testCommands: ["mvn test"], + smokeKind: "none", + evidence: ["Detected pom.xml"], + notes: [], + }; + } + + if (fileExists(cwd, "build.gradle") || fileExists(cwd, "build.gradle.kts")) { + const gradle = fileExists(cwd, "gradlew") ? "./gradlew" : "gradle"; + return { + ecosystem: "java", + appKind: "gradle", + appLabel: "Gradle project", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [], + buildCommands: [`${gradle} build`], + testCommands: [`${gradle} test`], + smokeKind: "none", + evidence: ["Detected Gradle build file"], + notes: [], + }; + } + + return null; +} + +function detectFallbackRecipe(cwd: string): VerifyRecipe { + const makeRecipe = detectMakeRecipe(cwd); + if (makeRecipe) return makeRecipe; + return { + ecosystem: "unknown", + appKind: "unknown", + appLabel: "Unknown project type", + shellInitCommands: defaultShellInit(), + bootstrapCommands: [], + installCommands: [], + buildCommands: [], + testCommands: [], + smokeKind: "none", + evidence: ["No known app metadata detected"], + notes: ["The verify sub-agent should inspect the repo directly and derive commands from the codebase."], + }; +} + +function inferFallbackRecipe(cwd: string, pkg: PackageJsonLike | null, packageManager: string | null): VerifyRecipe { + if (pkg) return detectNodeRecipe(cwd, pkg, packageManager); + return ( + detectPythonRecipe(cwd) ?? + detectGoRecipe(cwd) ?? + detectRustRecipe(cwd) ?? + detectJavaRecipe(cwd) ?? + detectFallbackRecipe(cwd) + ); +} + +export function normalizeVerifyRecipe(value: unknown): VerifyRecipe | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const raw = value as Record; + const asStrings = (input: unknown): string[] => + Array.isArray(input) + ? input.filter((v): v is string => typeof v === "string" && v.trim() !== "").map((v) => v.trim()) + : []; + const ecosystem = typeof raw.ecosystem === "string" ? raw.ecosystem.trim() : ""; + const appKind = typeof raw.appKind === "string" ? raw.appKind.trim() : ""; + const appLabel = typeof raw.appLabel === "string" ? raw.appLabel.trim() : ""; + const smokeKind = + raw.smokeKind === "http" || raw.smokeKind === "cli" || raw.smokeKind === "none" ? raw.smokeKind : "none"; + if (!ecosystem || !appKind || !appLabel) return null; + return { + ecosystem, + appKind, + appLabel, + shellInitCommands: asStrings(raw.shellInitCommands), + bootstrapCommands: asStrings(raw.bootstrapCommands), + installCommands: asStrings(raw.installCommands), + buildCommands: asStrings(raw.buildCommands), + testCommands: asStrings(raw.testCommands), + startCommand: typeof raw.startCommand === "string" && raw.startCommand.trim() ? raw.startCommand.trim() : undefined, + startPort: typeof raw.startPort === "string" && raw.startPort.trim() ? raw.startPort.trim() : undefined, + smokeKind, + smokeTarget: typeof raw.smokeTarget === "string" && raw.smokeTarget.trim() ? raw.smokeTarget.trim() : undefined, + evidence: asStrings(raw.evidence), + notes: asStrings(raw.notes), + }; +} + +export function inferVerifySmokeUrl(settings?: SandboxSettings): string | null { + const ports = settings?.ports ?? []; + if (ports.length !== 1) return null; + const hostPort = parseHostPort(ports[0]); + return hostPort ? `http://127.0.0.1:${hostPort}` : null; +} + +export function inferVerifyProjectProfile( + cwd: string, + baseSettings: SandboxSettings = {}, + recipeOverride?: VerifyRecipe | null, +): VerifyProjectProfile { + const pkg = readPackageJson(cwd); + const packageManager = detectPackageManager(cwd); + const recipe = recipeOverride ?? inferFallbackRecipe(cwd, pkg, packageManager); + const inferredDefaults: SandboxSettings = + recipe.smokeKind === "http" && recipe.startPort ? { ports: [`${recipe.startPort}:${recipe.startPort}`] } : {}; + const sandboxSettings = mergeSandboxSettings(inferredDefaults, baseSettings); + const smokeUrl = inferVerifySmokeUrl(sandboxSettings); + + const recipeWithRuntime: VerifyRecipe = { + ...recipe, + smokeTarget: recipe.smokeKind === "http" ? (smokeUrl ?? recipe.smokeTarget) : undefined, + }; + + if (!fs.existsSync(path.join(cwd, "node_modules")) && recipeWithRuntime.ecosystem === "node") { + recipeWithRuntime.notes = dedupe([ + ...recipeWithRuntime.notes, + "Host dependencies are not installed in node_modules. Verification may be limited unless a Shuru checkpoint already contains the needed runtime dependencies.", + ]); + } + + return { + appKind: normalizeVerifyAppKind(recipeWithRuntime.appKind), + appLabel: recipeWithRuntime.appLabel, + packageManager, + availableScripts: Object.keys(pkg?.scripts ?? {}), + hasNodeModules: fs.existsSync(path.join(cwd, "node_modules")), + sandboxSettings, + recipe: recipeWithRuntime, + }; +} diff --git a/src/verify/retry.ts b/src/verify/retry.ts new file mode 100644 index 0000000..30aa321 --- /dev/null +++ b/src/verify/retry.ts @@ -0,0 +1,62 @@ +import type { VerifyRetryStrategy } from "../types/index"; +import type { VerifyProjectProfile } from "./recipes"; + +const NODE_WEB_APP_KINDS = new Set(["nextjs", "vite", "astro", "sveltekit", "remix", "cra"]); + +function buildNativeModuleRetry(profile: VerifyProjectProfile): VerifyRetryStrategy | null { + if (!NODE_WEB_APP_KINDS.has(profile.appKind)) { + return null; + } + + const installCommand = + profile.packageManager === "bun" + ? "npm install --include=optional" + : profile.packageManager === "pnpm" + ? "pnpm install" + : profile.packageManager === "yarn" + ? "yarn install" + : "npm install --include=optional"; + + return { + id: "native-module-retry", + when: "Only if install or build fails with missing native bindings such as lightningcss, sharp, @next/swc, esbuild, rollup, or a missing *.linux-*.node module.", + reason: "Recover from platform-specific optional/native dependency installs without unbounded retry loops.", + commands: ["rm -rf node_modules", installCommand, ...profile.recipe.buildCommands], + }; +} + +function buildStartupRetry(profile: VerifyProjectProfile): VerifyRetryStrategy | null { + if (profile.recipe.smokeKind !== "http" || !profile.recipe.startCommand || !profile.recipe.startPort) { + return null; + } + return { + id: "startup-readiness-retry", + when: "Only if the app starts but never becomes reachable at the expected localhost URL.", + reason: "Give one deterministic retry with explicit host/port binding before declaring the app non-runnable.", + commands: [`HOST=0.0.0.0 PORT=${profile.recipe.startPort} ${profile.recipe.startCommand}`], + }; +} + +export function getVerifyRetryStrategies(profile: VerifyProjectProfile): VerifyRetryStrategy[] { + return [buildNativeModuleRetry(profile), buildStartupRetry(profile)].filter( + (strategy): strategy is VerifyRetryStrategy => Boolean(strategy), + ); +} + +export function buildRetryGuidance(profile: VerifyProjectProfile): string[] { + const strategies = getVerifyRetryStrategies(profile); + if (strategies.length === 0) { + return [ + "- Do not thrash. If a failure is not covered by a known retry strategy, report it directly instead of guessing.", + ]; + } + + return [ + "- Retry policy: at most one bounded retry per known failure class. Do not loop or improvise multiple package-manager swaps.", + ...strategies.flatMap((strategy) => [ + `- Retry strategy (${strategy.id}): ${strategy.when}`, + `- Reason: ${strategy.reason}`, + `- Commands: ${strategy.commands.join(" && ")}`, + ]), + ]; +} diff --git a/src/verify/runtime-prep.test.ts b/src/verify/runtime-prep.test.ts new file mode 100644 index 0000000..6bebb81 --- /dev/null +++ b/src/verify/runtime-prep.test.ts @@ -0,0 +1,47 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("./checkpoint", () => ({ + ensureVerifyCheckpoint: vi.fn(async () => ({ + created: true, + checkpointName: "verify-nextjs-test", + guestWorkdir: "/grok/verify/worktree", + })), +})); + +import { prepareVerifySandbox } from "./entrypoint"; + +const tempDirs: string[] = []; + +function makeTempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("prepareVerifySandbox", () => { + it("adds checkpoint-backed sandbox settings for verify execution", async () => { + const dir = makeTempDir("grok-verify-runtime-prep-"); + fs.writeFileSync( + path.join(dir, "package.json"), + JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2), + ); + fs.writeFileSync(path.join(dir, "package-lock.json"), ""); + + const prepared = await prepareVerifySandbox(dir, { allowNet: false }); + expect(prepared.profile.appKind).toBe("nextjs"); + expect(prepared.sandboxSettings.allowNet).toBe(true); + expect(prepared.sandboxSettings.from).toBe("verify-nextjs-test"); + expect(prepared.sandboxSettings.guestWorkdir).toBe("/grok/verify/worktree"); + expect(prepared.sandboxSettings.syncHostWorkspace).toBe(true); + expect(prepared.checkpoint?.created).toBe(true); + }); +});