feat: update verify command (#240)
* Update CHANGELOG.md * fix veriy pipeline * bump version
This commit is contained in:
1 parent
79ebd68039
commit
19150aeb01
23 files changed
+2012
-664
No files matched your search
+25
-7
@@ -7,18 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- Built-in `telegram-bridge` command for headless Telegram bridge entrypoint with shared approval persistence (#211)
|
||||
- Add CLI update checker (#223)
|
||||
- Add verify workflow
|
||||
## [1.1.3] - 2026-04-01
|
||||
|
||||
### Fixed
|
||||
- Issue with schedule modal (#226)
|
||||
- Host-side browser smoke tests, skill inheritance, and screenshot paths
|
||||
### Added
|
||||
- @-mention file autocomplete (#236)
|
||||
|
||||
## [1.1.2] - 2026-04-01
|
||||
|
||||
### Added
|
||||
- Switch computer sub-agent to agent-desktop (#233)
|
||||
|
||||
### Removed
|
||||
- Tracked telegram pair code from repo (#234)
|
||||
|
||||
## [1.1.1] - 2026-04-01
|
||||
|
||||
### Added
|
||||
- Verify workflow with sandboxed testing and browser smoke checks (#228)
|
||||
- Batch mode for headless Grok CLI runs (#231)
|
||||
|
||||
## [1.1.0] - 2026-03-26
|
||||
|
||||
### Added
|
||||
- CLI update checker (#223)
|
||||
|
||||
### Changed
|
||||
- Replace commit scan with PR security scan (#224)
|
||||
|
||||
### Fixed
|
||||
- Issue with schedule modal (#226)
|
||||
|
||||
## [1.0.0-rc7] - 2026-03-26
|
||||
|
||||
### Added
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "grok-dev",
|
||||
"version": "1.1.3",
|
||||
"version": "1.1.4",
|
||||
"description": "An open-source AI coding agent powered by Grok, built with Bun and OpenTUI.",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
|
||||
+142
-39
@@ -55,7 +55,8 @@ import {
|
||||
type SandboxSettings,
|
||||
} from "../utils/settings";
|
||||
import { discoverSkills, formatSkillsForPrompt } from "../utils/skills";
|
||||
import { buildVerifyDetectPrompt, normalizeVerifyRecipe } from "../verify/entrypoint";
|
||||
import { buildVerifyDetectPrompt, normalizeVerifyRecipe, prepareVerifySandbox } from "../verify/entrypoint";
|
||||
import { runVerifyOrchestration } from "../verify/orchestrator";
|
||||
import {
|
||||
type CompactionSettings,
|
||||
createCompactionSummaryMessage,
|
||||
@@ -322,6 +323,7 @@ function buildSubagentPrompt(
|
||||
const isVision = request.agent === "vision";
|
||||
const isVerify = request.agent === "verify";
|
||||
const isVerifyDetect = request.agent === "verify-detect";
|
||||
const isVerifyManifest = request.agent === "verify-manifest";
|
||||
const isComputer = request.agent === "computer";
|
||||
const mode: AgentMode = isExplore || isVerifyDetect ? "ask" : "agent";
|
||||
const role = custom
|
||||
@@ -332,11 +334,13 @@ function buildSubagentPrompt(
|
||||
? "You are the Vision sub-agent."
|
||||
: isVerifyDetect
|
||||
? "You are the Verify Detect sub-agent. You inspect a repository to produce a structured verification recipe. You are read-only."
|
||||
: isVerify
|
||||
? "You are the Verify sub-agent. You specialize in sandbox-aware local verification using builds, tests, app boot checks, and optional browser smoke tests."
|
||||
: isComputer
|
||||
? "You are the Computer sub-agent. You specialize in host desktop automation using accessibility snapshots, semantic element refs, screenshots, and careful mouse and keyboard actions."
|
||||
: "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work.";
|
||||
: isVerifyManifest
|
||||
? "You are the Verify Manifest sub-agent. You inspect a repository and create or update .grok/environment.json so verification can run reproducibly."
|
||||
: isVerify
|
||||
? "You are the Verify sub-agent. You specialize in sandbox-aware local verification using builds, tests, app boot checks, and optional browser smoke tests."
|
||||
: isComputer
|
||||
? "You are the Computer sub-agent. You specialize in host desktop automation using accessibility snapshots, semantic element refs, screenshots, and careful mouse and keyboard actions."
|
||||
: "You are the General sub-agent. You can investigate, edit files, and run commands to complete delegated work.";
|
||||
|
||||
const rules = isExplore
|
||||
? [
|
||||
@@ -350,31 +354,76 @@ function buildSubagentPrompt(
|
||||
"Read config files, package manifests, scripts, and source layout to understand the project.",
|
||||
"Return ONLY a valid JSON object with the VerifyRecipe schema. No markdown, no prose, no explanation outside the JSON.",
|
||||
]
|
||||
: isVision
|
||||
? ["Validate the image."]
|
||||
: isComputer
|
||||
? [
|
||||
"Operate carefully on the HOST desktop, not inside the shell sandbox.",
|
||||
"Start with `computer_snapshot` when possible. It returns stable refs like @e1 that remain valid until the next snapshot.",
|
||||
"Prefer accessibility refs over coordinates. Use `computer_click`, `computer_type`, `computer_scroll`, and `computer_get` with refs from the latest snapshot.",
|
||||
"After any meaningful UI transition, launch, dialog open, or menu change, take another `computer_snapshot` before reusing old refs.",
|
||||
"Use `computer_launch`, `computer_list_windows`, `computer_focus_window`, and `computer_wait` to manage apps and window state.",
|
||||
"Use `computer_press` for shortcuts like Enter or cmd+k. Use `computer_screenshot` only for visual confirmation or when the accessibility tree is insufficient.",
|
||||
"If `agent-desktop` is unavailable, permissions are missing, refs go stale, or the state is ambiguous, stop and return the blocker clearly to the parent agent.",
|
||||
"Do not perform destructive or high-risk desktop actions unless the delegated task explicitly requires them.",
|
||||
]
|
||||
: isVerify
|
||||
: isVerifyManifest
|
||||
? [
|
||||
"Focus on creating or updating .grok/environment.json as the primary verification contract for this repository.",
|
||||
"Read package.json and key config files to understand the project, then write .grok/environment.json.",
|
||||
"Prefer editing only .grok/environment.json unless the delegated task explicitly requires something else.",
|
||||
"",
|
||||
"SANDBOX ENVIRONMENT (Shuru):",
|
||||
"- OS: Debian GNU/Linux 13 (trixie)",
|
||||
"- Architecture: aarch64 (ARM64)",
|
||||
"- Pre-installed: NOTHING. No node, npm, npx, bun, python3, pip, go, cargo, java, or any runtime.",
|
||||
"- Only basic system tools exist (sh, apt-get, curl, etc).",
|
||||
"- Network access is available during bootstrap and install.",
|
||||
"- The workspace is mounted at /workspace.",
|
||||
"",
|
||||
"MANIFEST REQUIREMENTS:",
|
||||
"- bootstrapCommands: MUST install every runtime and build tool the project needs from scratch via apt-get or curl.",
|
||||
"- For Node.js/Next.js/Vite/etc: `apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm`",
|
||||
"- For Bun projects: also `curl -fsSL https://bun.sh/install | bash` and shellInitCommands with BUN_INSTALL/PATH exports.",
|
||||
"- For Python: `apt-get update && apt-get install -y python3 python3-pip python3-venv ca-certificates git`",
|
||||
"- For Go: `apt-get update && apt-get install -y golang ca-certificates git`",
|
||||
"- For Rust: `apt-get update && apt-get install -y curl ca-certificates git build-essential && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y`",
|
||||
"- installCommands: The package install command (npm install, pip install, etc).",
|
||||
"- buildCommands: Build commands if applicable.",
|
||||
"- testCommands: Test/lint commands if applicable.",
|
||||
"- startCommand + startPort: How to start the app for smoke testing.",
|
||||
"- smokeKind: 'http' if the app has a web UI, 'cli' for CLI tools, 'none' otherwise.",
|
||||
"- Do NOT leave bootstrapCommands empty. The sandbox has nothing.",
|
||||
"",
|
||||
"Return a concise summary of what you wrote and why.",
|
||||
]
|
||||
: isVision
|
||||
? ["Validate the image."]
|
||||
: isComputer
|
||||
? [
|
||||
"Focus on verification first. Do not make durable source edits unless the delegated task explicitly asks for fixes.",
|
||||
"Prefer the smallest meaningful set of validation commands and explain any environment blockers clearly.",
|
||||
"IMPORTANT: When the recipe includes a smoke target URL and a forwarded port, you MUST attempt browser smoke testing using agent-browser via the bash tool. The agent-browser command runs on the HOST, not inside the sandbox. It will work even in sandbox mode. Do not skip it or assume it is unavailable. Just run the command.",
|
||||
"Return a concise structured verification report for the parent agent.",
|
||||
"Operate carefully on the HOST desktop, not inside the shell sandbox.",
|
||||
"Start with `computer_snapshot` when possible. It returns stable refs like @e1 that remain valid until the next snapshot.",
|
||||
"Prefer accessibility refs over coordinates. Use `computer_click`, `computer_type`, `computer_scroll`, and `computer_get` with refs from the latest snapshot.",
|
||||
"After any meaningful UI transition, launch, dialog open, or menu change, take another `computer_snapshot` before reusing old refs.",
|
||||
"Use `computer_launch`, `computer_list_windows`, `computer_focus_window`, and `computer_wait` to manage apps and window state.",
|
||||
"Use `computer_press` for shortcuts like Enter or cmd+k. Use `computer_screenshot` only for visual confirmation or when the accessibility tree is insufficient.",
|
||||
"If `agent-desktop` is unavailable, permissions are missing, refs go stale, or the state is ambiguous, stop and return the blocker clearly to the parent agent.",
|
||||
"Do not perform destructive or high-risk desktop actions unless the delegated task explicitly requires them.",
|
||||
]
|
||||
: [
|
||||
"Work only on the delegated task below.",
|
||||
"Use tools directly instead of narrating your intent.",
|
||||
"Return a concise summary for the parent agent with key outcomes and any open risks.",
|
||||
];
|
||||
: isVerify
|
||||
? [
|
||||
"You are a QA engineer. Your job is to prove the app works end-to-end, not just that it builds.",
|
||||
"Do not make durable source edits unless the delegated task explicitly asks for fixes.",
|
||||
"",
|
||||
"MANDATORY VERIFICATION STEPS (do ALL of these in order):",
|
||||
"1. Install dependencies (run installCommands from the recipe).",
|
||||
"2. Build the project (run buildCommands from the recipe).",
|
||||
"3. Run tests/lint if available (run testCommands from the recipe).",
|
||||
"4. Start the app (run startCommand from the recipe in the background).",
|
||||
"5. Wait for the app to be ready (curl readiness check or agent-browser wait).",
|
||||
"6. Run browser smoke tests like a real human QA tester:",
|
||||
" - Open the app in the browser, record a video, take screenshots.",
|
||||
" - Navigate the app: click links, buttons, menus. Verify pages load.",
|
||||
" - Check for JavaScript console errors.",
|
||||
" - Spend 3-5 interactions testing the critical path.",
|
||||
"7. Stop recording, close browser, then stop the dev server.",
|
||||
"",
|
||||
"Do NOT stop after build/lint. Starting the app and testing it in the browser is the most important part.",
|
||||
"agent-browser commands run on the HOST, not inside the sandbox. They WILL work. Do not skip them.",
|
||||
"Return a concise verification report. Keep it compact but always include Evidence with artifact file paths.",
|
||||
]
|
||||
: [
|
||||
"Work only on the delegated task below.",
|
||||
"Use tools directly instead of narrating your intent.",
|
||||
"Return a concise summary for the parent agent with key outcomes and any open risks.",
|
||||
];
|
||||
|
||||
const instructionLines = custom?.instruction.trim() ? ["", "SUB-AGENT INSTRUCTIONS:", custom.instruction.trim()] : [];
|
||||
|
||||
@@ -952,15 +1001,25 @@ export class Agent {
|
||||
const isVision = agentKey === "vision";
|
||||
const isVerify = agentKey === "verify";
|
||||
const isVerifyDetect = agentKey === "verify-detect";
|
||||
const isVerifyManifest = agentKey === "verify-manifest";
|
||||
const isComputer = agentKey === "computer";
|
||||
const subagents = loadValidSubAgents();
|
||||
const custom =
|
||||
!isExplore && !isGeneral && !isVision && !isVerify && !isVerifyDetect && !isComputer
|
||||
!isExplore && !isGeneral && !isVision && !isVerify && !isVerifyDetect && !isVerifyManifest && !isComputer
|
||||
? findCustomSubagent(agentKey, subagents)
|
||||
: undefined;
|
||||
|
||||
if (!isExplore && !isGeneral && !isVision && !isVerify && !isVerifyDetect && !isComputer && !custom) {
|
||||
const message = `Unknown sub-agent "${agentKey}". Use general, explore, vision, verify, computer, or a configured name from ~/.grok/user-settings.json.`;
|
||||
if (
|
||||
!isExplore &&
|
||||
!isGeneral &&
|
||||
!isVision &&
|
||||
!isVerify &&
|
||||
!isVerifyDetect &&
|
||||
!isVerifyManifest &&
|
||||
!isComputer &&
|
||||
!custom
|
||||
) {
|
||||
const message = `Unknown sub-agent "${agentKey}". Use general, explore, vision, verify, verify-detect, verify-manifest, computer, or a configured name from ~/.grok/user-settings.json.`;
|
||||
return {
|
||||
success: false,
|
||||
output: message,
|
||||
@@ -976,10 +1035,22 @@ export class Agent {
|
||||
const verifySandboxOverrides: SandboxSettings = isVerify
|
||||
? { allowNet: true, allowedHosts: undefined, allowEphemeralInstall: true, hostBrowserCommandsOnHost: true }
|
||||
: {};
|
||||
let verifyPreparedSettings: SandboxSettings | null = null;
|
||||
let verifyPreparedRecipe: VerifyRecipe | null = null;
|
||||
if (isVerify) {
|
||||
const prepared = await prepareVerifySandbox(
|
||||
this.bash.getCwd(),
|
||||
{ ...this.bash.getSandboxSettings(), ...verifySandboxOverrides },
|
||||
undefined,
|
||||
onActivity,
|
||||
);
|
||||
verifyPreparedSettings = prepared.sandboxSettings;
|
||||
verifyPreparedRecipe = prepared.profile.recipe;
|
||||
}
|
||||
const childBash = new BashTool(this.bash.getCwd(), {
|
||||
sandboxMode: isVerify ? "shuru" : this.bash.getSandboxMode(),
|
||||
sandboxSettings: isVerify
|
||||
? { ...this.bash.getSandboxSettings(), ...verifySandboxOverrides }
|
||||
? (verifyPreparedSettings ?? { ...this.bash.getSandboxSettings(), ...verifySandboxOverrides })
|
||||
: this.bash.getSandboxSettings(),
|
||||
});
|
||||
const childBaseTools = createTools(childBash, provider, childMode);
|
||||
@@ -987,11 +1058,13 @@ export class Agent {
|
||||
? "Scanning the codebase"
|
||||
: isVerifyDetect
|
||||
? "Detecting verification recipe"
|
||||
: isVerify
|
||||
? "Preparing verification pass"
|
||||
: isComputer
|
||||
? "Preparing computer control pass"
|
||||
: "Planning delegated work";
|
||||
: isVerifyManifest
|
||||
? "Creating verification manifest"
|
||||
: isVerify
|
||||
? "Preparing verification pass"
|
||||
: isComputer
|
||||
? "Preparing computer control pass"
|
||||
: "Planning delegated work";
|
||||
let assistantText = "";
|
||||
let lastActivity = initialDetail;
|
||||
let childTools: ToolSet = childBaseTools;
|
||||
@@ -1047,9 +1120,14 @@ export class Agent {
|
||||
}
|
||||
}
|
||||
|
||||
const childPrompt =
|
||||
isVerify && verifyPreparedRecipe
|
||||
? `${request.prompt}\n\nPrepared verify recipe JSON (use this as the primary execution recipe and keep .grok/environment.json aligned with it if present):\n${JSON.stringify(verifyPreparedRecipe, null, 2)}`
|
||||
: request.prompt;
|
||||
|
||||
const childMessages = isVision
|
||||
? await buildVisionUserMessages(request.prompt, childBash.getCwd(), signal)
|
||||
: [{ role: "user" as const, content: request.prompt }];
|
||||
: [{ role: "user" as const, content: childPrompt }];
|
||||
|
||||
if (this.batchApi) {
|
||||
return await this.runTaskRequestBatch({
|
||||
@@ -1781,6 +1859,31 @@ export class Agent {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async runVerify(onProgress?: (detail: string) => void, abortSignal?: AbortSignal): Promise<ToolResult> {
|
||||
this.abortController = new AbortController();
|
||||
const signal = abortSignal ?? this.abortController.signal;
|
||||
const userModelMessage: ModelMessage = { role: "user", content: "/verify" };
|
||||
this.messages.push(userModelMessage);
|
||||
this.messageSeqs.push(null);
|
||||
|
||||
try {
|
||||
await this.consumeBackgroundNotifications();
|
||||
const result = await runVerifyOrchestration(this, { onProgress, abortSignal: signal });
|
||||
const assistantText = result.output || result.error || "Verification completed.";
|
||||
this.appendCompletedTurn(userModelMessage, [{ role: "assistant", content: assistantText }]);
|
||||
return result;
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
const failureText = signal.aborted ? "Verification aborted." : `Verification failed: ${msg}`;
|
||||
this.appendCompletedTurn(userModelMessage, [{ role: "assistant", content: failureText }]);
|
||||
return { success: false, output: failureText };
|
||||
} finally {
|
||||
if (this.abortController?.signal === signal) {
|
||||
this.abortController = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface ExecutedBatchTool {
|
||||
|
||||
@@ -110,6 +110,66 @@ describe("schedule daemon tools", () => {
|
||||
expect(result).toEqual({ success: true, output: "verified" });
|
||||
});
|
||||
|
||||
it("routes verify-detect task requests through the task tool", async () => {
|
||||
const runTask = vi.fn(async () => ({ success: true, output: "recipe" }));
|
||||
const tools = createTools(new BashTool("/tmp"), {} as never, "agent", {
|
||||
runTask,
|
||||
subagents: [],
|
||||
}) as Record<string, { execute: (input: unknown, context?: unknown) => Promise<unknown>; description?: string }>;
|
||||
|
||||
const taskTool = tools.task;
|
||||
expect(taskTool.description).toContain("`verify-detect`");
|
||||
|
||||
const result = (await taskTool.execute(
|
||||
{
|
||||
agent: "verify-detect",
|
||||
description: "Detect verification recipe",
|
||||
prompt: "Inspect the repository and return a VerifyRecipe JSON object.",
|
||||
},
|
||||
{ abortSignal: undefined },
|
||||
)) as { success: boolean; output: string };
|
||||
|
||||
expect(runTask).toHaveBeenCalledWith(
|
||||
{
|
||||
agent: "verify-detect",
|
||||
description: "Detect verification recipe",
|
||||
prompt: "Inspect the repository and return a VerifyRecipe JSON object.",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
expect(result).toEqual({ success: true, output: "recipe" });
|
||||
});
|
||||
|
||||
it("routes verify-manifest task requests through the task tool", async () => {
|
||||
const runTask = vi.fn(async () => ({ success: true, output: "manifest written" }));
|
||||
const tools = createTools(new BashTool("/tmp"), {} as never, "agent", {
|
||||
runTask,
|
||||
subagents: [],
|
||||
}) as Record<string, { execute: (input: unknown, context?: unknown) => Promise<unknown>; description?: string }>;
|
||||
|
||||
const taskTool = tools.task;
|
||||
expect(taskTool.description).toContain("`verify-manifest`");
|
||||
|
||||
const result = (await taskTool.execute(
|
||||
{
|
||||
agent: "verify-manifest",
|
||||
description: "Create verify manifest",
|
||||
prompt: "Inspect the repository and write .grok/environment.json.",
|
||||
},
|
||||
{ abortSignal: undefined },
|
||||
)) as { success: boolean; output: string };
|
||||
|
||||
expect(runTask).toHaveBeenCalledWith(
|
||||
{
|
||||
agent: "verify-manifest",
|
||||
description: "Create verify manifest",
|
||||
prompt: "Inspect the repository and write .grok/environment.json.",
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
expect(result).toEqual({ success: true, output: "manifest written" });
|
||||
});
|
||||
|
||||
it("exposes computer tools and routes computer task requests", async () => {
|
||||
const runTask = vi.fn(async () => ({ success: true, output: "computer-ready" }));
|
||||
const tools = createTools(new BashTool("/tmp"), {} as never, "agent", {
|
||||
|
||||
+12
-6
@@ -259,24 +259,30 @@ export function createTools(
|
||||
|
||||
if (options.runTask) {
|
||||
const customNames = (options.subagents ?? loadValidSubAgents()).map((agent) => agent.name);
|
||||
const taskAgentEnum = ["general", "explore", "vision", "verify", "computer", ...customNames] as [
|
||||
string,
|
||||
...string[],
|
||||
];
|
||||
const taskAgentEnum = [
|
||||
"general",
|
||||
"explore",
|
||||
"vision",
|
||||
"verify",
|
||||
"verify-detect",
|
||||
"verify-manifest",
|
||||
"computer",
|
||||
...customNames,
|
||||
] as [string, ...string[]];
|
||||
const customHint =
|
||||
customNames.length > 0
|
||||
? ` You may also use these user-defined sub-agents by exact name: ${customNames.join(", ")}.`
|
||||
: "";
|
||||
|
||||
tools.task = tool({
|
||||
description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, code quality work, verification, and computer-use flows instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution, \`explore\` for fast read-only investigation, \`vision\` for image validation, \`verify\` for sandbox-aware build, test, and smoke validation, and \`computer\` for host desktop screenshot/input workflows.${customHint} Provide a short description plus a detailed prompt for the child agent.`,
|
||||
description: `Delegate a focused foreground task to a sub-agent. Prefer this proactively for review, research, investigation, code quality work, verification, and computer-use flows instead of waiting for the user to request a sub-agent. Use \`general\` for multi-step execution, \`explore\` for fast read-only investigation, \`vision\` for image validation, \`verify\` for sandbox-aware build, test, and smoke validation, \`verify-detect\` for read-only verification recipe detection, \`verify-manifest\` to create or update a verification manifest, and \`computer\` for host desktop screenshot/input workflows.${customHint} Provide a short description plus a detailed prompt for the child agent.`,
|
||||
inputSchema: z.object({
|
||||
agent: z
|
||||
.enum(taskAgentEnum)
|
||||
.default("general")
|
||||
.describe(
|
||||
customNames.length > 0
|
||||
? "Built-in general, explore, vision, verify, or computer, or a configured custom sub-agent name from user settings"
|
||||
? "Built-in general, explore, vision, verify, verify-detect, verify-manifest, or computer, or a configured custom sub-agent name from user settings"
|
||||
: "Which sub-agent to use",
|
||||
),
|
||||
description: z.string().describe("A short label for the delegated task, such as 'Deep code quality analysis'"),
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ import {
|
||||
saveUserSettings,
|
||||
} from "./utils/settings";
|
||||
import { runUpdate } from "./utils/update-checker";
|
||||
import { getVerifyCliError, VERIFY_PROMPT } from "./verify/entrypoint";
|
||||
import { buildVerifyPrompt, getVerifyCliError } from "./verify/entrypoint";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
@@ -309,7 +309,7 @@ program
|
||||
}
|
||||
|
||||
await runHeadless(
|
||||
VERIFY_PROMPT,
|
||||
buildVerifyPrompt(process.cwd()),
|
||||
requireApiKey(config.apiKey),
|
||||
config.baseURL,
|
||||
config.model,
|
||||
|
||||
+60
-1
@@ -29,7 +29,14 @@ export interface Plan {
|
||||
questions?: PlanQuestion[];
|
||||
}
|
||||
|
||||
export type BuiltinSubagentId = "general" | "explore" | "vision" | "verify" | "verify-detect" | "computer";
|
||||
export type BuiltinSubagentId =
|
||||
| "general"
|
||||
| "explore"
|
||||
| "vision"
|
||||
| "verify"
|
||||
| "verify-detect"
|
||||
| "verify-manifest"
|
||||
| "computer";
|
||||
|
||||
export interface TaskRequest {
|
||||
agent: BuiltinSubagentId | string;
|
||||
@@ -104,6 +111,58 @@ export interface VerifyRecipe {
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export interface VerifyEnvironmentManifest {
|
||||
ecosystem?: string;
|
||||
appKind?: string;
|
||||
appLabel?: string;
|
||||
shellInit?: string[] | string;
|
||||
shellInitCommands?: string[] | string;
|
||||
bootstrap?: string[] | string;
|
||||
bootstrapCommands?: string[] | string;
|
||||
install?: string[] | string;
|
||||
installCommands?: string[] | string;
|
||||
build?: string[] | string;
|
||||
buildCommands?: string[] | string;
|
||||
test?: string[] | string;
|
||||
testCommands?: string[] | string;
|
||||
start?: string;
|
||||
startCommand?: string;
|
||||
startPort?: string;
|
||||
smokeKind?: "http" | "cli" | "none";
|
||||
smokeTarget?: string;
|
||||
evidence?: string[] | string;
|
||||
notes?: string[] | string;
|
||||
sandbox?: {
|
||||
allowNet?: boolean;
|
||||
allowedHosts?: string[];
|
||||
ports?: string[];
|
||||
cpus?: number;
|
||||
memory?: number;
|
||||
diskSize?: number;
|
||||
secrets?: Array<{ name: string; fromEnv: string; hosts: string[] }>;
|
||||
from?: string;
|
||||
verifyBaseFrom?: string;
|
||||
guestWorkdir?: string;
|
||||
syncHostWorkspace?: boolean;
|
||||
shellInit?: string[];
|
||||
hostBrowserCommandsOnHost?: boolean;
|
||||
};
|
||||
recipe?: Partial<VerifyRecipe> & Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface VerifyRetryStrategy {
|
||||
id: string;
|
||||
when: string;
|
||||
reason: string;
|
||||
commands: string[];
|
||||
}
|
||||
|
||||
export interface VerifyArtifact {
|
||||
kind: "log" | "screenshot" | "video";
|
||||
path: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface ToolResult {
|
||||
success: boolean;
|
||||
output?: string;
|
||||
|
||||
+4
-3
@@ -56,7 +56,7 @@ import {
|
||||
import { discoverSkills, formatSkillsForChat } from "../utils/skills";
|
||||
import { formatSubagentName } from "../utils/subagent-display";
|
||||
import { checkForUpdate, runUpdate, type UpdateCheckResult } from "../utils/update-checker";
|
||||
import { VERIFY_PROMPT } from "../verify/entrypoint";
|
||||
import { buildVerifyPrompt } from "../verify/entrypoint";
|
||||
import {
|
||||
buildSubagentBrowseRows,
|
||||
SUBAGENT_EDITOR_FIELDS,
|
||||
@@ -2088,7 +2088,7 @@ export function App({ agent, startupConfig, initialMessage, onExit }: AppProps)
|
||||
return true;
|
||||
}
|
||||
if (c === "/verify") {
|
||||
processMessage(VERIFY_PROMPT);
|
||||
processMessage(buildVerifyPrompt(agent.getCwd()));
|
||||
return true;
|
||||
}
|
||||
if (c === "/commit-push") {
|
||||
@@ -2117,6 +2117,7 @@ export function App({ agent, startupConfig, initialMessage, onExit }: AppProps)
|
||||
return false;
|
||||
},
|
||||
[
|
||||
agent,
|
||||
handleExit,
|
||||
openAgentsModal,
|
||||
openMcpModal,
|
||||
@@ -2184,7 +2185,7 @@ export function App({ agent, startupConfig, initialMessage, onExit }: AppProps)
|
||||
processMessage(REVIEW_PROMPT);
|
||||
break;
|
||||
case "verify":
|
||||
processMessage(VERIFY_PROMPT);
|
||||
processMessage(buildVerifyPrompt(agent.getCwd()));
|
||||
break;
|
||||
case "commit-push":
|
||||
processMessage(COMMIT_PUSH_PROMPT);
|
||||
|
||||
+18
-1
@@ -89,7 +89,15 @@ export interface CustomSubagentConfig {
|
||||
instruction: string;
|
||||
}
|
||||
|
||||
const RESERVED_SUBAGENT_NAMES = new Set(["general", "explore", "vision", "verify", "verify-detect", "computer"]);
|
||||
const RESERVED_SUBAGENT_NAMES = new Set([
|
||||
"general",
|
||||
"explore",
|
||||
"vision",
|
||||
"verify",
|
||||
"verify-detect",
|
||||
"verify-manifest",
|
||||
"computer",
|
||||
]);
|
||||
|
||||
export function isReservedSubagentName(name: string): boolean {
|
||||
return RESERVED_SUBAGENT_NAMES.has(name.trim().toLowerCase());
|
||||
@@ -306,6 +314,15 @@ export function normalizeSandboxSettings(raw: unknown): SandboxSettings {
|
||||
if (secrets.length > 0) result.secrets = secrets;
|
||||
}
|
||||
if (typeof raw.from === "string" && raw.from.trim()) result.from = raw.from.trim();
|
||||
if (typeof raw.verifyBaseFrom === "string" && raw.verifyBaseFrom.trim())
|
||||
result.verifyBaseFrom = raw.verifyBaseFrom.trim();
|
||||
if (typeof raw.allowEphemeralInstall === "boolean") result.allowEphemeralInstall = raw.allowEphemeralInstall;
|
||||
if (typeof raw.syncHostWorkspace === "boolean") result.syncHostWorkspace = raw.syncHostWorkspace;
|
||||
if (typeof raw.guestWorkdir === "string" && raw.guestWorkdir.trim()) result.guestWorkdir = raw.guestWorkdir.trim();
|
||||
if (Array.isArray(raw.shellInit)) {
|
||||
const shellInit = raw.shellInit.filter((line): line is string => typeof line === "string" && line.trim() !== "");
|
||||
if (shellInit.length > 0) result.shellInit = shellInit;
|
||||
}
|
||||
if (typeof raw.hostBrowserCommandsOnHost === "boolean")
|
||||
result.hostBrowserCommandsOnHost = raw.hostBrowserCommandsOnHost;
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ describe("formatSubagentName", () => {
|
||||
expect(formatSubagentName("vision")).toBe("Vision");
|
||||
expect(formatSubagentName("verify")).toBe("Verify");
|
||||
expect(formatSubagentName("verify-detect")).toBe("Verify Detect");
|
||||
expect(formatSubagentName("verify-manifest")).toBe("Verify Manifest");
|
||||
expect(formatSubagentName("computer")).toBe("Computer");
|
||||
});
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ export function formatSubagentName(agent: string): string {
|
||||
if (agent === "vision") return "Vision";
|
||||
if (agent === "verify") return "Verify";
|
||||
if (agent === "verify-detect") return "Verify Detect";
|
||||
if (agent === "verify-manifest") return "Verify Manifest";
|
||||
if (agent === "computer") return "Computer";
|
||||
if (!agent) return "Sub-agent";
|
||||
return agent.charAt(0).toUpperCase() + agent.slice(1);
|
||||
|
||||
@@ -3,15 +3,37 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("child_process", () => ({
|
||||
execFile: vi.fn(),
|
||||
}));
|
||||
vi.mock("child_process", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("child_process")>();
|
||||
return {
|
||||
...actual,
|
||||
execFile: vi.fn(),
|
||||
spawn: vi.fn(() => {
|
||||
const { EventEmitter } = require("events");
|
||||
const { Readable } = require("stream");
|
||||
const child = new EventEmitter();
|
||||
child.stdout = new Readable({
|
||||
read() {
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
child.stderr = new Readable({
|
||||
read() {
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
setTimeout(() => child.emit("close", 0), 0);
|
||||
return child;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
import { execFile } from "child_process";
|
||||
import { execFile, spawn } from "child_process";
|
||||
import { ensureVerifyCheckpoint, getVerifyCheckpointName } from "./checkpoint";
|
||||
import { inferVerifyProjectProfile } from "./entrypoint";
|
||||
|
||||
const execFileMock = vi.mocked(execFile);
|
||||
const spawnMock = vi.mocked(spawn);
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(prefix: string): string {
|
||||
@@ -22,6 +44,24 @@ function makeTempDir(prefix: string): string {
|
||||
|
||||
afterEach(() => {
|
||||
execFileMock.mockReset();
|
||||
spawnMock.mockReset();
|
||||
spawnMock.mockImplementation(() => {
|
||||
const { EventEmitter } = require("events");
|
||||
const { Readable } = require("stream");
|
||||
const child = new EventEmitter();
|
||||
child.stdout = new Readable({
|
||||
read() {
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
child.stderr = new Readable({
|
||||
read() {
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
setTimeout(() => child.emit("close", 0), 0);
|
||||
return child;
|
||||
});
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -46,13 +86,13 @@ describe("verify checkpoints", () => {
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, "package-lock.json"), "");
|
||||
|
||||
execFileMock.mockImplementation((command, args, _options, callback) => {
|
||||
execFileMock.mockImplementation((_command, args, _options, callback) => {
|
||||
const cb = callback as (error: Error | null, stdout: string, stderr: string) => void;
|
||||
if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "list") {
|
||||
cb(null, "", "");
|
||||
return {} as never;
|
||||
}
|
||||
cb(null, "created", "");
|
||||
cb(null, "", "");
|
||||
return {} as never;
|
||||
});
|
||||
|
||||
@@ -62,10 +102,11 @@ describe("verify checkpoints", () => {
|
||||
expect(result.created).toBe(true);
|
||||
expect(result.checkpointName).toMatch(/^verify-nextjs-/);
|
||||
expect(result.guestWorkdir).toBe("/grok/verify/worktree");
|
||||
expect(execFileMock).toHaveBeenCalledTimes(2);
|
||||
const createCall = execFileMock.mock.calls[1];
|
||||
expect(createCall[0]).toBe("shuru");
|
||||
const createArgs = createCall[1] as string[];
|
||||
expect(execFileMock).toHaveBeenCalledTimes(1);
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1);
|
||||
const spawnArgs = spawnMock.mock.calls[0];
|
||||
expect(spawnArgs[0]).toBe("shuru");
|
||||
const createArgs = spawnArgs[1] as string[];
|
||||
expect(createArgs.slice(0, 3)).toEqual(["checkpoint", "create", result.checkpointName!]);
|
||||
expect(createArgs.join(" ")).toContain("export DEBIAN_FRONTEND=noninteractive");
|
||||
});
|
||||
@@ -80,7 +121,7 @@ describe("verify checkpoints", () => {
|
||||
const profile = inferVerifyProjectProfile(dir);
|
||||
const checkpointName = getVerifyCheckpointName(dir, profile.recipe);
|
||||
|
||||
execFileMock.mockImplementation((command, args, _options, callback) => {
|
||||
execFileMock.mockImplementation((_command, args, _options, callback) => {
|
||||
const cb = callback as (error: Error | null, stdout: string, stderr: string) => void;
|
||||
if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "list") {
|
||||
cb(null, `${checkpointName}\n`, "");
|
||||
@@ -103,16 +144,12 @@ describe("verify checkpoints", () => {
|
||||
const profile = inferVerifyProjectProfile(dir);
|
||||
const checkpointName = getVerifyCheckpointName(dir, profile.recipe);
|
||||
|
||||
execFileMock.mockImplementation((command, args, _options, callback) => {
|
||||
execFileMock.mockImplementation((_command, args, _options, callback) => {
|
||||
const cb = callback as (error: Error | null, stdout: string, stderr: string) => void;
|
||||
if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "list") {
|
||||
cb(null, "", "");
|
||||
return {} as never;
|
||||
}
|
||||
if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "create") {
|
||||
cb(new Error("bun: not found"), "", "bun: not found");
|
||||
return {} as never;
|
||||
}
|
||||
if (Array.isArray(args) && args[0] === "checkpoint" && args[1] === "delete") {
|
||||
cb(null, "", "");
|
||||
return {} as never;
|
||||
@@ -121,10 +158,29 @@ describe("verify checkpoints", () => {
|
||||
return {} as never;
|
||||
});
|
||||
|
||||
spawnMock.mockImplementation(() => {
|
||||
const { EventEmitter } = require("events");
|
||||
const { Readable } = require("stream");
|
||||
const child = new EventEmitter();
|
||||
child.stdout = new Readable({
|
||||
read() {
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
child.stderr = new Readable({
|
||||
read() {
|
||||
this.push("bun: not found\n");
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
setTimeout(() => child.emit("close", 1), 0);
|
||||
return child;
|
||||
});
|
||||
|
||||
await expect(ensureVerifyCheckpoint(dir, profile, profile.sandboxSettings)).rejects.toThrow(
|
||||
`Verify checkpoint bootstrap failed for "${checkpointName}"`,
|
||||
);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(3);
|
||||
expect(execFileMock.mock.calls[2]?.[1]).toEqual(["checkpoint", "delete", checkpointName]);
|
||||
expect(execFileMock).toHaveBeenCalledTimes(2);
|
||||
expect(execFileMock.mock.calls[1]?.[1]).toEqual(["checkpoint", "delete", checkpointName]);
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,10 @@
|
||||
import { execFile } from "child_process";
|
||||
import { execFile, spawn } from "child_process";
|
||||
import { createHash } from "crypto";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import type { VerifyRecipe } from "../types/index";
|
||||
import type { SandboxSettings } from "../utils/settings";
|
||||
import type { VerifyProjectProfile } from "./entrypoint";
|
||||
import type { VerifyProjectProfile } from "./recipes";
|
||||
|
||||
const DEFAULT_VERIFY_GUEST_WORKDIR = "/grok/verify/worktree";
|
||||
|
||||
@@ -25,6 +25,51 @@ function execFileAsync(command: string, args: string[], cwd: string): Promise<{
|
||||
});
|
||||
}
|
||||
|
||||
function spawnWithProgress(
|
||||
command: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
onLine?: (line: string) => void,
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
env: { ...process.env, FORCE_COLOR: "0" },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (chunk: Buffer) => {
|
||||
const text = chunk.toString();
|
||||
stdout += text;
|
||||
if (onLine) {
|
||||
for (const line of text.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed) onLine(trimmed);
|
||||
}
|
||||
}
|
||||
});
|
||||
child.stderr.on("data", (chunk: Buffer) => {
|
||||
const text = chunk.toString();
|
||||
stderr += text;
|
||||
if (onLine) {
|
||||
for (const line of text.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed) onLine(trimmed);
|
||||
}
|
||||
}
|
||||
});
|
||||
child.on("error", (err) => reject(err));
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0) {
|
||||
reject(new Error((stderr || stdout || `Process exited with code ${code}`).trim()));
|
||||
} else {
|
||||
resolve({ stdout, stderr });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function stableRecipePayload(recipe: VerifyRecipe): string {
|
||||
return JSON.stringify({
|
||||
ecosystem: recipe.ecosystem,
|
||||
@@ -168,6 +213,7 @@ export async function ensureVerifyCheckpoint(
|
||||
cwd: string,
|
||||
profile: VerifyProjectProfile,
|
||||
settings: SandboxSettings,
|
||||
onProgress?: (detail: string) => void,
|
||||
): Promise<PreparedVerifyCheckpoint> {
|
||||
if (profile.recipe.installCommands.length === 0) {
|
||||
return { created: false };
|
||||
@@ -177,8 +223,9 @@ export async function ensureVerifyCheckpoint(
|
||||
const checkpoints: string[] = await listCheckpoints(cwd).catch((): string[] => []);
|
||||
if (!checkpoints.includes(checkpointName)) {
|
||||
const args = buildCheckpointCreateArgs(cwd, checkpointName, DEFAULT_VERIFY_GUEST_WORKDIR, profile.recipe, settings);
|
||||
onProgress?.(`Creating checkpoint: ${checkpointName}`);
|
||||
try {
|
||||
await execFileAsync("shuru", args, cwd);
|
||||
await spawnWithProgress("shuru", args, cwd, onProgress);
|
||||
} catch (error) {
|
||||
await deleteCheckpoint(cwd, checkpointName).catch(() => {});
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
@@ -187,5 +234,6 @@ export async function ensureVerifyCheckpoint(
|
||||
return { checkpointName, guestWorkdir: DEFAULT_VERIFY_GUEST_WORKDIR, created: true };
|
||||
}
|
||||
|
||||
onProgress?.(`Reusing checkpoint: ${checkpointName}`);
|
||||
return { checkpointName, guestWorkdir: DEFAULT_VERIFY_GUEST_WORKDIR, created: false };
|
||||
}
|
||||
@@ -5,12 +5,12 @@ import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { VerifyRecipe } from "../types/index";
|
||||
import {
|
||||
buildVerifyDetectPrompt,
|
||||
buildVerifyPrompt,
|
||||
buildVerifyTaskPrompt,
|
||||
createVerifyRuntimeConfig,
|
||||
getVerifyCliError,
|
||||
inferVerifyProjectProfile,
|
||||
inferVerifySmokeUrl,
|
||||
VERIFY_PROMPT,
|
||||
} from "./entrypoint";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
@@ -56,12 +56,15 @@ describe("verify entrypoint helpers", () => {
|
||||
expect(profile.packageManager).toBe("npm");
|
||||
expect(profile.sandboxSettings.ports).toEqual(["3000:3000"]);
|
||||
expect(profile.availableScripts).toEqual(["dev", "build", "lint"]);
|
||||
expect(profile.recipe.bootstrapCommands).toEqual([]);
|
||||
expect(profile.recipe.bootstrapCommands).toEqual([
|
||||
"apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm",
|
||||
]);
|
||||
expect(profile.recipe.shellInitCommands).toEqual(["export DEBIAN_FRONTEND=noninteractive"]);
|
||||
expect(profile.recipe.startCommand).toBe("npm run dev");
|
||||
expect(profile.recipe.testCommands).toContain("npm run lint");
|
||||
});
|
||||
|
||||
it("detects bun as package manager without hardcoded bootstrap", () => {
|
||||
it("detects bun as package manager and bootstraps bun plus node tooling for web apps", () => {
|
||||
const dir = makeTempDir("grok-verify-bun-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "package.json"),
|
||||
@@ -71,7 +74,17 @@ describe("verify entrypoint helpers", () => {
|
||||
|
||||
const profile = inferVerifyProjectProfile(dir);
|
||||
expect(profile.packageManager).toBe("bun");
|
||||
expect(profile.recipe.bootstrapCommands).toEqual([]);
|
||||
expect(profile.recipe.bootstrapCommands).toEqual([
|
||||
"apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm",
|
||||
"curl -fsSL https://bun.sh/install | bash",
|
||||
]);
|
||||
expect(profile.recipe.shellInitCommands).toEqual([
|
||||
"export DEBIAN_FRONTEND=noninteractive",
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable in test assertion
|
||||
'export BUN_INSTALL="${HOME}/.bun"',
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable in test assertion
|
||||
'export PATH="${BUN_INSTALL}/bin:$PATH"',
|
||||
]);
|
||||
expect(profile.recipe.installCommands).toContain("bun install");
|
||||
});
|
||||
|
||||
@@ -188,16 +201,20 @@ describe("verify entrypoint helpers", () => {
|
||||
expect(prompt).toContain("Bootstrap commands:");
|
||||
expect(prompt).toContain("Install commands:");
|
||||
expect(prompt).toContain("MUST run browser smoke tests");
|
||||
expect(prompt).toContain("--screenshot-dir .grok/verify-artifacts");
|
||||
expect(prompt).toContain("record start .grok/verify-artifacts/verify-smoke.webm");
|
||||
expect(prompt).toContain("record start");
|
||||
expect(prompt).toContain("record stop");
|
||||
expect(prompt).toContain("Browser Checks");
|
||||
expect(prompt).toContain("Results");
|
||||
expect(prompt).toContain("Evidence");
|
||||
expect(prompt).toContain("Recipe");
|
||||
expect(prompt).toContain("ephemeral dependency installs are allowed inside the sandbox");
|
||||
expect(prompt).toContain("prefer chaining install + build/test/start");
|
||||
expect(prompt).toContain("Blockers");
|
||||
expect(prompt).toContain("Ephemeral installs are allowed");
|
||||
expect(prompt).toContain("Probe the sandbox for runtimes");
|
||||
expect(prompt).toContain("Only install what is missing");
|
||||
expect(prompt).toContain("MANDATORY workflow");
|
||||
expect(prompt).toContain("Phase 4");
|
||||
expect(prompt).toContain("QA tester");
|
||||
expect(prompt).toContain(".grok/verify-artifacts");
|
||||
expect(prompt).toContain("markdown links");
|
||||
expect(prompt).toContain("Keep the report compact");
|
||||
expect(prompt).toContain("Evidence is mandatory even on failure");
|
||||
});
|
||||
|
||||
it("builds a detector prompt that requests JSON output", () => {
|
||||
@@ -207,6 +224,7 @@ describe("verify entrypoint helpers", () => {
|
||||
expect(prompt).toContain("Return ONLY valid JSON");
|
||||
expect(prompt).toContain('"bootstrapCommands": string[]');
|
||||
expect(prompt).toContain("Fallback hints from static detection");
|
||||
expect(prompt).toContain("probes for runtimes/tools first");
|
||||
});
|
||||
|
||||
it("explains ambiguous browser setup when multiple ports exist", () => {
|
||||
@@ -217,11 +235,24 @@ describe("verify entrypoint helpers", () => {
|
||||
expect(prompt).toContain("Skip browser checks unless the user clearly identifies");
|
||||
});
|
||||
|
||||
it("VERIFY_PROMPT drives both sub-agents", () => {
|
||||
expect(VERIFY_PROMPT).toContain("verify-detect");
|
||||
expect(VERIFY_PROMPT).toContain("Step 1");
|
||||
expect(VERIFY_PROMPT).toContain("Step 2");
|
||||
expect(VERIFY_PROMPT).toContain("task");
|
||||
it("buildVerifyPrompt drives both sub-agents with a draft recipe", () => {
|
||||
const dir = makeTempDir("grok-verify-prompt-dynamic-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "package.json"),
|
||||
JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2),
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, "package-lock.json"), "");
|
||||
|
||||
const prompt = buildVerifyPrompt(dir);
|
||||
expect(prompt).toContain("verify-manifest");
|
||||
expect(prompt).toContain("Step 1");
|
||||
expect(prompt).toContain("Step 2");
|
||||
expect(prompt).toContain("task");
|
||||
expect(prompt).toContain(".grok/environment.json");
|
||||
expect(prompt).toContain('"ecosystem"');
|
||||
expect(prompt).toContain('"bootstrapCommands"');
|
||||
expect(prompt).toContain("apt-get");
|
||||
expect(prompt).toContain("nodejs npm");
|
||||
});
|
||||
|
||||
it("uses an override recipe when creating the runtime config", () => {
|
||||
|
||||
+263
-567
@@ -1,68 +1,22 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import type { TaskRequest, VerifyRecipe } from "../types/index";
|
||||
import { mergeSandboxSettings, type SandboxSettings } from "../utils/settings";
|
||||
import type { SandboxSettings } from "../utils/settings";
|
||||
import { ensureVerifyCheckpoint, type PreparedVerifyCheckpoint } from "./checkpoint";
|
||||
import { loadVerifyEnvironment } from "./environment";
|
||||
import { buildBrowserGuidance, buildEvidenceGuidance, buildReadinessGuidance } from "./evidence";
|
||||
import {
|
||||
defaultShellInit,
|
||||
detectPackageManager,
|
||||
getNodeWebBootstrapCommands,
|
||||
getNodeWebShellInitCommands,
|
||||
inferVerifyProjectProfile,
|
||||
normalizeVerifyAppKind,
|
||||
type VerifyProjectProfile,
|
||||
} from "./recipes";
|
||||
import { buildRetryGuidance } from "./retry";
|
||||
|
||||
export const VERIFY_SUBAGENT_ID = "verify";
|
||||
export const VERIFY_TASK_DESCRIPTION = "Run local verification";
|
||||
|
||||
type VerifyAppKind =
|
||||
| "nextjs"
|
||||
| "vite"
|
||||
| "astro"
|
||||
| "sveltekit"
|
||||
| "remix"
|
||||
| "cra"
|
||||
| "node"
|
||||
| "django"
|
||||
| "python"
|
||||
| "go"
|
||||
| "rust"
|
||||
| "maven"
|
||||
| "gradle"
|
||||
| "make"
|
||||
| "unknown";
|
||||
|
||||
function normalizeVerifyAppKind(value: string): VerifyAppKind {
|
||||
return (
|
||||
[
|
||||
"nextjs",
|
||||
"vite",
|
||||
"astro",
|
||||
"sveltekit",
|
||||
"remix",
|
||||
"cra",
|
||||
"node",
|
||||
"django",
|
||||
"python",
|
||||
"go",
|
||||
"rust",
|
||||
"maven",
|
||||
"gradle",
|
||||
"make",
|
||||
"unknown",
|
||||
] as const
|
||||
).includes(value as VerifyAppKind)
|
||||
? (value as VerifyAppKind)
|
||||
: "unknown";
|
||||
}
|
||||
|
||||
interface PackageJsonLike {
|
||||
dependencies?: Record<string, string>;
|
||||
devDependencies?: Record<string, string>;
|
||||
scripts?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface VerifyProjectProfile {
|
||||
appKind: VerifyAppKind;
|
||||
appLabel: string;
|
||||
packageManager: string | null;
|
||||
availableScripts: string[];
|
||||
hasNodeModules: boolean;
|
||||
sandboxSettings: SandboxSettings;
|
||||
recipe: VerifyRecipe;
|
||||
}
|
||||
|
||||
export interface VerifyRuntimeConfig {
|
||||
sandboxMode: "shuru";
|
||||
sandboxSettings: SandboxSettings;
|
||||
@@ -71,468 +25,16 @@ export interface VerifyRuntimeConfig {
|
||||
checkpointCreated?: boolean;
|
||||
}
|
||||
|
||||
function fileExists(cwd: string, file: string): boolean {
|
||||
return fs.existsSync(path.join(cwd, file));
|
||||
}
|
||||
|
||||
function readTextFile(cwd: string, file: string): string | null {
|
||||
try {
|
||||
return fs.readFileSync(path.join(cwd, file), "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseHostPort(mapping: string): string | null {
|
||||
const match = mapping.trim().match(/^(\d+):(\d+)$/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function readPackageJson(cwd: string): PackageJsonLike | null {
|
||||
const raw = readTextFile(cwd, "package.json");
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw) as PackageJsonLike;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function detectPackageManager(cwd: string): string | null {
|
||||
const candidates: Array<[string, string]> = [
|
||||
["pnpm-lock.yaml", "pnpm"],
|
||||
["bun.lock", "bun"],
|
||||
["bun.lockb", "bun"],
|
||||
["yarn.lock", "yarn"],
|
||||
["package-lock.json", "npm"],
|
||||
["uv.lock", "uv"],
|
||||
["poetry.lock", "poetry"],
|
||||
["Pipfile.lock", "pipenv"],
|
||||
];
|
||||
|
||||
for (const [file, manager] of candidates) {
|
||||
if (fileExists(cwd, file)) {
|
||||
return manager;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
export interface PreparedVerifySandbox {
|
||||
profile: VerifyProjectProfile;
|
||||
sandboxSettings: SandboxSettings;
|
||||
checkpoint?: PreparedVerifyCheckpoint;
|
||||
}
|
||||
|
||||
function dedupe(values: Array<string | undefined | null>): string[] {
|
||||
return [...new Set(values.map((v) => v?.trim()).filter((v): v is string => Boolean(v)))];
|
||||
}
|
||||
|
||||
function defaultShellInit(): string[] {
|
||||
return ["export DEBIAN_FRONTEND=noninteractive"];
|
||||
}
|
||||
|
||||
function inferPortFromCommand(command: string | undefined): string | undefined {
|
||||
if (!command) return undefined;
|
||||
const flagMatch = command.match(/(?:--port|-p)\s+(\d{2,5})/);
|
||||
if (flagMatch) return flagMatch[1];
|
||||
const envMatch = command.match(/\bPORT=(\d{2,5})\b/);
|
||||
if (envMatch) return envMatch[1];
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseTargetNames(raw: string): string[] {
|
||||
return raw
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.match(/^([A-Za-z0-9_.-]+):(?:\s|$)/)?.[1])
|
||||
.filter((target): target is string => Boolean(target));
|
||||
}
|
||||
|
||||
function detectMakeRecipe(cwd: string): VerifyRecipe | null {
|
||||
const makefile = readTextFile(cwd, "Makefile");
|
||||
if (!makefile) return null;
|
||||
const targets = parseTargetNames(makefile);
|
||||
const has = (names: string[]) => names.find((name) => targets.includes(name));
|
||||
const install = has(["install", "setup", "bootstrap"]);
|
||||
const build = has(["build", "compile"]);
|
||||
const test = has(["test", "check"]);
|
||||
const run = has(["run", "start", "serve", "dev"]);
|
||||
|
||||
return {
|
||||
ecosystem: "make",
|
||||
appKind: "make",
|
||||
appLabel: "Makefile-driven project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: install ? [`make ${install}`] : [],
|
||||
buildCommands: build ? [`make ${build}`] : [],
|
||||
testCommands: test ? [`make ${test}`] : [],
|
||||
startCommand: run ? `make ${run}` : undefined,
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Makefile", `Targets: ${targets.join(", ") || "(none)"}`],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectNodeRecipe(cwd: string, pkg: PackageJsonLike, packageManager: string | null): VerifyRecipe {
|
||||
const scripts = pkg.scripts ?? {};
|
||||
const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
|
||||
let appKind: VerifyAppKind = "node";
|
||||
let appLabel = "Node.js app";
|
||||
let defaultPort: string | undefined;
|
||||
|
||||
if (deps.next) {
|
||||
appKind = "nextjs";
|
||||
appLabel = "Next.js";
|
||||
defaultPort = "3000";
|
||||
} else if (deps["@sveltejs/kit"]) {
|
||||
appKind = "sveltekit";
|
||||
appLabel = "SvelteKit";
|
||||
defaultPort = "5173";
|
||||
} else if (deps.astro) {
|
||||
appKind = "astro";
|
||||
appLabel = "Astro";
|
||||
defaultPort = "4321";
|
||||
} else if (deps["@remix-run/dev"] || deps["@remix-run/react"]) {
|
||||
appKind = "remix";
|
||||
appLabel = "Remix";
|
||||
defaultPort = "3000";
|
||||
} else if (deps["react-scripts"]) {
|
||||
appKind = "cra";
|
||||
appLabel = "Create React App";
|
||||
defaultPort = "3000";
|
||||
} else if (deps.vite) {
|
||||
appKind = "vite";
|
||||
appLabel = "Vite";
|
||||
defaultPort = "5173";
|
||||
}
|
||||
|
||||
const install = packageManager
|
||||
? packageManager === "pnpm"
|
||||
? "pnpm install"
|
||||
: packageManager === "bun"
|
||||
? "bun install"
|
||||
: packageManager === "yarn"
|
||||
? "yarn install"
|
||||
: "npm install"
|
||||
: undefined;
|
||||
const startCommand = scripts.dev ?? scripts.start;
|
||||
const startPort = inferPortFromCommand(startCommand) ?? defaultPort;
|
||||
const smokeKind: VerifyRecipe["smokeKind"] = startCommand && startPort ? "http" : "none";
|
||||
|
||||
return {
|
||||
ecosystem: "node",
|
||||
appKind,
|
||||
appLabel,
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: dedupe([install]),
|
||||
buildCommands: dedupe(
|
||||
[scripts.build, scripts.typecheck].map((script) => script && pickPackageScript(packageManager, scripts, script)),
|
||||
),
|
||||
testCommands: dedupe(
|
||||
["test", "check", "lint"]
|
||||
.filter((name) => scripts[name])
|
||||
.map((name) => pickPackageScript(packageManager, scripts, scripts[name]!)),
|
||||
),
|
||||
startCommand: startCommand ? pickPackageScript(packageManager, scripts, startCommand) : undefined,
|
||||
startPort,
|
||||
smokeKind,
|
||||
evidence: [`Detected package.json`, `Scripts: ${Object.keys(scripts).join(", ") || "(none)"}`],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function pickPackageScript(packageManager: string | null, scripts: Record<string, string>, body: string): string {
|
||||
const entry = Object.entries(scripts).find(([, scriptBody]) => scriptBody === body)?.[0];
|
||||
if (!entry) return body;
|
||||
const runner =
|
||||
packageManager === "pnpm"
|
||||
? "pnpm"
|
||||
: packageManager === "bun"
|
||||
? "bun"
|
||||
: packageManager === "yarn"
|
||||
? "yarn"
|
||||
: "npm run";
|
||||
return runner === "yarn" ? `yarn ${entry}` : runner === "bun" ? `bun run ${entry}` : `${runner} ${entry}`;
|
||||
}
|
||||
|
||||
function detectPythonRecipe(cwd: string): VerifyRecipe | null {
|
||||
const pyproject = readTextFile(cwd, "pyproject.toml");
|
||||
const requirements = readTextFile(cwd, "requirements.txt");
|
||||
const managePy = fileExists(cwd, "manage.py");
|
||||
if (!pyproject && !requirements && !managePy && !fileExists(cwd, "setup.py")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const lower = `${pyproject ?? ""}\n${requirements ?? ""}`.toLowerCase();
|
||||
const packageManager = detectPackageManager(cwd);
|
||||
const isDjango = managePy || lower.includes("django");
|
||||
const isFastApi = lower.includes("fastapi") || lower.includes("uvicorn");
|
||||
|
||||
let install = "pip install -r requirements.txt";
|
||||
if (packageManager === "uv") install = "uv sync";
|
||||
else if (packageManager === "poetry") install = "poetry install";
|
||||
else if (packageManager === "pipenv") install = "pipenv install";
|
||||
else if (pyproject && !requirements) install = "pip install -e .";
|
||||
|
||||
if (isDjango) {
|
||||
return {
|
||||
ecosystem: "python",
|
||||
appKind: "django",
|
||||
appLabel: "Django app",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [install],
|
||||
buildCommands: [],
|
||||
testCommands: ["python manage.py test"],
|
||||
startCommand: "python manage.py runserver 0.0.0.0:8000",
|
||||
startPort: "8000",
|
||||
smokeKind: "http",
|
||||
evidence: ["Detected manage.py", pyproject ? "Detected pyproject.toml" : undefined].filter(Boolean) as string[],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (isFastApi) {
|
||||
const appModule = fileExists(cwd, "main.py") ? "main:app" : fileExists(cwd, "app.py") ? "app:app" : "main:app";
|
||||
return {
|
||||
ecosystem: "python",
|
||||
appKind: "python",
|
||||
appLabel: "Python web app",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [install],
|
||||
buildCommands: [],
|
||||
testCommands: fileExists(cwd, "tests") ? ["pytest"] : [],
|
||||
startCommand: `uvicorn ${appModule} --host 0.0.0.0 --port 8000`,
|
||||
startPort: "8000",
|
||||
smokeKind: "http",
|
||||
evidence: ["Detected Python project", "Detected FastAPI/Uvicorn dependency"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ecosystem: "python",
|
||||
appKind: "python",
|
||||
appLabel: "Python project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [install],
|
||||
buildCommands: [],
|
||||
testCommands: fileExists(cwd, "tests") ? ["pytest"] : ["python -m unittest discover"],
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Python project"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectGoRecipe(cwd: string): VerifyRecipe | null {
|
||||
if (!fileExists(cwd, "go.mod")) return null;
|
||||
return {
|
||||
ecosystem: "go",
|
||||
appKind: "go",
|
||||
appLabel: "Go project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: ["go build ./..."],
|
||||
testCommands: ["go test ./..."],
|
||||
startCommand: fileExists(cwd, "main.go") ? "go run ." : undefined,
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected go.mod"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectRustRecipe(cwd: string): VerifyRecipe | null {
|
||||
if (!fileExists(cwd, "Cargo.toml")) return null;
|
||||
return {
|
||||
ecosystem: "rust",
|
||||
appKind: "rust",
|
||||
appLabel: "Rust project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: ["cargo build"],
|
||||
testCommands: ["cargo test"],
|
||||
startCommand: fileExists(cwd, path.join("src", "main.rs")) ? "cargo run" : undefined,
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Cargo.toml"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectJavaRecipe(cwd: string): VerifyRecipe | null {
|
||||
if (fileExists(cwd, "pom.xml")) {
|
||||
return {
|
||||
ecosystem: "java",
|
||||
appKind: "maven",
|
||||
appLabel: "Maven project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: ["mvn package"],
|
||||
testCommands: ["mvn test"],
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected pom.xml"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (fileExists(cwd, "build.gradle") || fileExists(cwd, "build.gradle.kts")) {
|
||||
const gradle = fileExists(cwd, "gradlew") ? "./gradlew" : "gradle";
|
||||
return {
|
||||
ecosystem: "java",
|
||||
appKind: "gradle",
|
||||
appLabel: "Gradle project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: [`${gradle} build`],
|
||||
testCommands: [`${gradle} test`],
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Gradle build file"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectFallbackRecipe(cwd: string): VerifyRecipe {
|
||||
const makeRecipe = detectMakeRecipe(cwd);
|
||||
if (makeRecipe) return makeRecipe;
|
||||
return {
|
||||
ecosystem: "unknown",
|
||||
appKind: "unknown",
|
||||
appLabel: "Unknown project type",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: [],
|
||||
testCommands: [],
|
||||
smokeKind: "none",
|
||||
evidence: ["No known app metadata detected"],
|
||||
notes: ["The verify sub-agent should inspect the repo directly and derive commands from the codebase."],
|
||||
};
|
||||
}
|
||||
|
||||
function inferFallbackRecipe(cwd: string, pkg: PackageJsonLike | null, packageManager: string | null): VerifyRecipe {
|
||||
if (pkg) return detectNodeRecipe(cwd, pkg, packageManager);
|
||||
return (
|
||||
detectPythonRecipe(cwd) ??
|
||||
detectGoRecipe(cwd) ??
|
||||
detectRustRecipe(cwd) ??
|
||||
detectJavaRecipe(cwd) ??
|
||||
detectFallbackRecipe(cwd)
|
||||
);
|
||||
}
|
||||
|
||||
export function normalizeVerifyRecipe(value: unknown): VerifyRecipe | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const asStrings = (input: unknown): string[] =>
|
||||
Array.isArray(input)
|
||||
? input.filter((v): v is string => typeof v === "string" && v.trim() !== "").map((v) => v.trim())
|
||||
: [];
|
||||
const ecosystem = typeof raw.ecosystem === "string" ? raw.ecosystem.trim() : "";
|
||||
const appKind = typeof raw.appKind === "string" ? raw.appKind.trim() : "";
|
||||
const appLabel = typeof raw.appLabel === "string" ? raw.appLabel.trim() : "";
|
||||
const smokeKind =
|
||||
raw.smokeKind === "http" || raw.smokeKind === "cli" || raw.smokeKind === "none" ? raw.smokeKind : "none";
|
||||
if (!ecosystem || !appKind || !appLabel) return null;
|
||||
return {
|
||||
ecosystem,
|
||||
appKind,
|
||||
appLabel,
|
||||
shellInitCommands: asStrings(raw.shellInitCommands),
|
||||
bootstrapCommands: asStrings(raw.bootstrapCommands),
|
||||
installCommands: asStrings(raw.installCommands),
|
||||
buildCommands: asStrings(raw.buildCommands),
|
||||
testCommands: asStrings(raw.testCommands),
|
||||
startCommand: typeof raw.startCommand === "string" && raw.startCommand.trim() ? raw.startCommand.trim() : undefined,
|
||||
startPort: typeof raw.startPort === "string" && raw.startPort.trim() ? raw.startPort.trim() : undefined,
|
||||
smokeKind,
|
||||
smokeTarget: typeof raw.smokeTarget === "string" && raw.smokeTarget.trim() ? raw.smokeTarget.trim() : undefined,
|
||||
evidence: asStrings(raw.evidence),
|
||||
notes: asStrings(raw.notes),
|
||||
};
|
||||
}
|
||||
|
||||
export function inferVerifySmokeUrl(settings?: SandboxSettings): string | null {
|
||||
const ports = settings?.ports ?? [];
|
||||
if (ports.length !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const hostPort = parseHostPort(ports[0]);
|
||||
return hostPort ? `http://127.0.0.1:${hostPort}` : null;
|
||||
}
|
||||
|
||||
export function inferVerifyProjectProfile(
|
||||
cwd: string,
|
||||
baseSettings: SandboxSettings = {},
|
||||
recipeOverride?: VerifyRecipe | null,
|
||||
): VerifyProjectProfile {
|
||||
const pkg = readPackageJson(cwd);
|
||||
const packageManager = detectPackageManager(cwd);
|
||||
const recipe = recipeOverride ?? inferFallbackRecipe(cwd, pkg, packageManager);
|
||||
const inferredDefaults: SandboxSettings =
|
||||
recipe.smokeKind === "http" && recipe.startPort ? { ports: [`${recipe.startPort}:${recipe.startPort}`] } : {};
|
||||
const sandboxSettings = mergeSandboxSettings(inferredDefaults, baseSettings);
|
||||
const smokeUrl = inferVerifySmokeUrl(sandboxSettings);
|
||||
|
||||
const recipeWithRuntime: VerifyRecipe = {
|
||||
...recipe,
|
||||
smokeTarget: recipe.smokeKind === "http" ? (smokeUrl ?? recipe.smokeTarget) : undefined,
|
||||
};
|
||||
|
||||
if (!fs.existsSync(path.join(cwd, "node_modules")) && recipeWithRuntime.ecosystem === "node") {
|
||||
recipeWithRuntime.notes = dedupe([
|
||||
...recipeWithRuntime.notes,
|
||||
"Host dependencies are not installed in node_modules. Verification may be limited unless a Shuru checkpoint already contains the needed runtime dependencies.",
|
||||
]);
|
||||
}
|
||||
|
||||
return {
|
||||
appKind: normalizeVerifyAppKind(recipeWithRuntime.appKind),
|
||||
appLabel: recipeWithRuntime.appLabel,
|
||||
packageManager,
|
||||
availableScripts: Object.keys(pkg?.scripts ?? {}),
|
||||
hasNodeModules: fs.existsSync(path.join(cwd, "node_modules")),
|
||||
sandboxSettings,
|
||||
recipe: recipeWithRuntime,
|
||||
};
|
||||
}
|
||||
|
||||
function buildBrowserGuidance(profile: VerifyProjectProfile): string[] {
|
||||
if (profile.recipe.smokeKind === "http" && profile.recipe.smokeTarget) {
|
||||
return [
|
||||
`- REQUIRED: After the dev server is running, you MUST run browser smoke tests against ${profile.recipe.smokeTarget}.`,
|
||||
"- The agent-browser command runs on the HOST, not inside the sandbox. It WILL work. Do not skip it or assume it is unavailable.",
|
||||
"- Run this exact sequence using the bash tool:",
|
||||
` mkdir -p .grok/verify-artifacts && agent-browser record start .grok/verify-artifacts/verify-smoke.webm && agent-browser --screenshot-dir .grok/verify-artifacts open ${profile.recipe.smokeTarget} && agent-browser wait --load networkidle && agent-browser --screenshot-dir .grok/verify-artifacts screenshot && agent-browser get title && agent-browser record stop && agent-browser close`,
|
||||
"- IMPORTANT: Use --screenshot-dir .grok/verify-artifacts to control where screenshots are saved. Do NOT pass a filename as a positional arg to the screenshot command.",
|
||||
"- IMPORTANT: Use `agent-browser record start <path>` and `agent-browser record stop` to capture a video recording of the smoke test flow.",
|
||||
"- If that command fails, report the exact error output. Do not preemptively skip browser checks.",
|
||||
"- Include both the screenshot and video file paths from the output in the Evidence section.",
|
||||
];
|
||||
}
|
||||
|
||||
if ((profile.sandboxSettings.ports?.length ?? 0) > 1) {
|
||||
return [
|
||||
"- Multiple forwarded ports are configured, so browser smoke testing is ambiguous.",
|
||||
"- Skip browser checks unless the user clearly identifies which forwarded localhost URL to verify.",
|
||||
];
|
||||
}
|
||||
|
||||
if (profile.recipe.smokeKind === "cli") {
|
||||
return ["- This project appears to need CLI-style runtime validation rather than browser smoke testing."];
|
||||
}
|
||||
|
||||
return [
|
||||
"- No unambiguous forwarded localhost URL is configured, so browser smoke testing is optional and should usually be skipped.",
|
||||
"- If the app can still be verified with bash-only checks, do that and explain why browser checks were skipped.",
|
||||
];
|
||||
}
|
||||
|
||||
function formatRecipeCommands(title: string, commands: string[]): string {
|
||||
return commands.length > 0 ? `- ${title}: ${commands.join(" ; ")}` : `- ${title}: (none inferred)`;
|
||||
}
|
||||
@@ -564,7 +66,9 @@ export function buildVerifyTaskPrompt(
|
||||
settings?: SandboxSettings,
|
||||
recipeOverride?: VerifyRecipe | null,
|
||||
): string {
|
||||
const profile = inferVerifyProjectProfile(cwd, settings, recipeOverride);
|
||||
const manifest = recipeOverride ? null : loadVerifyEnvironment(cwd, settings);
|
||||
const effectiveSettings = manifest?.sandboxSettings ?? settings;
|
||||
const profile = inferVerifyProjectProfile(cwd, effectiveSettings, recipeOverride ?? manifest?.recipe ?? null);
|
||||
const checkpoint = profile.sandboxSettings.from?.trim();
|
||||
const network = profile.sandboxSettings.allowNet
|
||||
? profile.sandboxSettings.allowedHosts?.length
|
||||
@@ -581,6 +85,7 @@ export function buildVerifyTaskPrompt(
|
||||
"- Then execute that recipe inside the active Shuru sandbox and report the result.",
|
||||
"",
|
||||
"Detected project context and inferred recipe:",
|
||||
...(manifest ? [`- Verify environment manifest: ${manifest.path}.`] : []),
|
||||
...buildProjectContextLines(profile),
|
||||
"",
|
||||
"Environment:",
|
||||
@@ -591,32 +96,47 @@ export function buildVerifyTaskPrompt(
|
||||
: "- No Shuru checkpoint is configured; use the current sandbox settings as-is.",
|
||||
"- Shuru runs are ephemeral in this version. Shell-side workspace edits do not persist back to the host.",
|
||||
"",
|
||||
"Required workflow:",
|
||||
"- Quickly inspect the code and config files to confirm or correct the inferred recipe before executing it.",
|
||||
"- If the inferred recipe is wrong, say what you changed and why before proceeding.",
|
||||
"- Prefer the recipe commands as the default execution plan.",
|
||||
"- In verify mode, ephemeral dependency installs are allowed inside the sandbox.",
|
||||
"- If the recipe needs install/setup work and no checkpoint already provides it, prefer chaining install + build/test/start in the same sandbox command so the installed dependencies remain available for that command.",
|
||||
"- Use background bash only when a dev server or watcher must stay alive while you continue verifying.",
|
||||
"- If an inferred install/setup step is blocked by sandbox persistence rules, explain the blocker clearly and continue with any valid non-setup checks.",
|
||||
"- IMPORTANT: agent-browser commands run on the HOST, not inside the sandbox. They WILL work. Do not skip browser checks or assume agent-browser is unavailable. Just run the command via the bash tool.",
|
||||
"- Always save a screenshot and include the screenshot file path in the final report so the user can verify visually.",
|
||||
"MANDATORY workflow (do ALL steps in order, do NOT stop after build/lint):",
|
||||
"",
|
||||
"Phase 1 — Setup:",
|
||||
"- Probe the sandbox for runtimes (`command -v node`, `command -v npm`, etc). Only install what is missing.",
|
||||
"- Ephemeral installs are allowed. Chain install + build in the same sandbox command if no checkpoint provides deps.",
|
||||
"",
|
||||
"Phase 2 — Build and test:",
|
||||
"- Run installCommands, buildCommands, and testCommands from the recipe.",
|
||||
"",
|
||||
"Phase 3 — Start the app (REQUIRED, do not skip):",
|
||||
"- Start the app using startCommand from the recipe, running it in the background.",
|
||||
"- Wait for the app to be ready: use a curl readiness loop or `agent-browser wait --load networkidle`.",
|
||||
"- If the app fails to start, report the error but still attempt to capture evidence (logs, screenshots).",
|
||||
"",
|
||||
"Phase 4 — Browser QA testing (REQUIRED, do not skip):",
|
||||
"- You are a QA tester. Open the app in the browser and test it like a human would.",
|
||||
"- agent-browser commands run on the HOST, not the sandbox. They WILL work. Do not skip them.",
|
||||
"- Record a video of the entire browser session.",
|
||||
"- Navigate the app: click links, buttons, menus. Verify pages load correctly.",
|
||||
"- Check for JavaScript console errors.",
|
||||
"- Spend 3-5 interactions testing the critical path. Take screenshots after each.",
|
||||
"- This is the most important phase. Build/lint passing means nothing if the app doesn't actually work.",
|
||||
"",
|
||||
"Phase 5 — Teardown:",
|
||||
"- Stop recording, close browser, THEN stop the dev server.",
|
||||
...buildReadinessGuidance(profile),
|
||||
...buildBrowserGuidance(profile),
|
||||
...buildRetryGuidance(profile),
|
||||
...buildEvidenceGuidance(),
|
||||
"",
|
||||
"Reporting requirements:",
|
||||
"- Return a concise structured report with these sections:",
|
||||
"- Return a concise structured report with these sections only:",
|
||||
" Summary",
|
||||
" Recipe",
|
||||
" Environment",
|
||||
" Commands Run",
|
||||
" Browser Checks",
|
||||
" Evidence",
|
||||
" Results",
|
||||
" Failures or Blockers",
|
||||
" Evidence",
|
||||
" Blockers",
|
||||
" Residual Risk",
|
||||
"- The Recipe section must say what recipe you used and whether you changed the inferred one.",
|
||||
"- If you captured screenshots or other browser artifacts, include their exact workspace-relative file paths in the Evidence section.",
|
||||
"- Use markdown links for screenshot paths when practical, otherwise include the plain relative paths.",
|
||||
"- Keep the report compact: prefer 1-3 short bullets per section, and do not paste large command logs unless they are essential blockers.",
|
||||
"- The Summary must say what recipe/source of truth you used and whether you changed the inferred/default one.",
|
||||
"- Evidence is mandatory even on failure. If you captured screenshots, video, or logs, include their exact workspace-relative file paths in the Evidence section.",
|
||||
"- Use markdown links for artifact paths when practical, otherwise include the plain relative paths.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -633,15 +153,22 @@ export function createVerifyTaskRequest(
|
||||
}
|
||||
|
||||
export function buildVerifyDetectPrompt(cwd: string, settings?: SandboxSettings): string {
|
||||
const fallbackProfile = inferVerifyProjectProfile(cwd, settings);
|
||||
const manifest = loadVerifyEnvironment(cwd, settings);
|
||||
const fallbackProfile = inferVerifyProjectProfile(
|
||||
cwd,
|
||||
manifest?.sandboxSettings ?? settings,
|
||||
manifest?.recipe ?? null,
|
||||
);
|
||||
return [
|
||||
"Inspect this repository and produce a structured verification recipe.",
|
||||
"",
|
||||
"Your job:",
|
||||
"- Read the codebase, config files, and any relevant docs or AGENTS guidance.",
|
||||
"- If environment.json or .grok/environment.json exists, treat it as the highest-priority source of truth and only fill in missing details.",
|
||||
"- Infer how the project should be installed, built, tested, and started.",
|
||||
"- Infer whether verification should use HTTP/browser smoke checks, CLI checks, or no runtime smoke step.",
|
||||
"- Prefer concrete commands that are likely to work in a fresh Debian Linux environment.",
|
||||
"- Design the recipe so verification probes for runtimes/tools first and only installs missing dependencies or toolchains when necessary.",
|
||||
"- Use the fallback hints below only as clues, not as the final answer.",
|
||||
"",
|
||||
"IMPORTANT for shellInitCommands and bootstrapCommands:",
|
||||
@@ -653,6 +180,7 @@ export function buildVerifyDetectPrompt(cwd: string, settings?: SandboxSettings)
|
||||
'- Example: ["apt-get update && apt-get install -y curl unzip ca-certificates && curl -fsSL https://bun.sh/install | bash", "apt-get install -y nodejs npm"]',
|
||||
"",
|
||||
"Fallback hints from static detection:",
|
||||
...(manifest ? [`- Detected verify environment manifest: ${manifest.path}.`] : []),
|
||||
...buildProjectContextLines(fallbackProfile),
|
||||
"",
|
||||
"Return ONLY valid JSON with this exact shape:",
|
||||
@@ -680,12 +208,104 @@ export function buildVerifyDetectPrompt(cwd: string, settings?: SandboxSettings)
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const NODE_ECOSYSTEMS = new Set(["node", "nodejs", "npm", "bun", "yarn", "pnpm"]);
|
||||
const PYTHON_ECOSYSTEMS = new Set(["python", "django", "fastapi"]);
|
||||
const GO_ECOSYSTEMS = new Set(["go", "golang"]);
|
||||
const RUST_ECOSYSTEMS = new Set(["rust", "cargo"]);
|
||||
|
||||
function inferBootstrapFromEcosystem(
|
||||
ecosystem: string,
|
||||
packageManager: string | null,
|
||||
): { bootstrap: string[]; shellInit: string[] } {
|
||||
const eco = ecosystem.toLowerCase();
|
||||
|
||||
if (
|
||||
NODE_ECOSYSTEMS.has(eco) ||
|
||||
eco.includes("node") ||
|
||||
eco.includes("next") ||
|
||||
eco.includes("react") ||
|
||||
eco.includes("vite")
|
||||
) {
|
||||
const bootstrap = [
|
||||
"apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm",
|
||||
];
|
||||
const shellInit = [...defaultShellInit()];
|
||||
if (packageManager === "bun") {
|
||||
bootstrap.push("curl -fsSL https://bun.sh/install | bash");
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template
|
||||
shellInit.push('export BUN_INSTALL="${HOME}/.bun"');
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template
|
||||
shellInit.push('export PATH="${BUN_INSTALL}/bin:$PATH"');
|
||||
}
|
||||
return { bootstrap, shellInit };
|
||||
}
|
||||
|
||||
if (PYTHON_ECOSYSTEMS.has(eco) || eco.includes("python") || eco.includes("django") || eco.includes("flask")) {
|
||||
return {
|
||||
bootstrap: ["apt-get update && apt-get install -y python3 python3-pip python3-venv ca-certificates git"],
|
||||
shellInit: defaultShellInit(),
|
||||
};
|
||||
}
|
||||
|
||||
if (GO_ECOSYSTEMS.has(eco) || eco.includes("go")) {
|
||||
return {
|
||||
bootstrap: ["apt-get update && apt-get install -y golang ca-certificates git"],
|
||||
shellInit: defaultShellInit(),
|
||||
};
|
||||
}
|
||||
|
||||
if (RUST_ECOSYSTEMS.has(eco) || eco.includes("rust")) {
|
||||
return {
|
||||
bootstrap: [
|
||||
"apt-get update && apt-get install -y curl ca-certificates git build-essential && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y",
|
||||
],
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template
|
||||
shellInit: [...defaultShellInit(), 'export PATH="${HOME}/.cargo/bin:$PATH"'],
|
||||
};
|
||||
}
|
||||
|
||||
return { bootstrap: [], shellInit: [] };
|
||||
}
|
||||
|
||||
function ensureBootstrapCommands(cwd: string, recipe: VerifyRecipe): VerifyRecipe {
|
||||
if (recipe.bootstrapCommands.length > 0) return recipe;
|
||||
if (recipe.installCommands.length === 0) return recipe;
|
||||
|
||||
const packageManager = detectPackageManager(cwd);
|
||||
|
||||
const appKind = normalizeVerifyAppKind(recipe.appKind);
|
||||
const webBootstrap = getNodeWebBootstrapCommands(packageManager, appKind);
|
||||
if (webBootstrap.length > 0) {
|
||||
const webShellInit = getNodeWebShellInitCommands(packageManager, appKind);
|
||||
return {
|
||||
...recipe,
|
||||
bootstrapCommands: webBootstrap,
|
||||
shellInitCommands: dedupe([...recipe.shellInitCommands, ...webShellInit]),
|
||||
};
|
||||
}
|
||||
|
||||
const { bootstrap, shellInit } = inferBootstrapFromEcosystem(recipe.ecosystem, packageManager);
|
||||
if (bootstrap.length === 0) return recipe;
|
||||
|
||||
return {
|
||||
...recipe,
|
||||
bootstrapCommands: bootstrap,
|
||||
shellInitCommands: dedupe([...recipe.shellInitCommands, ...shellInit]),
|
||||
};
|
||||
}
|
||||
|
||||
export function createVerifyRuntimeConfig(
|
||||
cwd: string,
|
||||
baseSettings: SandboxSettings = {},
|
||||
recipeOverride?: VerifyRecipe | null,
|
||||
): VerifyRuntimeConfig {
|
||||
const profile = inferVerifyProjectProfile(cwd, baseSettings, recipeOverride);
|
||||
const manifest = recipeOverride ? null : loadVerifyEnvironment(cwd, baseSettings);
|
||||
const profile = inferVerifyProjectProfile(
|
||||
cwd,
|
||||
manifest?.sandboxSettings ?? baseSettings,
|
||||
recipeOverride ?? manifest?.recipe ?? null,
|
||||
);
|
||||
profile.recipe = ensureBootstrapCommands(cwd, profile.recipe);
|
||||
const sandboxSettings = {
|
||||
...profile.sandboxSettings,
|
||||
allowNet: true,
|
||||
@@ -693,7 +313,7 @@ export function createVerifyRuntimeConfig(
|
||||
allowEphemeralInstall: true,
|
||||
hostBrowserCommandsOnHost: true,
|
||||
};
|
||||
sandboxSettings.shellInit = profile.recipe.shellInitCommands;
|
||||
sandboxSettings.shellInit = dedupe([...(sandboxSettings.shellInit ?? []), ...profile.recipe.shellInitCommands]);
|
||||
return {
|
||||
sandboxMode: "shuru",
|
||||
sandboxSettings,
|
||||
@@ -702,33 +322,101 @@ export function createVerifyRuntimeConfig(
|
||||
};
|
||||
}
|
||||
|
||||
export const VERIFY_PROMPT = [
|
||||
"Verify this project locally. Follow these steps in order using the `task` tool:",
|
||||
"",
|
||||
"Step 1: Run the `verify-detect` sub-agent to inspect the repository and produce a verification recipe.",
|
||||
"- agent: verify-detect",
|
||||
'- description: "Detect verification recipe"',
|
||||
"- The prompt should ask it to read config files, package manifests, scripts, and source layout, then return ONLY a JSON VerifyRecipe object.",
|
||||
"- The JSON must include: ecosystem, appKind, appLabel, shellInitCommands, bootstrapCommands, installCommands, buildCommands, testCommands, startCommand, startPort, smokeKind, smokeTarget, evidence, notes.",
|
||||
"",
|
||||
"Step 2: Once you have the recipe JSON from step 1, run the `verify` sub-agent to execute it.",
|
||||
"- agent: verify",
|
||||
'- description: "Run local verification"',
|
||||
"- You MUST include the complete recipe JSON from step 1 in the verify prompt.",
|
||||
"- You MUST also include these execution instructions in the verify prompt:",
|
||||
" - The verify agent runs inside a Shuru sandbox with full network access enabled.",
|
||||
" - Ephemeral dependency installs are allowed inside the sandbox.",
|
||||
" - agent-browser commands run on the HOST, not inside the sandbox. They WILL work.",
|
||||
" - If the recipe has a startCommand and startPort, start the app in the background and run browser smoke tests with agent-browser against http://127.0.0.1:<startPort>.",
|
||||
" - Use `agent-browser record start .grok/verify-artifacts/verify-smoke.webm` before opening the page and `agent-browser record stop` after to capture video.",
|
||||
" - Use `agent-browser --screenshot-dir .grok/verify-artifacts screenshot` to capture screenshots.",
|
||||
" - Produce a structured verification report with: Summary, Recipe, Environment, Commands Run, Browser Checks, Evidence (with file paths), Results, Failures or Blockers, Residual Risk.",
|
||||
"",
|
||||
"Important:",
|
||||
"- Do NOT perform verification work yourself. Delegate each step to the respective sub-agent using the `task` tool.",
|
||||
"- Run step 1 first, then step 2. Do not skip or combine them.",
|
||||
"- After step 2 completes, relay the verification report back to the user.",
|
||||
].join("\n");
|
||||
export async function prepareVerifySandbox(
|
||||
cwd: string,
|
||||
baseSettings: SandboxSettings = {},
|
||||
recipeOverride?: VerifyRecipe | null,
|
||||
onProgress?: (detail: string) => void,
|
||||
): Promise<PreparedVerifySandbox> {
|
||||
const runtime = createVerifyRuntimeConfig(cwd, baseSettings, recipeOverride);
|
||||
onProgress?.("Preparing verify checkpoint");
|
||||
const checkpoint = await ensureVerifyCheckpoint(cwd, runtime.profile, runtime.sandboxSettings, onProgress);
|
||||
if (checkpoint.checkpointName) {
|
||||
runtime.sandboxSettings.from = checkpoint.checkpointName;
|
||||
if (checkpoint.guestWorkdir) {
|
||||
runtime.sandboxSettings.guestWorkdir = checkpoint.guestWorkdir;
|
||||
runtime.sandboxSettings.syncHostWorkspace = true;
|
||||
}
|
||||
onProgress?.(
|
||||
checkpoint.created
|
||||
? `Created verify checkpoint: ${checkpoint.checkpointName}`
|
||||
: `Using verify checkpoint: ${checkpoint.checkpointName}`,
|
||||
);
|
||||
} else {
|
||||
onProgress?.("No verify checkpoint needed");
|
||||
}
|
||||
return {
|
||||
profile: runtime.profile,
|
||||
sandboxSettings: runtime.sandboxSettings,
|
||||
checkpoint,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildVerifyPrompt(cwd: string): string {
|
||||
const profile = inferVerifyProjectProfile(cwd);
|
||||
const recipe = ensureBootstrapCommands(cwd, profile.recipe);
|
||||
const draftJson = JSON.stringify(
|
||||
{
|
||||
recipe: {
|
||||
ecosystem: recipe.ecosystem,
|
||||
appKind: recipe.appKind,
|
||||
appLabel: recipe.appLabel,
|
||||
shellInitCommands: recipe.shellInitCommands,
|
||||
bootstrapCommands: recipe.bootstrapCommands,
|
||||
installCommands: recipe.installCommands,
|
||||
buildCommands: recipe.buildCommands,
|
||||
testCommands: recipe.testCommands,
|
||||
startCommand: recipe.startCommand,
|
||||
startPort: recipe.startPort,
|
||||
smokeKind: recipe.smokeKind,
|
||||
smokeTarget: recipe.smokeTarget,
|
||||
evidence: recipe.evidence,
|
||||
notes: recipe.notes,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
return [
|
||||
"Verify this project locally. Follow these steps in order using the `task` tool:",
|
||||
"",
|
||||
"Step 1: Run the `verify-manifest` sub-agent to create or update `.grok/environment.json`.",
|
||||
"- agent: verify-manifest",
|
||||
'- description: "Create verify manifest"',
|
||||
"- Include this draft manifest JSON directly in the prompt so the sub-agent can refine and write it quickly:",
|
||||
"",
|
||||
"```json",
|
||||
draftJson,
|
||||
"```",
|
||||
"",
|
||||
"- Tell the sub-agent: Review this draft recipe against package.json and key config files. Adjust if needed, then write .grok/environment.json. If the draft looks correct, write it as-is. Do not over-research.",
|
||||
"- IMPORTANT: The Shuru sandbox is Debian 13 trixie on aarch64 (ARM64) with NOTHING pre-installed. bootstrapCommands MUST install every runtime from scratch (e.g. nodejs, npm, python3, go, cargo).",
|
||||
"",
|
||||
"Step 2: After the manifest is written, run the `verify` sub-agent.",
|
||||
"- agent: verify",
|
||||
'- description: "Run local verification"',
|
||||
"- Tell it to use `.grok/environment.json` as the source of truth.",
|
||||
"- Include these execution instructions:",
|
||||
" - Runs inside a Shuru sandbox (Debian 13 trixie, aarch64/ARM64) with full network access.",
|
||||
" - The sandbox has NOTHING pre-installed unless bootstrapCommands in the manifest already ran during checkpoint creation.",
|
||||
" - Ephemeral installs allowed.",
|
||||
" - Probe for runtimes first (`command -v node`, etc), only install what is missing.",
|
||||
" - agent-browser runs on the HOST, not the sandbox. It WILL work.",
|
||||
" - If recipe has startCommand + startPort, start app in background and run browser smoke tests.",
|
||||
" - Use `agent-browser record start .grok/verify-artifacts/verify-smoke.webm` before opening the page.",
|
||||
" - Use `agent-browser --screenshot-dir .grok/verify-artifacts screenshot` after the page loads.",
|
||||
" - CRITICAL: Stop the recording (`agent-browser record stop`) and close the browser (`agent-browser close`) BEFORE stopping the dev server. The server must stay alive until all browser commands finish.",
|
||||
" - Return a concise report: Summary, Results, Evidence (mandatory), Blockers, Residual Risk.",
|
||||
"",
|
||||
"Important:",
|
||||
"- Do NOT perform verification work yourself. Delegate each step via `task`.",
|
||||
"- Run step 1 first, then step 2.",
|
||||
"- After step 2, relay the verification report to the user.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export const VERIFY_PROMPT = "__DYNAMIC__";
|
||||
|
||||
export function getVerifyCliError(options: { hasPrompt?: boolean; hasMessageArgs?: boolean }): string | null {
|
||||
if (options.hasPrompt) {
|
||||
@@ -741,3 +429,11 @@ export function getVerifyCliError(options: { hasPrompt?: boolean; hasMessageArgs
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export {
|
||||
defaultShellInit,
|
||||
inferVerifyProjectProfile,
|
||||
inferVerifySmokeUrl,
|
||||
normalizeVerifyRecipe,
|
||||
type VerifyProjectProfile,
|
||||
} from "./recipes";
|
||||
@@ -0,0 +1,119 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { loadVerifyEnvironment, saveVerifyEnvironment } from "./environment";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(prefix: string): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("verify environment manifest", () => {
|
||||
it("loads a root environment.json manifest and normalizes sandbox settings", () => {
|
||||
const dir = makeTempDir("grok-verify-env-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "environment.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
ecosystem: "node",
|
||||
appKind: "nextjs",
|
||||
appLabel: "Demo app",
|
||||
install: ["npm ci"],
|
||||
build: ["npm run build"],
|
||||
start: "npm run start",
|
||||
startPort: "3000",
|
||||
smokeKind: "http",
|
||||
sandbox: {
|
||||
from: "verify-node-web",
|
||||
verifyBaseFrom: "verify-node-web-base",
|
||||
shellInit: ["export FOO=bar"],
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const loaded = loadVerifyEnvironment(dir, { allowNet: true });
|
||||
expect(loaded?.path).toBe(path.join(dir, "environment.json"));
|
||||
expect(loaded?.recipe.installCommands).toEqual(["npm ci"]);
|
||||
expect(loaded?.recipe.buildCommands).toEqual(["npm run build"]);
|
||||
expect(loaded?.recipe.startCommand).toBe("npm run start");
|
||||
expect(loaded?.sandboxSettings.from).toBe("verify-node-web");
|
||||
expect(loaded?.sandboxSettings.verifyBaseFrom).toBe("verify-node-web-base");
|
||||
expect(loaded?.sandboxSettings.shellInit).toEqual(["export FOO=bar"]);
|
||||
expect(loaded?.sandboxSettings.allowNet).toBe(true);
|
||||
});
|
||||
|
||||
it("prefers .grok/environment.json over the root manifest", () => {
|
||||
const dir = makeTempDir("grok-verify-env-precedence-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "environment.json"),
|
||||
JSON.stringify({
|
||||
ecosystem: "node",
|
||||
appKind: "vite",
|
||||
appLabel: "Root",
|
||||
install: ["npm install"],
|
||||
smokeKind: "none",
|
||||
}),
|
||||
);
|
||||
fs.mkdirSync(path.join(dir, ".grok"));
|
||||
fs.writeFileSync(
|
||||
path.join(dir, ".grok", "environment.json"),
|
||||
JSON.stringify({
|
||||
ecosystem: "node",
|
||||
appKind: "nextjs",
|
||||
appLabel: "Nested",
|
||||
install: ["npm ci"],
|
||||
smokeKind: "none",
|
||||
}),
|
||||
);
|
||||
|
||||
const loaded = loadVerifyEnvironment(dir);
|
||||
expect(loaded?.path).toBe(path.join(dir, ".grok", "environment.json"));
|
||||
expect(loaded?.recipe.appKind).toBe("nextjs");
|
||||
expect(loaded?.recipe.installCommands).toEqual(["npm ci"]);
|
||||
});
|
||||
|
||||
it("writes a generated .grok/environment.json manifest", () => {
|
||||
const dir = makeTempDir("grok-verify-env-write-");
|
||||
const written = saveVerifyEnvironment(
|
||||
dir,
|
||||
{
|
||||
ecosystem: "node",
|
||||
appKind: "nextjs",
|
||||
appLabel: "Generated app",
|
||||
shellInitCommands: ["export DEBIAN_FRONTEND=noninteractive"],
|
||||
bootstrapCommands: ["apt-get update"],
|
||||
installCommands: ["npm ci"],
|
||||
buildCommands: ["npm run build"],
|
||||
testCommands: ["npm run lint"],
|
||||
startCommand: "npm run start",
|
||||
startPort: "3000",
|
||||
smokeKind: "http",
|
||||
smokeTarget: "http://127.0.0.1:3000",
|
||||
evidence: ["Detected package.json"],
|
||||
notes: ["Generated by verify"],
|
||||
},
|
||||
{ verifyBaseFrom: "verify-node-web", ports: ["3000:3000"], allowEphemeralInstall: true },
|
||||
);
|
||||
|
||||
expect(written).toBe(path.join(dir, ".grok", "environment.json"));
|
||||
const loaded = loadVerifyEnvironment(dir);
|
||||
expect(loaded?.path).toBe(written);
|
||||
expect(loaded?.recipe.installCommands).toEqual(["npm ci"]);
|
||||
expect(loaded?.sandboxSettings.verifyBaseFrom).toBe("verify-node-web");
|
||||
expect(loaded?.sandboxSettings.ports).toEqual(["3000:3000"]);
|
||||
expect(loaded?.sandboxSettings.allowEphemeralInstall).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import type { VerifyEnvironmentManifest, VerifyRecipe } from "../types/index";
|
||||
import { mergeSandboxSettings, normalizeSandboxSettings, type SandboxSettings } from "../utils/settings";
|
||||
import { normalizeVerifyRecipe } from "./recipes";
|
||||
|
||||
export interface LoadedVerifyEnvironment {
|
||||
path: string;
|
||||
recipe: VerifyRecipe;
|
||||
sandboxSettings: SandboxSettings;
|
||||
}
|
||||
|
||||
const VERIFY_ENVIRONMENT_FILES = [".grok/environment.json", "environment.json"];
|
||||
const GENERATED_VERIFY_ENVIRONMENT = ".grok/environment.json";
|
||||
|
||||
function readJson(filePath: string): VerifyEnvironmentManifest | null {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8")) as VerifyEnvironmentManifest;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
if (typeof value === "string" && value.trim()) return [value.trim()];
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value
|
||||
.filter((entry): entry is string => typeof entry === "string" && entry.trim() !== "")
|
||||
.map((entry) => entry.trim());
|
||||
}
|
||||
|
||||
function normalizeManifestRecipe(raw: VerifyEnvironmentManifest): VerifyRecipe | null {
|
||||
const base = raw.recipe && typeof raw.recipe === "object" ? raw.recipe : raw;
|
||||
return normalizeVerifyRecipe({
|
||||
ecosystem: base.ecosystem ?? raw.ecosystem ?? "custom",
|
||||
appKind: base.appKind ?? raw.appKind ?? "unknown",
|
||||
appLabel: base.appLabel ?? raw.appLabel ?? "Custom verify environment",
|
||||
shellInitCommands: base.shellInitCommands ?? base.shellInit ?? raw.shellInitCommands ?? raw.shellInit ?? [],
|
||||
bootstrapCommands: base.bootstrapCommands ?? base.bootstrap ?? raw.bootstrapCommands ?? raw.bootstrap ?? [],
|
||||
installCommands: base.installCommands ?? base.install ?? raw.installCommands ?? raw.install ?? [],
|
||||
buildCommands: base.buildCommands ?? base.build ?? raw.buildCommands ?? raw.build ?? [],
|
||||
testCommands: base.testCommands ?? base.test ?? raw.testCommands ?? raw.test ?? [],
|
||||
startCommand: base.startCommand ?? base.start ?? raw.startCommand ?? raw.start,
|
||||
startPort: base.startPort ?? raw.startPort,
|
||||
smokeKind: base.smokeKind ?? raw.smokeKind ?? "none",
|
||||
smokeTarget: base.smokeTarget ?? raw.smokeTarget,
|
||||
evidence: asStringArray(base.evidence ?? raw.evidence),
|
||||
notes: asStringArray(base.notes ?? raw.notes),
|
||||
});
|
||||
}
|
||||
|
||||
export function loadVerifyEnvironment(cwd: string, baseSettings: SandboxSettings = {}): LoadedVerifyEnvironment | null {
|
||||
for (const candidate of VERIFY_ENVIRONMENT_FILES) {
|
||||
const filePath = path.join(cwd, candidate);
|
||||
if (!fs.existsSync(filePath)) continue;
|
||||
const manifest = readJson(filePath);
|
||||
if (!manifest) continue;
|
||||
const recipe = normalizeManifestRecipe(manifest);
|
||||
if (!recipe) continue;
|
||||
const sandboxSettings = mergeSandboxSettings(baseSettings, normalizeSandboxSettings(manifest.sandbox));
|
||||
return {
|
||||
path: filePath,
|
||||
recipe,
|
||||
sandboxSettings,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function pickPersistentSandboxSettings(settings: SandboxSettings): SandboxSettings {
|
||||
const persistent: SandboxSettings = {};
|
||||
if (settings.allowNet !== undefined) persistent.allowNet = settings.allowNet;
|
||||
if (settings.allowedHosts?.length) persistent.allowedHosts = settings.allowedHosts;
|
||||
if (settings.ports?.length) persistent.ports = settings.ports;
|
||||
if (settings.cpus) persistent.cpus = settings.cpus;
|
||||
if (settings.memory) persistent.memory = settings.memory;
|
||||
if (settings.diskSize) persistent.diskSize = settings.diskSize;
|
||||
if (settings.secrets?.length) persistent.secrets = settings.secrets;
|
||||
if (settings.from) persistent.from = settings.from;
|
||||
if (settings.verifyBaseFrom) persistent.verifyBaseFrom = settings.verifyBaseFrom;
|
||||
if (settings.guestWorkdir) persistent.guestWorkdir = settings.guestWorkdir;
|
||||
if (settings.syncHostWorkspace !== undefined) persistent.syncHostWorkspace = settings.syncHostWorkspace;
|
||||
if (settings.shellInit?.length) persistent.shellInit = settings.shellInit;
|
||||
return persistent;
|
||||
}
|
||||
|
||||
export function saveVerifyEnvironment(
|
||||
cwd: string,
|
||||
recipe: VerifyRecipe,
|
||||
sandboxSettings: SandboxSettings = {},
|
||||
): string {
|
||||
const filePath = path.join(cwd, GENERATED_VERIFY_ENVIRONMENT);
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
const payload: VerifyEnvironmentManifest = {
|
||||
recipe: {
|
||||
ecosystem: recipe.ecosystem,
|
||||
appKind: recipe.appKind,
|
||||
appLabel: recipe.appLabel,
|
||||
shellInitCommands: recipe.shellInitCommands,
|
||||
bootstrapCommands: recipe.bootstrapCommands,
|
||||
installCommands: recipe.installCommands,
|
||||
buildCommands: recipe.buildCommands,
|
||||
testCommands: recipe.testCommands,
|
||||
startCommand: recipe.startCommand,
|
||||
startPort: recipe.startPort,
|
||||
smokeKind: recipe.smokeKind,
|
||||
smokeTarget: recipe.smokeTarget,
|
||||
evidence: recipe.evidence,
|
||||
notes: recipe.notes,
|
||||
},
|
||||
sandbox: pickPersistentSandboxSettings(sandboxSettings),
|
||||
};
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
return filePath;
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { VerifyArtifact } from "../types/index";
|
||||
import type { VerifyProjectProfile } from "./recipes";
|
||||
|
||||
export const VERIFY_ARTIFACT_DIR = ".grok/verify-artifacts";
|
||||
export const VERIFY_SCREENSHOT_PATH = `${VERIFY_ARTIFACT_DIR}/verify-screenshot.png`;
|
||||
export const VERIFY_VIDEO_PATH = `${VERIFY_ARTIFACT_DIR}/verify-smoke.webm`;
|
||||
export const VERIFY_LOG_PATH = `${VERIFY_ARTIFACT_DIR}/app.log`;
|
||||
|
||||
export function getVerifyArtifacts(): VerifyArtifact[] {
|
||||
return [
|
||||
{ kind: "log", path: VERIFY_LOG_PATH, description: "Captured app stdout/stderr from the verify run" },
|
||||
{
|
||||
kind: "screenshot",
|
||||
path: VERIFY_SCREENSHOT_PATH,
|
||||
description: "Final browser screenshot from smoke verification",
|
||||
},
|
||||
{ kind: "video", path: VERIFY_VIDEO_PATH, description: "Recorded browser smoke flow" },
|
||||
];
|
||||
}
|
||||
|
||||
export function buildReadinessGuidance(profile: VerifyProjectProfile): string[] {
|
||||
if (profile.recipe.smokeKind !== "http" || !profile.recipe.startCommand || !profile.recipe.startPort) {
|
||||
return [
|
||||
"- If you start any process, capture stdout/stderr into a log file under .grok/verify-artifacts whenever practical.",
|
||||
];
|
||||
}
|
||||
|
||||
const target = profile.recipe.smokeTarget ?? `http://127.0.0.1:${profile.recipe.startPort}`;
|
||||
return [
|
||||
`- Start the app with logs redirected to ${VERIFY_LOG_PATH} so failures leave evidence.`,
|
||||
`- Before running browser checks, wait for readiness at ${target}. Prefer a bounded curl loop or equivalent readiness check instead of guessing.`,
|
||||
"- If readiness never succeeds, include the final log tail and exact readiness command output in the report.",
|
||||
];
|
||||
}
|
||||
|
||||
export function buildBrowserGuidance(profile: VerifyProjectProfile): string[] {
|
||||
if (profile.recipe.smokeKind === "http" && profile.recipe.smokeTarget) {
|
||||
return [
|
||||
`- REQUIRED: After the dev server is running, you MUST run browser smoke tests against ${profile.recipe.smokeTarget}.`,
|
||||
"- The agent-browser command runs on the HOST, not inside the sandbox. It WILL work. Do not skip it or assume it is unavailable.",
|
||||
"- CRITICAL SEQUENCING: The dev server MUST stay running for the entire browser test. Do NOT stop the server until after recording is stopped and the browser is closed.",
|
||||
"",
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable reference in prompt text
|
||||
"- IMPORTANT: All artifact paths MUST be absolute. Use `$(pwd)/${VERIFY_ARTIFACT_DIR}` to construct them. Example: `$(pwd)/.grok/verify-artifacts/verify-smoke.webm`.",
|
||||
"",
|
||||
"- Browser smoke test procedure (run each step as a SEPARATE bash tool call):",
|
||||
` 1. mkdir -p ${VERIFY_ARTIFACT_DIR}`,
|
||||
` 2. agent-browser record start "$(pwd)/${VERIFY_VIDEO_PATH}"`,
|
||||
" 3. sleep 1",
|
||||
` 4. agent-browser --screenshot-dir "$(pwd)/${VERIFY_ARTIFACT_DIR}" open ${profile.recipe.smokeTarget}`,
|
||||
" 5. agent-browser wait --load networkidle",
|
||||
" 6. sleep 2",
|
||||
` 7. agent-browser --screenshot-dir "$(pwd)/${VERIFY_ARTIFACT_DIR}" screenshot`,
|
||||
" 8. agent-browser get title",
|
||||
"",
|
||||
"- INTERACTIVE TESTING (do this now, while recording is still running):",
|
||||
" Act like a real user doing an integration test. The goal is to verify the app actually works, not just that it loads.",
|
||||
" - Run each agent-browser command as a SEPARATE bash tool call. Do NOT chain them.",
|
||||
" - After each interaction, run `sleep 2` so the recording captures the result.",
|
||||
" - Use `agent-browser snapshot` to see the page structure and interactive elements.",
|
||||
" - Click links, buttons, and navigation using `agent-browser click <ref>` (refs from snapshot).",
|
||||
" - After each click, `agent-browser wait --load networkidle && sleep 2`, then snapshot and screenshot.",
|
||||
" - Check for console errors using `agent-browser console`.",
|
||||
" - Test at least: home page loads with content, primary navigation works, no JS errors in console.",
|
||||
" - If the app has forms, try filling one. If it has a login page, verify it renders.",
|
||||
" - Spend 3-5 interactions exploring the critical path. Do not rush.",
|
||||
"",
|
||||
"- Teardown (after interactive testing, run each as a separate bash call):",
|
||||
" 9. agent-browser record stop",
|
||||
" 10. sleep 2",
|
||||
` 11. ls -la "$(pwd)/${VERIFY_VIDEO_PATH}" (verify the video file exists and has a non-zero size)`,
|
||||
" 12. agent-browser close",
|
||||
" 13. Only NOW stop the dev server (kill the background process).",
|
||||
"",
|
||||
`- Keep all screenshots and video under ${VERIFY_ARTIFACT_DIR}. Include their exact paths in the Evidence section.`,
|
||||
"- If the video file does not exist after `record stop`, note that in Blockers.",
|
||||
"- If any browser command fails, report the exact error output. Do not skip browser checks.",
|
||||
];
|
||||
}
|
||||
|
||||
if ((profile.sandboxSettings.ports?.length ?? 0) > 1) {
|
||||
return [
|
||||
"- Multiple forwarded ports are configured, so browser smoke testing is ambiguous.",
|
||||
"- Skip browser checks unless the user clearly identifies which forwarded localhost URL to verify.",
|
||||
];
|
||||
}
|
||||
|
||||
if (profile.recipe.smokeKind === "cli") {
|
||||
return ["- This project appears to need CLI-style runtime validation rather than browser smoke testing."];
|
||||
}
|
||||
|
||||
return [
|
||||
"- No unambiguous forwarded localhost URL is configured, so browser smoke testing is optional and should usually be skipped.",
|
||||
"- If the app can still be verified with bash-only checks, do that and explain why browser checks were skipped.",
|
||||
];
|
||||
}
|
||||
|
||||
export function buildEvidenceGuidance(): string[] {
|
||||
const artifacts = getVerifyArtifacts();
|
||||
return [
|
||||
"- Always include exact artifact file paths in the Evidence section when they exist.",
|
||||
...artifacts.map((artifact) => `- Artifact target: ${artifact.path} (${artifact.description}).`),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("./checkpoint", () => ({
|
||||
ensureVerifyCheckpoint: vi.fn(async () => ({ created: false })),
|
||||
}));
|
||||
|
||||
import { ensureVerifyCheckpoint } from "./checkpoint";
|
||||
import { prepareVerifyRun, runVerifyOrchestration } from "./orchestrator";
|
||||
|
||||
const ensureVerifyCheckpointMock = vi.mocked(ensureVerifyCheckpoint);
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(prefix: string): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
ensureVerifyCheckpointMock.mockReset();
|
||||
ensureVerifyCheckpointMock.mockResolvedValue({ created: false });
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("verify orchestrator", () => {
|
||||
it("uses the environment manifest as the highest-priority recipe source", async () => {
|
||||
const dir = makeTempDir("grok-verify-orch-manifest-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "environment.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
ecosystem: "custom",
|
||||
appKind: "node",
|
||||
appLabel: "Manifest app",
|
||||
install: ["npm ci"],
|
||||
build: ["npm run build"],
|
||||
smokeKind: "none",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const agent = {
|
||||
getCwd: () => dir,
|
||||
getSandboxSettings: () => ({}),
|
||||
setSandboxSettings: vi.fn(),
|
||||
detectVerifyRecipe: vi.fn(async () => null),
|
||||
runTaskRequest: vi.fn(async () => ({ success: true, output: "ok" })),
|
||||
};
|
||||
|
||||
const prepared = await prepareVerifyRun(agent, {});
|
||||
expect(prepared.profile.recipe.ecosystem).toBe("custom");
|
||||
expect(prepared.profile.recipe.installCommands).toEqual(["npm ci"]);
|
||||
expect(prepared.usedVerifyDetect).toBe(false);
|
||||
expect(agent.detectVerifyRecipe).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("creates .grok/environment.json from verify-detect when no manifest exists", async () => {
|
||||
const dir = makeTempDir("grok-verify-orch-generate-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "package.json"),
|
||||
JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2),
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, "package-lock.json"), "");
|
||||
|
||||
const agent = {
|
||||
getCwd: () => dir,
|
||||
getSandboxSettings: () => ({}),
|
||||
setSandboxSettings: vi.fn(),
|
||||
detectVerifyRecipe: vi.fn(async () => ({
|
||||
ecosystem: "custom",
|
||||
appKind: "node",
|
||||
appLabel: "Agent recipe",
|
||||
shellInitCommands: ["export FOO=bar"],
|
||||
bootstrapCommands: ["echo bootstrap"],
|
||||
installCommands: ["npm ci"],
|
||||
buildCommands: ["npm run build"],
|
||||
testCommands: [],
|
||||
startCommand: "npm run start",
|
||||
startPort: "3000",
|
||||
smokeKind: "http" as const,
|
||||
smokeTarget: "http://127.0.0.1:3000",
|
||||
evidence: ["agent-detect"],
|
||||
notes: ["generated"],
|
||||
})),
|
||||
runTaskRequest: vi.fn(async () => ({ success: true, output: "ok" })),
|
||||
};
|
||||
|
||||
const prepared = await prepareVerifyRun(agent, {});
|
||||
expect(prepared.manifestPath).toBe(path.join(dir, ".grok", "environment.json"));
|
||||
expect(fs.existsSync(prepared.manifestPath!)).toBe(true);
|
||||
expect(prepared.usedVerifyDetect).toBe(true);
|
||||
expect(prepared.profile.recipe.installCommands).toEqual(["npm ci"]);
|
||||
expect(prepared.profile.recipe.evidence).toContain("agent-detect");
|
||||
});
|
||||
|
||||
it("does not create a manifest when verify-detect returns no recipe", async () => {
|
||||
const dir = makeTempDir("grok-verify-orch-no-manifest-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "package.json"),
|
||||
JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2),
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, "package-lock.json"), "");
|
||||
|
||||
const agent = {
|
||||
getCwd: () => dir,
|
||||
getSandboxSettings: () => ({}),
|
||||
setSandboxSettings: vi.fn(),
|
||||
detectVerifyRecipe: vi.fn(async () => null),
|
||||
runTaskRequest: vi.fn(async () => ({ success: true, output: "ok" })),
|
||||
};
|
||||
|
||||
const prepared = await prepareVerifyRun(agent, {});
|
||||
expect(prepared.manifestPath).toBeUndefined();
|
||||
expect(fs.existsSync(path.join(dir, ".grok", "environment.json"))).toBe(false);
|
||||
expect(prepared.usedVerifyDetect).toBe(false);
|
||||
});
|
||||
|
||||
it("restores sandbox settings after running verification and wires checkpoint settings", async () => {
|
||||
const dir = makeTempDir("grok-verify-orch-run-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "package.json"),
|
||||
JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2),
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, "package-lock.json"), "");
|
||||
|
||||
ensureVerifyCheckpointMock.mockResolvedValue({
|
||||
created: true,
|
||||
checkpointName: "verify-next-demo",
|
||||
guestWorkdir: "/grok/verify/worktree",
|
||||
});
|
||||
|
||||
const originalSettings = { allowNet: false, from: "base" };
|
||||
const agent = {
|
||||
getCwd: () => dir,
|
||||
getSandboxSettings: vi.fn(() => originalSettings),
|
||||
setSandboxSettings: vi.fn(),
|
||||
detectVerifyRecipe: vi.fn(async () => null),
|
||||
runTaskRequest: vi.fn(async () => ({ success: true, output: "verified" })),
|
||||
};
|
||||
|
||||
const result = await runVerifyOrchestration(agent, {});
|
||||
expect(result.success).toBe(true);
|
||||
expect(agent.setSandboxSettings).toHaveBeenCalledTimes(2);
|
||||
expect(agent.setSandboxSettings.mock.calls[0]?.[0]).toMatchObject({
|
||||
from: "verify-next-demo",
|
||||
guestWorkdir: "/grok/verify/worktree",
|
||||
syncHostWorkspace: true,
|
||||
allowNet: true,
|
||||
});
|
||||
expect(agent.setSandboxSettings.mock.calls[1]?.[0]).toBe(originalSettings);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { TaskRequest, ToolResult, VerifyRecipe } from "../types/index";
|
||||
import type { SandboxSettings } from "../utils/settings";
|
||||
import { ensureVerifyCheckpoint, type PreparedVerifyCheckpoint } from "./checkpoint";
|
||||
import { buildVerifyTaskPrompt } from "./entrypoint";
|
||||
import { loadVerifyEnvironment, saveVerifyEnvironment } from "./environment";
|
||||
import { inferVerifyProjectProfile, type VerifyProjectProfile } from "./recipes";
|
||||
|
||||
export interface VerifyAgentLike {
|
||||
getCwd(): string;
|
||||
getSandboxSettings(): SandboxSettings;
|
||||
setSandboxSettings(settings: SandboxSettings): void;
|
||||
detectVerifyRecipe(settings?: SandboxSettings, abortSignal?: AbortSignal): Promise<VerifyRecipe | null>;
|
||||
runTaskRequest(
|
||||
request: TaskRequest,
|
||||
onActivity?: (detail: string) => void,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<ToolResult>;
|
||||
}
|
||||
|
||||
export interface PreparedVerifyRun {
|
||||
profile: VerifyProjectProfile;
|
||||
sandboxSettings: SandboxSettings;
|
||||
taskRequest: TaskRequest;
|
||||
checkpoint?: PreparedVerifyCheckpoint;
|
||||
manifestPath?: string;
|
||||
usedVerifyDetect: boolean;
|
||||
}
|
||||
|
||||
export interface VerifyOrchestratorOptions {
|
||||
onProgress?: (detail: string) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
function buildRuntimeSandboxSettings(profile: VerifyProjectProfile): SandboxSettings {
|
||||
return {
|
||||
...profile.sandboxSettings,
|
||||
allowNet: true,
|
||||
allowedHosts: undefined,
|
||||
allowEphemeralInstall: true,
|
||||
hostBrowserCommandsOnHost: true,
|
||||
shellInit: [...new Set([...(profile.sandboxSettings.shellInit ?? []), ...profile.recipe.shellInitCommands])],
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareVerifyRun(
|
||||
agent: VerifyAgentLike,
|
||||
options: VerifyOrchestratorOptions = {},
|
||||
): Promise<PreparedVerifyRun> {
|
||||
const cwd = agent.getCwd();
|
||||
const baseSettings = agent.getSandboxSettings();
|
||||
const manifest = loadVerifyEnvironment(cwd, baseSettings);
|
||||
|
||||
if (manifest) {
|
||||
options.onProgress?.(`Loaded verify environment manifest: ${manifest.path}`);
|
||||
} else {
|
||||
options.onProgress?.("No verify environment manifest found; running verify-detect to generate one");
|
||||
}
|
||||
|
||||
let profile = inferVerifyProjectProfile(cwd, manifest?.sandboxSettings ?? baseSettings, manifest?.recipe ?? null);
|
||||
let usedVerifyDetect = false;
|
||||
let manifestPath = manifest?.path;
|
||||
|
||||
if (!manifest) {
|
||||
const detectedRecipe = await agent.detectVerifyRecipe(baseSettings, options.abortSignal);
|
||||
if (detectedRecipe) {
|
||||
usedVerifyDetect = true;
|
||||
profile = inferVerifyProjectProfile(cwd, baseSettings, detectedRecipe);
|
||||
options.onProgress?.(`verify-detect selected recipe for ${profile.appLabel}`);
|
||||
manifestPath = saveVerifyEnvironment(cwd, profile.recipe, profile.sandboxSettings);
|
||||
options.onProgress?.(`Created verify environment manifest: ${manifestPath}`);
|
||||
} else {
|
||||
options.onProgress?.(
|
||||
"verify-detect did not return a usable recipe; keeping deterministic fallback without writing a manifest",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const sandboxSettings = buildRuntimeSandboxSettings(profile);
|
||||
options.onProgress?.("Preparing verify checkpoint");
|
||||
const checkpoint = await ensureVerifyCheckpoint(cwd, profile, sandboxSettings);
|
||||
if (checkpoint.checkpointName) {
|
||||
sandboxSettings.from = checkpoint.checkpointName;
|
||||
if (checkpoint.guestWorkdir) {
|
||||
sandboxSettings.guestWorkdir = checkpoint.guestWorkdir;
|
||||
sandboxSettings.syncHostWorkspace = true;
|
||||
}
|
||||
options.onProgress?.(
|
||||
checkpoint.created
|
||||
? `Created verify checkpoint: ${checkpoint.checkpointName}`
|
||||
: `Using verify checkpoint: ${checkpoint.checkpointName}`,
|
||||
);
|
||||
} else {
|
||||
options.onProgress?.("No verify checkpoint needed for this recipe");
|
||||
}
|
||||
|
||||
const taskRequest: TaskRequest = {
|
||||
agent: "verify",
|
||||
description: "Run local verification",
|
||||
prompt: buildVerifyTaskPrompt(cwd, sandboxSettings, profile.recipe),
|
||||
};
|
||||
|
||||
return {
|
||||
profile,
|
||||
sandboxSettings,
|
||||
taskRequest,
|
||||
checkpoint,
|
||||
manifestPath,
|
||||
usedVerifyDetect,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runVerifyOrchestration(
|
||||
agent: VerifyAgentLike,
|
||||
options: VerifyOrchestratorOptions = {},
|
||||
): Promise<ToolResult> {
|
||||
const originalSandboxSettings = agent.getSandboxSettings();
|
||||
const prepared = await prepareVerifyRun(agent, options);
|
||||
agent.setSandboxSettings(prepared.sandboxSettings);
|
||||
try {
|
||||
options.onProgress?.("Running verify sub-agent");
|
||||
const result = await agent.runTaskRequest(prepared.taskRequest, options.onProgress, options.abortSignal);
|
||||
return {
|
||||
...result,
|
||||
verifyRecipe: prepared.profile.recipe,
|
||||
};
|
||||
} finally {
|
||||
agent.setSandboxSettings(originalSandboxSettings);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,516 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import type { VerifyRecipe } from "../types/index";
|
||||
import { mergeSandboxSettings, type SandboxSettings } from "../utils/settings";
|
||||
|
||||
export type VerifyAppKind =
|
||||
| "nextjs"
|
||||
| "vite"
|
||||
| "astro"
|
||||
| "sveltekit"
|
||||
| "remix"
|
||||
| "cra"
|
||||
| "node"
|
||||
| "django"
|
||||
| "python"
|
||||
| "go"
|
||||
| "rust"
|
||||
| "maven"
|
||||
| "gradle"
|
||||
| "make"
|
||||
| "unknown";
|
||||
|
||||
export interface VerifyProjectProfile {
|
||||
appKind: VerifyAppKind;
|
||||
appLabel: string;
|
||||
packageManager: string | null;
|
||||
availableScripts: string[];
|
||||
hasNodeModules: boolean;
|
||||
sandboxSettings: SandboxSettings;
|
||||
recipe: VerifyRecipe;
|
||||
}
|
||||
|
||||
interface PackageJsonLike {
|
||||
dependencies?: Record<string, string>;
|
||||
devDependencies?: Record<string, string>;
|
||||
scripts?: Record<string, string>;
|
||||
}
|
||||
|
||||
function fileExists(cwd: string, file: string): boolean {
|
||||
return fs.existsSync(path.join(cwd, file));
|
||||
}
|
||||
|
||||
function readTextFile(cwd: string, file: string): string | null {
|
||||
try {
|
||||
return fs.readFileSync(path.join(cwd, file), "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readPackageJson(cwd: string): PackageJsonLike | null {
|
||||
const raw = readTextFile(cwd, "package.json");
|
||||
if (!raw) return null;
|
||||
try {
|
||||
return JSON.parse(raw) as PackageJsonLike;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function detectPackageManager(cwd: string): string | null {
|
||||
const candidates: Array<[string, string]> = [
|
||||
["pnpm-lock.yaml", "pnpm"],
|
||||
["bun.lock", "bun"],
|
||||
["bun.lockb", "bun"],
|
||||
["yarn.lock", "yarn"],
|
||||
["package-lock.json", "npm"],
|
||||
["uv.lock", "uv"],
|
||||
["poetry.lock", "poetry"],
|
||||
["Pipfile.lock", "pipenv"],
|
||||
];
|
||||
|
||||
for (const [file, manager] of candidates) {
|
||||
if (fileExists(cwd, file)) return manager;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function dedupe(values: Array<string | undefined | null>): string[] {
|
||||
return [...new Set(values.map((v) => v?.trim()).filter((v): v is string => Boolean(v)))];
|
||||
}
|
||||
|
||||
export function defaultShellInit(): string[] {
|
||||
return ["export DEBIAN_FRONTEND=noninteractive"];
|
||||
}
|
||||
|
||||
const NODE_WEB_APP_KINDS = new Set<VerifyAppKind>(["nextjs", "vite", "astro", "sveltekit", "remix", "cra"]);
|
||||
|
||||
export function getNodeWebShellInitCommands(packageManager: string | null, appKind: VerifyAppKind): string[] {
|
||||
const commands = [...defaultShellInit()];
|
||||
if (!NODE_WEB_APP_KINDS.has(appKind)) {
|
||||
return commands;
|
||||
}
|
||||
if (packageManager === "bun") {
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template
|
||||
commands.push('export BUN_INSTALL="${HOME}/.bun"');
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: shell variable, not JS template
|
||||
commands.push('export PATH="${BUN_INSTALL}/bin:$PATH"');
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
export function getNodeWebBootstrapCommands(packageManager: string | null, appKind: VerifyAppKind): string[] {
|
||||
if (!NODE_WEB_APP_KINDS.has(appKind)) {
|
||||
return [];
|
||||
}
|
||||
const commands = [
|
||||
"apt-get update && apt-get install -y curl unzip ca-certificates git python3 make g++ pkg-config nodejs npm",
|
||||
];
|
||||
if (packageManager === "bun") {
|
||||
commands.push("curl -fsSL https://bun.sh/install | bash");
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
|
||||
function parseHostPort(mapping: string): string | null {
|
||||
const match = mapping.trim().match(/^(\d+):(\d+)$/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function inferPortFromCommand(command: string | undefined): string | undefined {
|
||||
if (!command) return undefined;
|
||||
const flagMatch = command.match(/(?:--port|-p)\s+(\d{2,5})/);
|
||||
if (flagMatch) return flagMatch[1];
|
||||
const envMatch = command.match(/\bPORT=(\d{2,5})\b/);
|
||||
if (envMatch) return envMatch[1];
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseTargetNames(raw: string): string[] {
|
||||
return raw
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.match(/^([A-Za-z0-9_.-]+):(?:\s|$)/)?.[1])
|
||||
.filter((target): target is string => Boolean(target));
|
||||
}
|
||||
|
||||
export function normalizeVerifyAppKind(value: string): VerifyAppKind {
|
||||
return (
|
||||
[
|
||||
"nextjs",
|
||||
"vite",
|
||||
"astro",
|
||||
"sveltekit",
|
||||
"remix",
|
||||
"cra",
|
||||
"node",
|
||||
"django",
|
||||
"python",
|
||||
"go",
|
||||
"rust",
|
||||
"maven",
|
||||
"gradle",
|
||||
"make",
|
||||
"unknown",
|
||||
] as const
|
||||
).includes(value as VerifyAppKind)
|
||||
? (value as VerifyAppKind)
|
||||
: "unknown";
|
||||
}
|
||||
|
||||
function pickPackageScript(packageManager: string | null, scripts: Record<string, string>, body: string): string {
|
||||
const entry = Object.entries(scripts).find(([, scriptBody]) => scriptBody === body)?.[0];
|
||||
if (!entry) return body;
|
||||
const runner =
|
||||
packageManager === "pnpm"
|
||||
? "pnpm"
|
||||
: packageManager === "bun"
|
||||
? "bun"
|
||||
: packageManager === "yarn"
|
||||
? "yarn"
|
||||
: "npm run";
|
||||
return runner === "yarn" ? `yarn ${entry}` : runner === "bun" ? `bun run ${entry}` : `${runner} ${entry}`;
|
||||
}
|
||||
|
||||
function detectMakeRecipe(cwd: string): VerifyRecipe | null {
|
||||
const makefile = readTextFile(cwd, "Makefile");
|
||||
if (!makefile) return null;
|
||||
const targets = parseTargetNames(makefile);
|
||||
const has = (names: string[]) => names.find((name) => targets.includes(name));
|
||||
const install = has(["install", "setup", "bootstrap"]);
|
||||
const build = has(["build", "compile"]);
|
||||
const test = has(["test", "check"]);
|
||||
const run = has(["run", "start", "serve", "dev"]);
|
||||
|
||||
return {
|
||||
ecosystem: "make",
|
||||
appKind: "make",
|
||||
appLabel: "Makefile-driven project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: install ? [`make ${install}`] : [],
|
||||
buildCommands: build ? [`make ${build}`] : [],
|
||||
testCommands: test ? [`make ${test}`] : [],
|
||||
startCommand: run ? `make ${run}` : undefined,
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Makefile", `Targets: ${targets.join(", ") || "(none)"}`],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectNodeRecipe(_cwd: string, pkg: PackageJsonLike, packageManager: string | null): VerifyRecipe {
|
||||
const scripts = pkg.scripts ?? {};
|
||||
const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
|
||||
let appKind: VerifyAppKind = "node";
|
||||
let appLabel = "Node.js app";
|
||||
let defaultPort: string | undefined;
|
||||
|
||||
if (deps.next) {
|
||||
appKind = "nextjs";
|
||||
appLabel = "Next.js";
|
||||
defaultPort = "3000";
|
||||
} else if (deps["@sveltejs/kit"]) {
|
||||
appKind = "sveltekit";
|
||||
appLabel = "SvelteKit";
|
||||
defaultPort = "5173";
|
||||
} else if (deps.astro) {
|
||||
appKind = "astro";
|
||||
appLabel = "Astro";
|
||||
defaultPort = "4321";
|
||||
} else if (deps["@remix-run/dev"] || deps["@remix-run/react"]) {
|
||||
appKind = "remix";
|
||||
appLabel = "Remix";
|
||||
defaultPort = "3000";
|
||||
} else if (deps["react-scripts"]) {
|
||||
appKind = "cra";
|
||||
appLabel = "Create React App";
|
||||
defaultPort = "3000";
|
||||
} else if (deps.vite) {
|
||||
appKind = "vite";
|
||||
appLabel = "Vite";
|
||||
defaultPort = "5173";
|
||||
}
|
||||
|
||||
const install = packageManager
|
||||
? packageManager === "pnpm"
|
||||
? "pnpm install"
|
||||
: packageManager === "bun"
|
||||
? "bun install"
|
||||
: packageManager === "yarn"
|
||||
? "yarn install"
|
||||
: "npm install"
|
||||
: undefined;
|
||||
const startCommand = scripts.dev ?? scripts.start;
|
||||
const startPort = inferPortFromCommand(startCommand) ?? defaultPort;
|
||||
const smokeKind: VerifyRecipe["smokeKind"] = startCommand && startPort ? "http" : "none";
|
||||
|
||||
return {
|
||||
ecosystem: "node",
|
||||
appKind,
|
||||
appLabel,
|
||||
shellInitCommands: getNodeWebShellInitCommands(packageManager, appKind),
|
||||
bootstrapCommands: getNodeWebBootstrapCommands(packageManager, appKind),
|
||||
installCommands: dedupe([install]),
|
||||
buildCommands: dedupe(
|
||||
[scripts.build, scripts.typecheck].map((script) => script && pickPackageScript(packageManager, scripts, script)),
|
||||
),
|
||||
testCommands: dedupe(
|
||||
["test", "check", "lint"]
|
||||
.filter((name) => scripts[name])
|
||||
.map((name) => pickPackageScript(packageManager, scripts, scripts[name]!)),
|
||||
),
|
||||
startCommand: startCommand ? pickPackageScript(packageManager, scripts, startCommand) : undefined,
|
||||
startPort,
|
||||
smokeKind,
|
||||
evidence: ["Detected package.json", `Scripts: ${Object.keys(scripts).join(", ") || "(none)"}`],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectPythonRecipe(cwd: string): VerifyRecipe | null {
|
||||
const pyproject = readTextFile(cwd, "pyproject.toml");
|
||||
const requirements = readTextFile(cwd, "requirements.txt");
|
||||
const managePy = fileExists(cwd, "manage.py");
|
||||
if (!pyproject && !requirements && !managePy && !fileExists(cwd, "setup.py")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const lower = `${pyproject ?? ""}\n${requirements ?? ""}`.toLowerCase();
|
||||
const packageManager = detectPackageManager(cwd);
|
||||
const isDjango = managePy || lower.includes("django");
|
||||
const isFastApi = lower.includes("fastapi") || lower.includes("uvicorn");
|
||||
|
||||
let install = "pip install -r requirements.txt";
|
||||
if (packageManager === "uv") install = "uv sync";
|
||||
else if (packageManager === "poetry") install = "poetry install";
|
||||
else if (packageManager === "pipenv") install = "pipenv install";
|
||||
else if (pyproject && !requirements) install = "pip install -e .";
|
||||
|
||||
if (isDjango) {
|
||||
return {
|
||||
ecosystem: "python",
|
||||
appKind: "django",
|
||||
appLabel: "Django app",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [install],
|
||||
buildCommands: [],
|
||||
testCommands: ["python manage.py test"],
|
||||
startCommand: "python manage.py runserver 0.0.0.0:8000",
|
||||
startPort: "8000",
|
||||
smokeKind: "http",
|
||||
evidence: ["Detected manage.py", pyproject ? "Detected pyproject.toml" : undefined].filter(Boolean) as string[],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (isFastApi) {
|
||||
const appModule = fileExists(cwd, "main.py") ? "main:app" : fileExists(cwd, "app.py") ? "app:app" : "main:app";
|
||||
return {
|
||||
ecosystem: "python",
|
||||
appKind: "python",
|
||||
appLabel: "Python web app",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [install],
|
||||
buildCommands: [],
|
||||
testCommands: fileExists(cwd, "tests") ? ["pytest"] : [],
|
||||
startCommand: `uvicorn ${appModule} --host 0.0.0.0 --port 8000`,
|
||||
startPort: "8000",
|
||||
smokeKind: "http",
|
||||
evidence: ["Detected Python project", "Detected FastAPI/Uvicorn dependency"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ecosystem: "python",
|
||||
appKind: "python",
|
||||
appLabel: "Python project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [install],
|
||||
buildCommands: [],
|
||||
testCommands: fileExists(cwd, "tests") ? ["pytest"] : ["python -m unittest discover"],
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Python project"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectGoRecipe(cwd: string): VerifyRecipe | null {
|
||||
if (!fileExists(cwd, "go.mod")) return null;
|
||||
return {
|
||||
ecosystem: "go",
|
||||
appKind: "go",
|
||||
appLabel: "Go project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: ["go build ./..."],
|
||||
testCommands: ["go test ./..."],
|
||||
startCommand: fileExists(cwd, "main.go") ? "go run ." : undefined,
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected go.mod"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectRustRecipe(cwd: string): VerifyRecipe | null {
|
||||
if (!fileExists(cwd, "Cargo.toml")) return null;
|
||||
return {
|
||||
ecosystem: "rust",
|
||||
appKind: "rust",
|
||||
appLabel: "Rust project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: ["cargo build"],
|
||||
testCommands: ["cargo test"],
|
||||
startCommand: fileExists(cwd, path.join("src", "main.rs")) ? "cargo run" : undefined,
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Cargo.toml"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
function detectJavaRecipe(cwd: string): VerifyRecipe | null {
|
||||
if (fileExists(cwd, "pom.xml")) {
|
||||
return {
|
||||
ecosystem: "java",
|
||||
appKind: "maven",
|
||||
appLabel: "Maven project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: ["mvn package"],
|
||||
testCommands: ["mvn test"],
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected pom.xml"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (fileExists(cwd, "build.gradle") || fileExists(cwd, "build.gradle.kts")) {
|
||||
const gradle = fileExists(cwd, "gradlew") ? "./gradlew" : "gradle";
|
||||
return {
|
||||
ecosystem: "java",
|
||||
appKind: "gradle",
|
||||
appLabel: "Gradle project",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: [`${gradle} build`],
|
||||
testCommands: [`${gradle} test`],
|
||||
smokeKind: "none",
|
||||
evidence: ["Detected Gradle build file"],
|
||||
notes: [],
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function detectFallbackRecipe(cwd: string): VerifyRecipe {
|
||||
const makeRecipe = detectMakeRecipe(cwd);
|
||||
if (makeRecipe) return makeRecipe;
|
||||
return {
|
||||
ecosystem: "unknown",
|
||||
appKind: "unknown",
|
||||
appLabel: "Unknown project type",
|
||||
shellInitCommands: defaultShellInit(),
|
||||
bootstrapCommands: [],
|
||||
installCommands: [],
|
||||
buildCommands: [],
|
||||
testCommands: [],
|
||||
smokeKind: "none",
|
||||
evidence: ["No known app metadata detected"],
|
||||
notes: ["The verify sub-agent should inspect the repo directly and derive commands from the codebase."],
|
||||
};
|
||||
}
|
||||
|
||||
function inferFallbackRecipe(cwd: string, pkg: PackageJsonLike | null, packageManager: string | null): VerifyRecipe {
|
||||
if (pkg) return detectNodeRecipe(cwd, pkg, packageManager);
|
||||
return (
|
||||
detectPythonRecipe(cwd) ??
|
||||
detectGoRecipe(cwd) ??
|
||||
detectRustRecipe(cwd) ??
|
||||
detectJavaRecipe(cwd) ??
|
||||
detectFallbackRecipe(cwd)
|
||||
);
|
||||
}
|
||||
|
||||
export function normalizeVerifyRecipe(value: unknown): VerifyRecipe | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const raw = value as Record<string, unknown>;
|
||||
const asStrings = (input: unknown): string[] =>
|
||||
Array.isArray(input)
|
||||
? input.filter((v): v is string => typeof v === "string" && v.trim() !== "").map((v) => v.trim())
|
||||
: [];
|
||||
const ecosystem = typeof raw.ecosystem === "string" ? raw.ecosystem.trim() : "";
|
||||
const appKind = typeof raw.appKind === "string" ? raw.appKind.trim() : "";
|
||||
const appLabel = typeof raw.appLabel === "string" ? raw.appLabel.trim() : "";
|
||||
const smokeKind =
|
||||
raw.smokeKind === "http" || raw.smokeKind === "cli" || raw.smokeKind === "none" ? raw.smokeKind : "none";
|
||||
if (!ecosystem || !appKind || !appLabel) return null;
|
||||
return {
|
||||
ecosystem,
|
||||
appKind,
|
||||
appLabel,
|
||||
shellInitCommands: asStrings(raw.shellInitCommands),
|
||||
bootstrapCommands: asStrings(raw.bootstrapCommands),
|
||||
installCommands: asStrings(raw.installCommands),
|
||||
buildCommands: asStrings(raw.buildCommands),
|
||||
testCommands: asStrings(raw.testCommands),
|
||||
startCommand: typeof raw.startCommand === "string" && raw.startCommand.trim() ? raw.startCommand.trim() : undefined,
|
||||
startPort: typeof raw.startPort === "string" && raw.startPort.trim() ? raw.startPort.trim() : undefined,
|
||||
smokeKind,
|
||||
smokeTarget: typeof raw.smokeTarget === "string" && raw.smokeTarget.trim() ? raw.smokeTarget.trim() : undefined,
|
||||
evidence: asStrings(raw.evidence),
|
||||
notes: asStrings(raw.notes),
|
||||
};
|
||||
}
|
||||
|
||||
export function inferVerifySmokeUrl(settings?: SandboxSettings): string | null {
|
||||
const ports = settings?.ports ?? [];
|
||||
if (ports.length !== 1) return null;
|
||||
const hostPort = parseHostPort(ports[0]);
|
||||
return hostPort ? `http://127.0.0.1:${hostPort}` : null;
|
||||
}
|
||||
|
||||
export function inferVerifyProjectProfile(
|
||||
cwd: string,
|
||||
baseSettings: SandboxSettings = {},
|
||||
recipeOverride?: VerifyRecipe | null,
|
||||
): VerifyProjectProfile {
|
||||
const pkg = readPackageJson(cwd);
|
||||
const packageManager = detectPackageManager(cwd);
|
||||
const recipe = recipeOverride ?? inferFallbackRecipe(cwd, pkg, packageManager);
|
||||
const inferredDefaults: SandboxSettings =
|
||||
recipe.smokeKind === "http" && recipe.startPort ? { ports: [`${recipe.startPort}:${recipe.startPort}`] } : {};
|
||||
const sandboxSettings = mergeSandboxSettings(inferredDefaults, baseSettings);
|
||||
const smokeUrl = inferVerifySmokeUrl(sandboxSettings);
|
||||
|
||||
const recipeWithRuntime: VerifyRecipe = {
|
||||
...recipe,
|
||||
smokeTarget: recipe.smokeKind === "http" ? (smokeUrl ?? recipe.smokeTarget) : undefined,
|
||||
};
|
||||
|
||||
if (!fs.existsSync(path.join(cwd, "node_modules")) && recipeWithRuntime.ecosystem === "node") {
|
||||
recipeWithRuntime.notes = dedupe([
|
||||
...recipeWithRuntime.notes,
|
||||
"Host dependencies are not installed in node_modules. Verification may be limited unless a Shuru checkpoint already contains the needed runtime dependencies.",
|
||||
]);
|
||||
}
|
||||
|
||||
return {
|
||||
appKind: normalizeVerifyAppKind(recipeWithRuntime.appKind),
|
||||
appLabel: recipeWithRuntime.appLabel,
|
||||
packageManager,
|
||||
availableScripts: Object.keys(pkg?.scripts ?? {}),
|
||||
hasNodeModules: fs.existsSync(path.join(cwd, "node_modules")),
|
||||
sandboxSettings,
|
||||
recipe: recipeWithRuntime,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { VerifyRetryStrategy } from "../types/index";
|
||||
import type { VerifyProjectProfile } from "./recipes";
|
||||
|
||||
const NODE_WEB_APP_KINDS = new Set(["nextjs", "vite", "astro", "sveltekit", "remix", "cra"]);
|
||||
|
||||
function buildNativeModuleRetry(profile: VerifyProjectProfile): VerifyRetryStrategy | null {
|
||||
if (!NODE_WEB_APP_KINDS.has(profile.appKind)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const installCommand =
|
||||
profile.packageManager === "bun"
|
||||
? "npm install --include=optional"
|
||||
: profile.packageManager === "pnpm"
|
||||
? "pnpm install"
|
||||
: profile.packageManager === "yarn"
|
||||
? "yarn install"
|
||||
: "npm install --include=optional";
|
||||
|
||||
return {
|
||||
id: "native-module-retry",
|
||||
when: "Only if install or build fails with missing native bindings such as lightningcss, sharp, @next/swc, esbuild, rollup, or a missing *.linux-*.node module.",
|
||||
reason: "Recover from platform-specific optional/native dependency installs without unbounded retry loops.",
|
||||
commands: ["rm -rf node_modules", installCommand, ...profile.recipe.buildCommands],
|
||||
};
|
||||
}
|
||||
|
||||
function buildStartupRetry(profile: VerifyProjectProfile): VerifyRetryStrategy | null {
|
||||
if (profile.recipe.smokeKind !== "http" || !profile.recipe.startCommand || !profile.recipe.startPort) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
id: "startup-readiness-retry",
|
||||
when: "Only if the app starts but never becomes reachable at the expected localhost URL.",
|
||||
reason: "Give one deterministic retry with explicit host/port binding before declaring the app non-runnable.",
|
||||
commands: [`HOST=0.0.0.0 PORT=${profile.recipe.startPort} ${profile.recipe.startCommand}`],
|
||||
};
|
||||
}
|
||||
|
||||
export function getVerifyRetryStrategies(profile: VerifyProjectProfile): VerifyRetryStrategy[] {
|
||||
return [buildNativeModuleRetry(profile), buildStartupRetry(profile)].filter(
|
||||
(strategy): strategy is VerifyRetryStrategy => Boolean(strategy),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildRetryGuidance(profile: VerifyProjectProfile): string[] {
|
||||
const strategies = getVerifyRetryStrategies(profile);
|
||||
if (strategies.length === 0) {
|
||||
return [
|
||||
"- Do not thrash. If a failure is not covered by a known retry strategy, report it directly instead of guessing.",
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
"- Retry policy: at most one bounded retry per known failure class. Do not loop or improvise multiple package-manager swaps.",
|
||||
...strategies.flatMap((strategy) => [
|
||||
`- Retry strategy (${strategy.id}): ${strategy.when}`,
|
||||
`- Reason: ${strategy.reason}`,
|
||||
`- Commands: ${strategy.commands.join(" && ")}`,
|
||||
]),
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
vi.mock("./checkpoint", () => ({
|
||||
ensureVerifyCheckpoint: vi.fn(async () => ({
|
||||
created: true,
|
||||
checkpointName: "verify-nextjs-test",
|
||||
guestWorkdir: "/grok/verify/worktree",
|
||||
})),
|
||||
}));
|
||||
|
||||
import { prepareVerifySandbox } from "./entrypoint";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(prefix: string): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("prepareVerifySandbox", () => {
|
||||
it("adds checkpoint-backed sandbox settings for verify execution", async () => {
|
||||
const dir = makeTempDir("grok-verify-runtime-prep-");
|
||||
fs.writeFileSync(
|
||||
path.join(dir, "package.json"),
|
||||
JSON.stringify({ dependencies: { next: "15.0.0" }, scripts: { dev: "next dev", build: "next build" } }, null, 2),
|
||||
);
|
||||
fs.writeFileSync(path.join(dir, "package-lock.json"), "");
|
||||
|
||||
const prepared = await prepareVerifySandbox(dir, { allowNet: false });
|
||||
expect(prepared.profile.appKind).toBe("nextjs");
|
||||
expect(prepared.sandboxSettings.allowNet).toBe(true);
|
||||
expect(prepared.sandboxSettings.from).toBe("verify-nextjs-test");
|
||||
expect(prepared.sandboxSettings.guestWorkdir).toBe("/grok/verify/worktree");
|
||||
expect(prepared.sandboxSettings.syncHostWorkspace).toBe(true);
|
||||
expect(prepared.checkpoint?.created).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user