feat: add cascade strategy probe, intercept pipeline step, buildTemplatedUrl

src/cascade.ts

@@ -0,0 +1,217 @@
+/**
+ * Strategy Cascade: automatic strategy downgrade chain.
+ *
+ * Probes an API endpoint starting from the simplest strategy (PUBLIC)
+ * and automatically downgrades through the strategy tiers until one works:
+ *
+ *   PUBLIC → COOKIE → HEADER → INTERCEPT → UI
+ *
+ * This eliminates the need for manual strategy selection — the system
+ * automatically finds the minimum-privilege strategy that works.
+ */
+
+import { Strategy } from './registry.js';
+
+/** Strategy cascade order (simplest → most complex) */
+const CASCADE_ORDER: Strategy[] = [
+  Strategy.PUBLIC,
+  Strategy.COOKIE,
+  Strategy.HEADER,
+  Strategy.INTERCEPT,
+  Strategy.UI,
+];
+
+interface ProbeResult {
+  strategy: Strategy;
+  success: boolean;
+  statusCode?: number;
+  hasData?: boolean;
+  error?: string;
+  responsePreview?: string;
+}
+
+interface CascadeResult {
+  bestStrategy: Strategy;
+  probes: ProbeResult[];
+  confidence: number;
+}
+
+/**
+ * Probe an endpoint with a specific strategy.
+ * Returns whether the probe succeeded and basic response info.
+ */
+export async function probeEndpoint(
+  page: any,
+  url: string,
+  strategy: Strategy,
+  opts: { timeout?: number } = {},
+): Promise<ProbeResult> {
+  const result: ProbeResult = { strategy, success: false };
+
+  try {
+    switch (strategy) {
+      case Strategy.PUBLIC: {
+        // Try direct fetch without browser (no credentials)
+        const js = `
+          async () => {
+            try {
+              const resp = await fetch(${JSON.stringify(url)});
+              const status = resp.status;
+              if (!resp.ok) return { status, ok: false };
+              const text = await resp.text();
+              let hasData = false;
+              try {
+                const json = JSON.parse(text);
+                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+                  typeof json === 'object' && Object.keys(json).length > 0);
+              } catch {}
+              return { status, ok: true, hasData, preview: text.slice(0, 200) };
+            } catch (e) { return { ok: false, error: e.message }; }
+          }
+        `;
+        const resp = await page.evaluate(js);
+        result.statusCode = resp?.status;
+        result.success = resp?.ok && resp?.hasData;
+        result.hasData = resp?.hasData;
+        result.responsePreview = resp?.preview;
+        break;
+      }
+
+      case Strategy.COOKIE: {
+        // Fetch with credentials: 'include' (uses browser cookies)
+        const js = `
+          async () => {
+            try {
+              const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+              const status = resp.status;
+              if (!resp.ok) return { status, ok: false };
+              const text = await resp.text();
+              let hasData = false;
+              try {
+                const json = JSON.parse(text);
+                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+                  typeof json === 'object' && Object.keys(json).length > 0);
+                // Check for API-level error codes (common in Chinese sites)
+                if (json.code !== undefined && json.code !== 0) hasData = false;
+              } catch {}
+              return { status, ok: true, hasData, preview: text.slice(0, 200) };
+            } catch (e) { return { ok: false, error: e.message }; }
+          }
+        `;
+        const resp = await page.evaluate(js);
+        result.statusCode = resp?.status;
+        result.success = resp?.ok && resp?.hasData;
+        result.hasData = resp?.hasData;
+        result.responsePreview = resp?.preview;
+        break;
+      }
+
+      case Strategy.HEADER: {
+        // Fetch with credentials + try to extract common auth headers
+        const js = `
+          async () => {
+            try {
+              // Try to extract CSRF tokens from cookies
+              const cookies = document.cookie.split(';').map(c => c.trim());
+              const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
+              
+              const headers = {};
+              if (csrf) {
+                headers['X-Csrf-Token'] = csrf;
+                headers['X-XSRF-Token'] = csrf;
+              }
+              
+              const resp = await fetch(${JSON.stringify(url)}, {
+                credentials: 'include',
+                headers
+              });
+              const status = resp.status;
+              if (!resp.ok) return { status, ok: false };
+              const text = await resp.text();
+              let hasData = false;
+              try {
+                const json = JSON.parse(text);
+                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+                  typeof json === 'object' && Object.keys(json).length > 0);
+                if (json.code !== undefined && json.code !== 0) hasData = false;
+              } catch {}
+              return { status, ok: true, hasData, preview: text.slice(0, 200) };
+            } catch (e) { return { ok: false, error: e.message }; }
+          }
+        `;
+        const resp = await page.evaluate(js);
+        result.statusCode = resp?.status;
+        result.success = resp?.ok && resp?.hasData;
+        result.hasData = resp?.hasData;
+        result.responsePreview = resp?.preview;
+        break;
+      }
+
+      case Strategy.INTERCEPT:
+      case Strategy.UI:
+        // These require specific implementation per-site
+        // Mark as needing manual implementation
+        result.success = false;
+        result.error = `Strategy ${strategy} requires site-specific implementation`;
+        break;
+    }
+  } catch (err: any) {
+    result.success = false;
+    result.error = err.message ?? String(err);
+  }
+
+  return result;
+}
+
+/**
+ * Run the cascade: try each strategy in order until one works.
+ * Returns the simplest working strategy.
+ */
+export async function cascadeProbe(
+  page: any,
+  url: string,
+  opts: { maxStrategy?: Strategy; timeout?: number } = {},
+): Promise<CascadeResult> {
+  const maxIdx = opts.maxStrategy
+    ? CASCADE_ORDER.indexOf(opts.maxStrategy)
+    : CASCADE_ORDER.indexOf(Strategy.HEADER); // Don't auto-try INTERCEPT/UI
+
+  const probes: ProbeResult[] = [];
+
+  for (let i = 0; i <= Math.min(maxIdx, CASCADE_ORDER.length - 1); i++) {
+    const strategy = CASCADE_ORDER[i];
+    const probe = await probeEndpoint(page, url, strategy, opts);
+    probes.push(probe);
+
+    if (probe.success) {
+      return {
+        bestStrategy: strategy,
+        probes,
+        confidence: 1.0 - (i * 0.1), // Higher confidence for simpler strategies
+      };
+    }
+  }
+
+  // None worked — default to COOKIE (most common for logged-in sites)
+  return {
+    bestStrategy: Strategy.COOKIE,
+    probes,
+    confidence: 0.3,
+  };
+}
+
+/**
+ * Render cascade results for display.
+ */
+export function renderCascadeResult(result: CascadeResult): string {
+  const lines = [
+    `Strategy Cascade: ${result.bestStrategy} (${(result.confidence * 100).toFixed(0)}% confidence)`,
+  ];
+  for (const probe of result.probes) {
+    const icon = probe.success ? '✅' : '❌';
+    const status = probe.statusCode ? ` [${probe.statusCode}]` : '';
+    const err = probe.error ? ` — ${probe.error}` : '';
+    lines.push(`  ${icon} ${probe.strategy}${status}${err}`);
+  }
+  return lines.join('\n');
+}

src/main.ts

@@ -58,6 +58,17 @@ program.command('synthesize').description('Synthesize CLIs from explore').argume
 program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
   .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: PlaywrightMCP, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; });
 
+program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('<url>').option('--site <name>')
+  .action(async (url, opts) => {
+    const { cascadeProbe, renderCascadeResult } = await import('./cascade.js');
+    const result = await browserSession(PlaywrightMCP, async (page) => {
+      // Navigate to the site first for cookie context
+      try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {}
+      return cascadeProbe(page, url);
+    });
+    console.log(renderCascadeResult(result));
+  });
+
 // ── Dynamic site commands ──────────────────────────────────────────────────
 
 const registry = getRegistry();

src/pipeline.ts

@@ -208,7 +208,85 @@ async function executeStep(page: any, op: string, params: any, data: any, args:
       if (!Array.isArray(data)) return data;
       return data.slice(0, Number(render(params, { args, data })));
     }
-    case 'intercept': return data;
+    case 'intercept': {
+      // Declarative XHR interception step
+      // Usage:
+      //   intercept:
+      //     trigger: "navigate:https://..." | "evaluate:store.note.fetch()" | "click:ref"
+      //     capture: "api/pattern"     # URL substring to match
+      //     timeout: 5                 # seconds to wait for matching request
+      //     select: "data.items"       # optional: extract sub-path from response
+      const cfg = typeof params === 'object' ? params : {};
+      const trigger = cfg.trigger ?? '';
+      const capturePattern = cfg.capture ?? '';
+      const timeout = cfg.timeout ?? 8;
+      const selectPath = cfg.select ?? null;
+
+      if (!capturePattern) return data;
+
+      // Step 1: Execute the trigger action
+      if (trigger.startsWith('navigate:')) {
+        const url = render(trigger.slice('navigate:'.length), { args, data });
+        await page.goto(String(url));
+      } else if (trigger.startsWith('evaluate:')) {
+        const js = trigger.slice('evaluate:'.length);
+        await page.evaluate(normalizeEvaluateSource(render(js, { args, data }) as string));
+      } else if (trigger.startsWith('click:')) {
+        const ref = render(trigger.slice('click:'.length), { args, data });
+        await page.click(String(ref).replace(/^@/, ''));
+      } else if (trigger === 'scroll') {
+        await page.scroll('down');
+      }
+
+      // Step 2: Wait a bit for network requests to fire
+      await page.wait(Math.min(timeout, 3));
+
+      // Step 3: Get network requests and find matching ones
+      const rawNetwork = await page.networkRequests(false);
+      const matchingResponses: any[] = [];
+
+      if (typeof rawNetwork === 'string') {
+        // Parse the network output to find matching URLs
+        const lines = rawNetwork.split('\n');
+        for (const line of lines) {
+          const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
+          if (match) {
+            const [, method, url, status] = match;
+            if (url.includes(capturePattern) && status === '200') {
+              // Re-fetch the matching URL to get the response body
+              try {
+                const body = await page.evaluate(`
+                  async () => {
+                    try {
+                      const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+                      if (!resp.ok) return null;
+                      return await resp.json();
+                    } catch { return null; }
+                  }
+                `);
+                if (body) matchingResponses.push(body);
+              } catch {}
+            }
+          }
+        }
+      }
+
+      // Step 4: Select from response if specified
+      let result = matchingResponses.length === 1 ? matchingResponses[0] :
+                   matchingResponses.length > 1 ? matchingResponses : data;
+
+      if (selectPath && result) {
+        let current = result;
+        for (const part of String(selectPath).split('.')) {
+          if (current && typeof current === 'object' && !Array.isArray(current)) {
+            current = current[part];
+          } else break;
+        }
+        result = current ?? result;
+      }
+
+      return result;
+    }
     default: return data;
   }
 }