From c019f8c6f2ad190cf6ad8454edddf34da70d13e8 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sun, 15 Mar 2026 06:45:05 +0800 Subject: [PATCH] feat: add cascade strategy probe, intercept pipeline step, buildTemplatedUrl --- src/cascade.ts | 217 ++++++++++++++++++++++++++++++++++++++++++++++++ src/main.ts | 11 +++ src/pipeline.ts | 80 +++++++++++++++++- 3 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 src/cascade.ts diff --git a/src/cascade.ts b/src/cascade.ts new file mode 100644 index 0000000..bb523b1 --- /dev/null +++ b/src/cascade.ts @@ -0,0 +1,217 @@ +/** + * Strategy Cascade: automatic strategy downgrade chain. + * + * Probes an API endpoint starting from the simplest strategy (PUBLIC) + * and automatically downgrades through the strategy tiers until one works: + * + * PUBLIC → COOKIE → HEADER → INTERCEPT → UI + * + * This eliminates the need for manual strategy selection — the system + * automatically finds the minimum-privilege strategy that works. + */ + +import { Strategy } from './registry.js'; + +/** Strategy cascade order (simplest → most complex) */ +const CASCADE_ORDER: Strategy[] = [ + Strategy.PUBLIC, + Strategy.COOKIE, + Strategy.HEADER, + Strategy.INTERCEPT, + Strategy.UI, +]; + +interface ProbeResult { + strategy: Strategy; + success: boolean; + statusCode?: number; + hasData?: boolean; + error?: string; + responsePreview?: string; +} + +interface CascadeResult { + bestStrategy: Strategy; + probes: ProbeResult[]; + confidence: number; +} + +/** + * Probe an endpoint with a specific strategy. + * Returns whether the probe succeeded and basic response info. + */ +export async function probeEndpoint( + page: any, + url: string, + strategy: Strategy, + opts: { timeout?: number } = {}, +): Promise { + const result: ProbeResult = { strategy, success: false }; + + try { + switch (strategy) { + case Strategy.PUBLIC: { + // Try direct fetch without browser (no credentials) + const js = ` + async () => { + try { + const resp = await fetch(${JSON.stringify(url)}); + const status = resp.status; + if (!resp.ok) return { status, ok: false }; + const text = await resp.text(); + let hasData = false; + try { + const json = JSON.parse(text); + hasData = !!json && (Array.isArray(json) ? json.length > 0 : + typeof json === 'object' && Object.keys(json).length > 0); + } catch {} + return { status, ok: true, hasData, preview: text.slice(0, 200) }; + } catch (e) { return { ok: false, error: e.message }; } + } + `; + const resp = await page.evaluate(js); + result.statusCode = resp?.status; + result.success = resp?.ok && resp?.hasData; + result.hasData = resp?.hasData; + result.responsePreview = resp?.preview; + break; + } + + case Strategy.COOKIE: { + // Fetch with credentials: 'include' (uses browser cookies) + const js = ` + async () => { + try { + const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' }); + const status = resp.status; + if (!resp.ok) return { status, ok: false }; + const text = await resp.text(); + let hasData = false; + try { + const json = JSON.parse(text); + hasData = !!json && (Array.isArray(json) ? json.length > 0 : + typeof json === 'object' && Object.keys(json).length > 0); + // Check for API-level error codes (common in Chinese sites) + if (json.code !== undefined && json.code !== 0) hasData = false; + } catch {} + return { status, ok: true, hasData, preview: text.slice(0, 200) }; + } catch (e) { return { ok: false, error: e.message }; } + } + `; + const resp = await page.evaluate(js); + result.statusCode = resp?.status; + result.success = resp?.ok && resp?.hasData; + result.hasData = resp?.hasData; + result.responsePreview = resp?.preview; + break; + } + + case Strategy.HEADER: { + // Fetch with credentials + try to extract common auth headers + const js = ` + async () => { + try { + // Try to extract CSRF tokens from cookies + const cookies = document.cookie.split(';').map(c => c.trim()); + const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('='); + + const headers = {}; + if (csrf) { + headers['X-Csrf-Token'] = csrf; + headers['X-XSRF-Token'] = csrf; + } + + const resp = await fetch(${JSON.stringify(url)}, { + credentials: 'include', + headers + }); + const status = resp.status; + if (!resp.ok) return { status, ok: false }; + const text = await resp.text(); + let hasData = false; + try { + const json = JSON.parse(text); + hasData = !!json && (Array.isArray(json) ? json.length > 0 : + typeof json === 'object' && Object.keys(json).length > 0); + if (json.code !== undefined && json.code !== 0) hasData = false; + } catch {} + return { status, ok: true, hasData, preview: text.slice(0, 200) }; + } catch (e) { return { ok: false, error: e.message }; } + } + `; + const resp = await page.evaluate(js); + result.statusCode = resp?.status; + result.success = resp?.ok && resp?.hasData; + result.hasData = resp?.hasData; + result.responsePreview = resp?.preview; + break; + } + + case Strategy.INTERCEPT: + case Strategy.UI: + // These require specific implementation per-site + // Mark as needing manual implementation + result.success = false; + result.error = `Strategy ${strategy} requires site-specific implementation`; + break; + } + } catch (err: any) { + result.success = false; + result.error = err.message ?? String(err); + } + + return result; +} + +/** + * Run the cascade: try each strategy in order until one works. + * Returns the simplest working strategy. + */ +export async function cascadeProbe( + page: any, + url: string, + opts: { maxStrategy?: Strategy; timeout?: number } = {}, +): Promise { + const maxIdx = opts.maxStrategy + ? CASCADE_ORDER.indexOf(opts.maxStrategy) + : CASCADE_ORDER.indexOf(Strategy.HEADER); // Don't auto-try INTERCEPT/UI + + const probes: ProbeResult[] = []; + + for (let i = 0; i <= Math.min(maxIdx, CASCADE_ORDER.length - 1); i++) { + const strategy = CASCADE_ORDER[i]; + const probe = await probeEndpoint(page, url, strategy, opts); + probes.push(probe); + + if (probe.success) { + return { + bestStrategy: strategy, + probes, + confidence: 1.0 - (i * 0.1), // Higher confidence for simpler strategies + }; + } + } + + // None worked — default to COOKIE (most common for logged-in sites) + return { + bestStrategy: Strategy.COOKIE, + probes, + confidence: 0.3, + }; +} + +/** + * Render cascade results for display. + */ +export function renderCascadeResult(result: CascadeResult): string { + const lines = [ + `Strategy Cascade: ${result.bestStrategy} (${(result.confidence * 100).toFixed(0)}% confidence)`, + ]; + for (const probe of result.probes) { + const icon = probe.success ? '✅' : '❌'; + const status = probe.statusCode ? ` [${probe.statusCode}]` : ''; + const err = probe.error ? ` — ${probe.error}` : ''; + lines.push(` ${icon} ${probe.strategy}${status}${err}`); + } + return lines.join('\n'); +} diff --git a/src/main.ts b/src/main.ts index 19e86e5..4ab6f69 100644 --- a/src/main.ts +++ b/src/main.ts @@ -58,6 +58,17 @@ program.command('synthesize').description('Synthesize CLIs from explore').argume program.command('generate').description('One-shot: explore → synthesize → register').argument('').option('--goal ').option('--site ') .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: PlaywrightMCP, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; }); +program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('').option('--site ') + .action(async (url, opts) => { + const { cascadeProbe, renderCascadeResult } = await import('./cascade.js'); + const result = await browserSession(PlaywrightMCP, async (page) => { + // Navigate to the site first for cookie context + try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {} + return cascadeProbe(page, url); + }); + console.log(renderCascadeResult(result)); + }); + // ── Dynamic site commands ────────────────────────────────────────────────── const registry = getRegistry(); diff --git a/src/pipeline.ts b/src/pipeline.ts index 4018870..810570d 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -208,7 +208,85 @@ async function executeStep(page: any, op: string, params: any, data: any, args: if (!Array.isArray(data)) return data; return data.slice(0, Number(render(params, { args, data }))); } - case 'intercept': return data; + case 'intercept': { + // Declarative XHR interception step + // Usage: + // intercept: + // trigger: "navigate:https://..." | "evaluate:store.note.fetch()" | "click:ref" + // capture: "api/pattern" # URL substring to match + // timeout: 5 # seconds to wait for matching request + // select: "data.items" # optional: extract sub-path from response + const cfg = typeof params === 'object' ? params : {}; + const trigger = cfg.trigger ?? ''; + const capturePattern = cfg.capture ?? ''; + const timeout = cfg.timeout ?? 8; + const selectPath = cfg.select ?? null; + + if (!capturePattern) return data; + + // Step 1: Execute the trigger action + if (trigger.startsWith('navigate:')) { + const url = render(trigger.slice('navigate:'.length), { args, data }); + await page.goto(String(url)); + } else if (trigger.startsWith('evaluate:')) { + const js = trigger.slice('evaluate:'.length); + await page.evaluate(normalizeEvaluateSource(render(js, { args, data }) as string)); + } else if (trigger.startsWith('click:')) { + const ref = render(trigger.slice('click:'.length), { args, data }); + await page.click(String(ref).replace(/^@/, '')); + } else if (trigger === 'scroll') { + await page.scroll('down'); + } + + // Step 2: Wait a bit for network requests to fire + await page.wait(Math.min(timeout, 3)); + + // Step 3: Get network requests and find matching ones + const rawNetwork = await page.networkRequests(false); + const matchingResponses: any[] = []; + + if (typeof rawNetwork === 'string') { + // Parse the network output to find matching URLs + const lines = rawNetwork.split('\n'); + for (const line of lines) { + const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i); + if (match) { + const [, method, url, status] = match; + if (url.includes(capturePattern) && status === '200') { + // Re-fetch the matching URL to get the response body + try { + const body = await page.evaluate(` + async () => { + try { + const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' }); + if (!resp.ok) return null; + return await resp.json(); + } catch { return null; } + } + `); + if (body) matchingResponses.push(body); + } catch {} + } + } + } + } + + // Step 4: Select from response if specified + let result = matchingResponses.length === 1 ? matchingResponses[0] : + matchingResponses.length > 1 ? matchingResponses : data; + + if (selectPath && result) { + let current = result; + for (const part of String(selectPath).split('.')) { + if (current && typeof current === 'object' && !Array.isArray(current)) { + current = current[part]; + } else break; + } + result = current ?? result; + } + + return result; + } default: return data; } }