diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index dcc6e0c..d193ba5 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -4,6 +4,7 @@ import { WebSocket } from 'ws'; import type { IPage } from '../types.js'; +import { wrapForEval } from './utils.js'; export interface CDPTarget { type?: string; @@ -218,16 +219,6 @@ class CDPPage implements IPage { throw new Error('Method not implemented.'); } } - -function wrapForEval(js: string): string { - const code = js.trim(); - if (!code) return 'undefined'; - if (/^\([\s\S]*\)\s*\(.*\)\s*$/.test(code)) return code; - if (/^(async\s+)?(\([^)]*\)|[A-Za-z_]\w*)\s*=>/.test(code)) return `(${code})()`; - if (/^(async\s+)?function[\s(]/.test(code)) return `(${code})()`; - return code; -} - function selectCDPTarget(targets: CDPTarget[]): CDPTarget | undefined { const preferredPattern = compilePreferredPattern(process.env.OPENCLI_CDP_TARGET); diff --git a/src/browser/page.ts b/src/browser/page.ts index e3bff9c..700588f 100644 --- a/src/browser/page.ts +++ b/src/browser/page.ts @@ -13,6 +13,7 @@ import { formatSnapshot } from '../snapshotFormatter.js'; import type { IPage } from '../types.js'; import { sendCommand } from './daemon-client.js'; +import { wrapForEval } from './utils.js'; /** * Page — implements IPage by talking to the daemon via HTTP. @@ -285,27 +286,4 @@ export class Page implements IPage { } } -// ─── Helpers ───────────────────────────────────────────────────────── - -/** - * Wrap JS code for CDP Runtime.evaluate: - * - Already an IIFE `(...)()` → send as-is - * - Arrow/function literal → wrap as IIFE `(code)()` - * - `new Promise(...)` or raw expression → send as-is (expression) - */ -function wrapForEval(js: string): string { - const code = js.trim(); - if (!code) return 'undefined'; - - // Already an IIFE: `(async () => { ... })()` or `(function() {...})()` - if (/^\([\s\S]*\)\s*\(.*\)\s*$/.test(code)) return code; - - // Arrow function: `() => ...` or `async () => ...` - if (/^(async\s+)?(\([^)]*\)|[A-Za-z_]\w*)\s*=>/.test(code)) return `(${code})()`; - - // Function declaration: `function ...` or `async function ...` - if (/^(async\s+)?function[\s(]/.test(code)) return `(${code})()`; - - // Everything else: bare expression, `new Promise(...)`, etc. → evaluate directly - return code; -} +// (End of file) diff --git a/src/browser/utils.ts b/src/browser/utils.ts new file mode 100644 index 0000000..4001796 --- /dev/null +++ b/src/browser/utils.ts @@ -0,0 +1,27 @@ +/** + * Utility functions for browser operations + */ + +/** + * Wrap JS code for CDP Runtime.evaluate: + * - Already an IIFE `(...)()` → send as-is + * - Arrow/function literal → wrap as IIFE `(code)()` + * - `new Promise(...)` or raw expression → send as-is (expression) + */ +export function wrapForEval(js: string): string { + if (typeof js !== 'string') return 'undefined'; + const code = js.trim(); + if (!code) return 'undefined'; + + // Already an IIFE: `(async () => { ... })()` or `(function() {...})()` + if (/^\([\s\S]*\)\s*\(.*\)\s*$/.test(code)) return code; + + // Arrow function: `() => ...` or `async () => ...` + if (/^(async\s+)?(\([^)]*\)|[A-Za-z_]\w*)\s*=>/.test(code)) return `(${code})()`; + + // Function declaration: `function ...` or `async function ...` + if (/^(async\s+)?function[\s(]/.test(code)) return `(${code})()`; + + // Everything else: bare expression, `new Promise(...)`, etc. → evaluate directly + return code; +} diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..92ee5fd --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,185 @@ +import { Command } from 'commander'; +import chalk from 'chalk'; +import { executeCommand } from './engine.js'; +import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js'; +import { render as renderOutput } from './output.js'; +import { BrowserBridge, CDPBridge } from './browser/index.js'; +import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js'; +import { PKG_VERSION } from './version.js'; +import { printCompletionScript } from './completion.js'; +import { CliError } from './errors.js'; + +export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void { + const program = new Command(); + program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION); + + // ── Built-in commands ────────────────────────────────────────────────────── + + program.command('list').description('List all available CLI commands').option('-f, --format ', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)') + .action((opts) => { + const registry = getRegistry(); + const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b))); + const rows = commands.map(c => ({ + command: fullName(c), + site: c.site, + name: c.name, + description: c.description, + strategy: strategyLabel(c), + browser: c.browser, + args: c.args.map(a => a.name).join(', '), + })); + const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format; + if (fmt !== 'table') { + renderOutput(rows, { + fmt, + columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args'], + title: 'opencli/list', + source: 'opencli list', + }); + return; + } + const sites = new Map(); + for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); } + console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log(); + for (const [site, cmds] of sites) { + console.log(chalk.bold.cyan(` ${site}`)); + for (const cmd of cmds) { const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`); console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(` — ${cmd.description}`) : ''}`); } + console.log(); + } + console.log(chalk.dim(` ${commands.length} commands across ${sites.size} sites`)); console.log(); + }); + + program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name') + .action(async (target) => { + const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); + console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); + }); + + program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false) + .action(async (target, opts) => { + const { verifyClis, renderVerifyReport } = await import('./verify.js'); + const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); + console.log(renderVerifyReport(r)); + process.exitCode = r.ok ? 0 : 1; + }); + + program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('').option('--site ').option('--goal ').option('--wait ', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click ', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")') + .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserFactory as any, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); }); + + program.command('synthesize').description('Synthesize CLIs from explore').argument('').option('--top ', '', '3') + .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); }); + + program.command('generate').description('One-shot: explore → synthesize → register').argument('').option('--goal ').option('--site ') + .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; const r = await generateCliFromUrl({ url, BrowserFactory: BrowserFactory as any, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; }); + + program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('').option('--site ') + .action(async (url, opts) => { + const { cascadeProbe, renderCascadeResult } = await import('./cascade.js'); + const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; + const result = await browserSession(BrowserFactory as any, async (page) => { + // Navigate to the site first for cookie context + try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {} + return cascadeProbe(page, url); + }); + console.log(renderCascadeResult(result)); + }); + + program.command('doctor') + .description('Diagnose opencli browser bridge connectivity') + .option('--live', 'Test browser connectivity (requires Chrome running)', false) + .action(async (opts) => { + const { runBrowserDoctor, renderBrowserDoctorReport } = await import('./doctor.js'); + const report = await runBrowserDoctor({ live: opts.live, cliVersion: PKG_VERSION }); + console.log(renderBrowserDoctorReport(report)); + }); + + program.command('setup') + .description('Interactive setup: verify browser bridge connectivity') + .action(async () => { + const { runSetup } = await import('./setup.js'); + await runSetup({ cliVersion: PKG_VERSION }); + }); + + program.command('completion') + .description('Output shell completion script') + .argument('', 'Shell type: bash, zsh, or fish') + .action((shell) => { + printCompletionScript(shell); + }); + + // ── Dynamic site commands ────────────────────────────────────────────────── + + const registry = getRegistry(); + const siteGroups = new Map(); + + for (const [, cmd] of registry) { + let siteCmd = siteGroups.get(cmd.site); + if (!siteCmd) { siteCmd = program.command(cmd.site).description(`${cmd.site} commands`); siteGroups.set(cmd.site, siteCmd); } + const subCmd = siteCmd.command(cmd.name).description(cmd.description); + + // Register positional args first, then named options + const positionalArgs: typeof cmd.args = []; + for (const arg of cmd.args) { + if (arg.positional) { + const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`; + subCmd.argument(bracket, arg.help ?? ''); + positionalArgs.push(arg); + } else { + const flag = arg.required ? `--${arg.name} ` : `--${arg.name} [value]`; + if (arg.required) subCmd.requiredOption(flag, arg.help ?? ''); + else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default)); + else subCmd.option(flag, arg.help ?? ''); + } + } + subCmd.option('-f, --format ', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false); + + subCmd.action(async (...actionArgs: any[]) => { + // Commander passes positional args first, then options object, then the Command + const actionOpts = actionArgs[positionalArgs.length] ?? {}; + const startTime = Date.now(); + const kwargs: Record = {}; + + // Collect positional args + for (let i = 0; i < positionalArgs.length; i++) { + const arg = positionalArgs[i]; + const v = actionArgs[i]; + if (v !== undefined) kwargs[arg.name] = v; + } + + // Collect named options + for (const arg of cmd.args) { + if (arg.positional) continue; + const camelName = arg.name.replace(/-([a-z])/g, (_m, ch: string) => ch.toUpperCase()); + const v = actionOpts[arg.name] ?? actionOpts[camelName]; + if (v !== undefined) kwargs[arg.name] = v; + } + + try { + if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1'; + let result: any; + if (cmd.browser) { + const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; + result = await browserSession(BrowserFactory as any, async (page) => { + return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }); + }); + } else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); } + if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) { + console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`)); + } + renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) }); + } catch (err: any) { + if (err instanceof CliError) { + console.error(chalk.red(`Error [${err.code}]: ${err.message}`)); + if (err.hint) console.error(chalk.yellow(`Hint: ${err.hint}`)); + } else if (actionOpts.verbose && err.stack) { + console.error(chalk.red(err.stack)); + } else { + console.error(chalk.red(`Error: ${err.message ?? err}`)); + } + process.exitCode = 1; + } + }); + } + + program.parse(); +} diff --git a/src/engine.ts b/src/engine.ts index 6b78d0b..1a46bce 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -28,12 +28,14 @@ export async function discoverClis(...dirs: string[]): Promise { // Fast path: try manifest first (production / post-build) for (const dir of dirs) { const manifestPath = path.resolve(dir, '..', 'cli-manifest.json'); - if (fs.existsSync(manifestPath)) { - loadFromManifest(manifestPath, dir); + try { + await fs.promises.access(manifestPath); + await loadFromManifest(manifestPath, dir); continue; // Skip filesystem scan for this directory + } catch { + // Fallback: runtime filesystem scan (development) + await discoverClisFromFs(dir); } - // Fallback: runtime filesystem scan (development) - await discoverClisFromFs(dir); } } @@ -42,9 +44,10 @@ export async function discoverClis(...dirs: string[]): Promise { * YAML pipelines are inlined — zero YAML parsing at runtime. * TS modules are deferred — loaded lazily on first execution. */ -function loadFromManifest(manifestPath: string, clisDir: string): void { +async function loadFromManifest(manifestPath: string, clisDir: string): Promise { try { - const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as any[]; + const raw = await fs.promises.readFile(manifestPath, 'utf-8'); + const manifest = JSON.parse(raw) as any[]; for (const entry of manifest) { if (entry.type === 'yaml') { // YAML pipelines fully inlined in manifest — register directly @@ -94,15 +97,19 @@ function loadFromManifest(manifestPath: string, clisDir: string): void { * Fallback: traditional filesystem scan (used during development with tsx). */ async function discoverClisFromFs(dir: string): Promise { - if (!fs.existsSync(dir)) return; + try { await fs.promises.access(dir); } catch { return; } const promises: Promise[] = []; - for (const site of fs.readdirSync(dir)) { + const sites = await fs.promises.readdir(dir); + + for (const site of sites) { const siteDir = path.join(dir, site); - if (!fs.statSync(siteDir).isDirectory()) continue; - for (const file of fs.readdirSync(siteDir)) { + const stat = await fs.promises.stat(siteDir); + if (!stat.isDirectory()) continue; + const files = await fs.promises.readdir(siteDir); + for (const file of files) { const filePath = path.join(siteDir, file); if (file.endsWith('.yaml') || file.endsWith('.yml')) { - registerYamlCli(filePath, site); + promises.push(registerYamlCli(filePath, site)); } else if ( (file.endsWith('.js') && !file.endsWith('.d.js')) || (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts')) @@ -118,9 +125,9 @@ async function discoverClisFromFs(dir: string): Promise { await Promise.all(promises); } -function registerYamlCli(filePath: string, defaultSite: string): void { +async function registerYamlCli(filePath: string, defaultSite: string): Promise { try { - const raw = fs.readFileSync(filePath, 'utf-8'); + const raw = await fs.promises.readFile(filePath, 'utf-8'); const def = yaml.load(raw) as any; if (!def || typeof def !== 'object') return; diff --git a/src/explore.ts b/src/explore.ts index 1eaa0fc..d26a803 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -10,6 +10,9 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js'; import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js'; +import { detectFramework } from './scripts/framework.js'; +import { discoverStores } from './scripts/store.js'; +import { interactFuzz } from './scripts/interact.js'; // ── Site name detection ──────────────────────────────────────────────────── @@ -201,63 +204,11 @@ function inferStrategy(authIndicators: string[]): string { // ── Framework detection ──────────────────────────────────────────────────── -const FRAMEWORK_DETECT_JS = ` - () => { - const r = {}; - try { - const app = document.querySelector('#app'); - r.vue3 = !!(app && app.__vue_app__); - r.vue2 = !!(app && app.__vue__); - r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]'); - r.nextjs = !!window.__NEXT_DATA__; - r.nuxt = !!window.__NUXT__; - if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); } - } catch {} - return r; - } -`; +const FRAMEWORK_DETECT_JS = detectFramework.toString(); // ── Store discovery ──────────────────────────────────────────────────────── -const STORE_DISCOVER_JS = ` - () => { - const stores = []; - try { - const app = document.querySelector('#app'); - if (!app?.__vue_app__) return stores; - const gp = app.__vue_app__.config?.globalProperties; - - // Pinia stores - const pinia = gp?.$pinia; - if (pinia?._s) { - pinia._s.forEach((store, id) => { - const actions = []; - const stateKeys = []; - for (const k in store) { - try { - if (k.startsWith('$') || k.startsWith('_')) continue; - if (typeof store[k] === 'function') actions.push(k); - else stateKeys.push(k); - } catch {} - } - stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) }); - }); - } - - // Vuex store modules - const vuex = gp?.$store; - if (vuex?._modules?.root?._children) { - const children = vuex._modules.root._children; - for (const [modName, mod] of Object.entries(children)) { - const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20); - const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15); - stores.push({ type: 'vuex', id: modName, actions, stateKeys }); - } - } - } catch {} - return stores; - } -`; +const STORE_DISCOVER_JS = discoverStores.toString(); export interface DiscoveredStore { type: 'pinia' | 'vuex'; @@ -268,27 +219,7 @@ export interface DiscoveredStore { // ── Auto-Interaction (Fuzzing) ───────────────────────────────────────────── -const INTERACT_FUZZ_JS = ` - async () => { - const sleep = ms => new Promise(r => setTimeout(r, ms)); - const clickables = Array.from(document.querySelectorAll( - 'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]' - )).slice(0, 15); // limit to 15 to avoid endless loops - - let clicked = 0; - for (const el of clickables) { - try { - const rect = el.getBoundingClientRect(); - if (rect.width > 0 && rect.height > 0) { - el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); - clicked++; - await sleep(300); // give it time to trigger network - } - } catch {} - } - return clicked; - } -`; +const INTERACT_FUZZ_JS = interactFuzz.toString(); // ── Main explore function ────────────────────────────────────────────────── @@ -310,8 +241,8 @@ export async function exploreUrl( await page.goto(url); await page.wait(waitSeconds); - // Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist) - for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); } + // Step 2: Auto-scroll to trigger lazy loading intelligently + await page.autoScroll({ times: 3, delayMs: 1500 }).catch(() => {}); // Step 2.5: Interactive Fuzzing (if requested) if (opts.auto) { @@ -345,11 +276,27 @@ export async function exploreUrl( const rawNetwork = await page.networkRequests(false); const networkEntries = parseNetworkRequests(rawNetwork); - // Step 5: For JSON endpoints, re-fetch response body in-browser - const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200); - for (const ep of jsonEndpoints.slice(0, 10)) { + // Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe + const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody); + for (const ep of jsonEndpoints.slice(0, 5)) { try { - const body = await page.evaluate(`async () => { try { const r = await fetch(${JSON.stringify(ep.url)}, {credentials:'include'}); if (!r.ok) return null; const d = await r.json(); return JSON.stringify(d).slice(0,10000); } catch { return null; } }`); + const body = await page.evaluate(`async () => { + let iframe = null; + try { + iframe = document.createElement('iframe'); + iframe.style.display = 'none'; + document.body.appendChild(iframe); + const cleanFetch = iframe.contentWindow.fetch || window.fetch; + const r = await cleanFetch(${JSON.stringify(ep.url)}, { credentials: 'include' }); + if (!r.ok) return null; + const d = await r.json(); + return JSON.stringify(d).slice(0, 10000); + } catch { + return null; + } finally { + if (iframe && iframe.parentNode) iframe.parentNode.removeChild(iframe); + } + }`); if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} } else if (body && typeof body === 'object') ep.responseBody = body; } catch {} @@ -455,7 +402,7 @@ export async function exploreUrl( const siteName = opts.site ?? detectSiteName(metadata.url || url); const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName); - fs.mkdirSync(targetDir, { recursive: true }); + await fs.promises.mkdir(targetDir, { recursive: true }); const result = { site: siteName, target_url: url, final_url: metadata.url, title: metadata.title, @@ -466,24 +413,26 @@ export async function exploreUrl( }; // Write artifacts - fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify({ + const writeTasks = []; + writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({ site: siteName, target_url: url, final_url: metadata.url, title: metadata.title, framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })), top_strategy: topStrategy, explored_at: new Date().toISOString(), - }, null, 2)); - fs.writeFileSync(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({ + }, null, 2))); + writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({ pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status, contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams, itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0, detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators, - })), null, 2)); - fs.writeFileSync(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)); - fs.writeFileSync(path.join(targetDir, 'auth.json'), JSON.stringify({ + })), null, 2))); + writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2))); + writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({ top_strategy: topStrategy, indicators: [...allAuth], framework, - }, null, 2)); + }, null, 2))); if (stores.length > 0) { - fs.writeFileSync(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)); + writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2))); } + await Promise.all(writeTasks); return { ...result, out_dir: targetDir }; })(), { timeout: exploreTimeout, label: `Explore ${url}` }); diff --git a/src/main.ts b/src/main.ts index d7d3306..d4c5e58 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,16 +6,9 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { Command } from 'commander'; -import chalk from 'chalk'; -import { discoverClis, executeCommand } from './engine.js'; -import { Strategy, type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js'; -import { render as renderOutput } from './output.js'; -import { BrowserBridge, CDPBridge } from './browser/index.js'; -import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js'; -import { PKG_VERSION } from './version.js'; -import { getCompletions, printCompletionScript } from './completion.js'; -import { CliError } from './errors.js'; +import { discoverClis } from './engine.js'; +import { getCompletions } from './completion.js'; +import { runCli } from './cli.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -45,175 +38,4 @@ if (getCompIdx !== -1) { process.exit(0); } -const program = new Command(); -program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION); - -// ── Built-in commands ────────────────────────────────────────────────────── - -program.command('list').description('List all available CLI commands').option('-f, --format ', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)') - .action((opts) => { - const registry = getRegistry(); - const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b))); - const rows = commands.map(c => ({ - command: fullName(c), - site: c.site, - name: c.name, - description: c.description, - strategy: strategyLabel(c), - browser: c.browser, - args: c.args.map(a => a.name).join(', '), - })); - const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format; - if (fmt !== 'table') { - renderOutput(rows, { - fmt, - columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args'], - title: 'opencli/list', - source: 'opencli list', - }); - return; - } - const sites = new Map(); - for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); } - console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log(); - for (const [site, cmds] of sites) { - console.log(chalk.bold.cyan(` ${site}`)); - for (const cmd of cmds) { const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`); console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(` — ${cmd.description}`) : ''}`); } - console.log(); - } - console.log(chalk.dim(` ${commands.length} commands across ${sites.size} sites`)); console.log(); - }); - -program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name') - .action(async (target) => { - const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); - console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); - }); - -program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false) - .action(async (target, opts) => { - const { verifyClis, renderVerifyReport } = await import('./verify.js'); - const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); - console.log(renderVerifyReport(r)); - process.exitCode = r.ok ? 0 : 1; - }); - -program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('').option('--site ').option('--goal ').option('--wait ', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click ', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")') - .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserFactory as any, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); }); - -program.command('synthesize').description('Synthesize CLIs from explore').argument('').option('--top ', '', '3') - .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); }); - -program.command('generate').description('One-shot: explore → synthesize → register').argument('').option('--goal ').option('--site ') - .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; const r = await generateCliFromUrl({ url, BrowserFactory: BrowserFactory as any, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; }); - -program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('').option('--site ') - .action(async (url, opts) => { - const { cascadeProbe, renderCascadeResult } = await import('./cascade.js'); - const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; - const result = await browserSession(BrowserFactory as any, async (page) => { - // Navigate to the site first for cookie context - try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {} - return cascadeProbe(page, url); - }); - console.log(renderCascadeResult(result)); - }); - -program.command('doctor') - .description('Diagnose opencli browser bridge connectivity') - .option('--live', 'Test browser connectivity (requires Chrome running)', false) - .action(async (opts) => { - const { runBrowserDoctor, renderBrowserDoctorReport } = await import('./doctor.js'); - const report = await runBrowserDoctor({ live: opts.live, cliVersion: PKG_VERSION }); - console.log(renderBrowserDoctorReport(report)); - }); - -program.command('setup') - .description('Interactive setup: verify browser bridge connectivity') - .action(async () => { - const { runSetup } = await import('./setup.js'); - await runSetup({ cliVersion: PKG_VERSION }); - }); - -program.command('completion') - .description('Output shell completion script') - .argument('', 'Shell type: bash, zsh, or fish') - .action((shell) => { - printCompletionScript(shell); - }); - -// ── Dynamic site commands ────────────────────────────────────────────────── - -const registry = getRegistry(); -const siteGroups = new Map(); - -for (const [, cmd] of registry) { - let siteCmd = siteGroups.get(cmd.site); - if (!siteCmd) { siteCmd = program.command(cmd.site).description(`${cmd.site} commands`); siteGroups.set(cmd.site, siteCmd); } - const subCmd = siteCmd.command(cmd.name).description(cmd.description); - - // Register positional args first, then named options - const positionalArgs: typeof cmd.args = []; - for (const arg of cmd.args) { - if (arg.positional) { - const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`; - subCmd.argument(bracket, arg.help ?? ''); - positionalArgs.push(arg); - } else { - const flag = arg.required ? `--${arg.name} ` : `--${arg.name} [value]`; - if (arg.required) subCmd.requiredOption(flag, arg.help ?? ''); - else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default)); - else subCmd.option(flag, arg.help ?? ''); - } - } - subCmd.option('-f, --format ', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false); - - subCmd.action(async (...actionArgs: any[]) => { - // Commander passes positional args first, then options object, then the Command - const actionOpts = actionArgs[positionalArgs.length] ?? {}; - const startTime = Date.now(); - const kwargs: Record = {}; - - // Collect positional args - for (let i = 0; i < positionalArgs.length; i++) { - const arg = positionalArgs[i]; - const v = actionArgs[i]; - if (v !== undefined) kwargs[arg.name] = v; - } - - // Collect named options - for (const arg of cmd.args) { - if (arg.positional) continue; - const camelName = arg.name.replace(/-([a-z])/g, (_m, ch: string) => ch.toUpperCase()); - const v = actionOpts[arg.name] ?? actionOpts[camelName]; - if (v !== undefined) kwargs[arg.name] = v; - } - - try { - if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1'; - let result: any; - if (cmd.browser) { - const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; - result = await browserSession(BrowserFactory as any, async (page) => { - return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }); - }); - } else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); } - if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) { - console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`)); - } - renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) }); - } catch (err: any) { - if (err instanceof CliError) { - console.error(chalk.red(`Error [${err.code}]: ${err.message}`)); - if (err.hint) console.error(chalk.yellow(`Hint: ${err.hint}`)); - } else if (actionOpts.verbose && err.stack) { - console.error(chalk.red(err.stack)); - } else { - console.error(chalk.red(`Error: ${err.message ?? err}`)); - } - process.exitCode = 1; - } - }); -} - -program.parse(); +runCli(BUILTIN_CLIS, USER_CLIS); diff --git a/src/scripts/framework.ts b/src/scripts/framework.ts new file mode 100644 index 0000000..ad70c6e --- /dev/null +++ b/src/scripts/framework.ts @@ -0,0 +1,20 @@ +/** + * Injected script for detecting frontend frameworks (Vue, React, Next, Nuxt, etc.) + */ +export function detectFramework() { + const r: Record = {}; + try { + const app = document.querySelector('#app') as any; + r.vue3 = !!(app && app.__vue_app__); + r.vue2 = !!(app && app.__vue__); + r.react = !!(window as any).__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]'); + r.nextjs = !!(window as any).__NEXT_DATA__; + r.nuxt = !!(window as any).__NUXT__; + if (r.vue3 && app.__vue_app__) { + const gp = app.__vue_app__.config?.globalProperties; + r.pinia = !!(gp && gp.$pinia); + r.vuex = !!(gp && gp.$store); + } + } catch {} + return r; +} diff --git a/src/scripts/interact.ts b/src/scripts/interact.ts new file mode 100644 index 0000000..852d69d --- /dev/null +++ b/src/scripts/interact.ts @@ -0,0 +1,22 @@ +/** + * Injected script for interactive fuzzing (clicking elements to trigger lazy loading) + */ +export async function interactFuzz() { + const sleep = (ms: number) => new Promise(r => setTimeout(r, ms)); + const clickables = Array.from(document.querySelectorAll( + 'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]' + )).slice(0, 15); // limit to a small number to avoid endless loops + + let clicked = 0; + for (const el of clickables) { + try { + const rect = el.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); + clicked++; + await sleep(300); // give it time to trigger network + } + } catch {} + } + return clicked; +} diff --git a/src/scripts/store.ts b/src/scripts/store.ts new file mode 100644 index 0000000..996aad7 --- /dev/null +++ b/src/scripts/store.ts @@ -0,0 +1,40 @@ +/** + * Injected script for discovering Pinia or Vuex stores and their actions/state representations + */ +export function discoverStores() { + const stores: Array<{ type: string; id: string; actions: string[]; stateKeys: string[] }> = []; + try { + const app = document.querySelector('#app') as any; + if (!app?.__vue_app__) return stores; + const gp = app.__vue_app__.config?.globalProperties; + + // Pinia stores + const pinia = gp?.$pinia; + if (pinia?._s) { + pinia._s.forEach((store: any, id: string) => { + const actions: string[] = []; + const stateKeys: string[] = []; + for (const k in store) { + try { + if (k.startsWith('$') || k.startsWith('_')) continue; + if (typeof store[k] === 'function') actions.push(k); + else stateKeys.push(k); + } catch {} + } + stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) }); + }); + } + + // Vuex store modules + const vuex = gp?.$store; + if (vuex?._modules?.root?._children) { + const children = vuex._modules.root._children; + for (const [modName, mod] of Object.entries(children)) { + const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20); + const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15); + stores.push({ type: 'vuex', id: modName, actions, stateKeys }); + } + } + } catch {} + return stores; +}