From a7e530722647fe6c269b2f9623b4d4f690611ce5 Mon Sep 17 00:00:00 2001 From: jackwener Date: Fri, 20 Mar 2026 00:48:39 +0800 Subject: [PATCH] feat: Add Chrome DevTools Protocol (CDP) support as an alternative browser automation backend, configurable via `OPENCLI_CDP_ENDPOINT`. --- src/browser.test.ts | 46 +++++++ src/browser/cdp.ts | 304 +++++++++++++++++++++++++++++++++++++++++++ src/browser/index.ts | 4 + src/main.ts | 12 +- 4 files changed, 361 insertions(+), 5 deletions(-) create mode 100644 src/browser/cdp.ts diff --git a/src/browser.test.ts b/src/browser.test.ts index e9bfc18..6fea348 100644 --- a/src/browser.test.ts +++ b/src/browser.test.ts @@ -43,6 +43,52 @@ describe('browser helpers', () => { it('times out slow promises', async () => { await expect(__test__.withTimeoutMs(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout'); }); + + it('prefers the real Electron app target over DevTools and blank pages', () => { + const target = __test__.selectCDPTarget([ + { + type: 'page', + title: 'DevTools - localhost:9224', + url: 'devtools://devtools/bundled/inspector.html', + webSocketDebuggerUrl: 'ws://127.0.0.1:9224/devtools', + }, + { + type: 'page', + title: '', + url: 'about:blank', + webSocketDebuggerUrl: 'ws://127.0.0.1:9224/blank', + }, + { + type: 'app', + title: 'Antigravity', + url: 'http://localhost:3000/', + webSocketDebuggerUrl: 'ws://127.0.0.1:9224/app', + }, + ]); + + expect(target?.webSocketDebuggerUrl).toBe('ws://127.0.0.1:9224/app'); + }); + + it('honors OPENCLI_CDP_TARGET when multiple inspectable targets exist', () => { + vi.stubEnv('OPENCLI_CDP_TARGET', 'codex'); + + const target = __test__.selectCDPTarget([ + { + type: 'app', + title: 'Cursor', + url: 'http://localhost:3000/cursor', + webSocketDebuggerUrl: 'ws://127.0.0.1:9226/cursor', + }, + { + type: 'app', + title: 'OpenAI Codex', + url: 'http://localhost:3000/codex', + webSocketDebuggerUrl: 'ws://127.0.0.1:9226/codex', + }, + ]); + + expect(target?.webSocketDebuggerUrl).toBe('ws://127.0.0.1:9226/codex'); + }); }); describe('BrowserBridge state', () => { diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts new file mode 100644 index 0000000..dcc6e0c --- /dev/null +++ b/src/browser/cdp.ts @@ -0,0 +1,304 @@ +/** + * CDP client — implements IPage by connecting directly to a Chrome/Electron CDP WebSocket. + */ + +import { WebSocket } from 'ws'; +import type { IPage } from '../types.js'; + +export interface CDPTarget { + type?: string; + url?: string; + title?: string; + webSocketDebuggerUrl?: string; +} + +export class CDPBridge { + private _ws: WebSocket | null = null; + private _idCounter = 0; + private _pending = new Map void; reject: (err: Error) => void }>(); + + async connect(opts?: { timeout?: number }): Promise { + const endpoint = process.env.OPENCLI_CDP_ENDPOINT; + if (!endpoint) throw new Error('OPENCLI_CDP_ENDPOINT is not set'); + + // If it's a direct ws:// URL, use it. Otherwise, fetch the /json endpoint to find a page. + let wsUrl = endpoint; + if (endpoint.startsWith('http')) { + const res = await fetch(`${endpoint.replace(/\/$/, '')}/json`); + if (!res.ok) throw new Error(`Failed to fetch CDP targets: ${res.statusText}`); + const targets = await res.json() as CDPTarget[]; + const target = selectCDPTarget(targets); + if (!target || !target.webSocketDebuggerUrl) { + throw new Error('No inspectable targets found at CDP endpoint'); + } + wsUrl = target.webSocketDebuggerUrl; + } + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + const timeout = setTimeout(() => reject(new Error('CDP connect timeout')), opts?.timeout ?? 10000); + + ws.on('open', () => { + clearTimeout(timeout); + this._ws = ws; + resolve(new CDPPage(this)); + }); + + ws.on('error', (err) => { + clearTimeout(timeout); + reject(err); + }); + + ws.on('message', (data) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.id && this._pending.has(msg.id)) { + const { resolve, reject } = this._pending.get(msg.id)!; + this._pending.delete(msg.id); + if (msg.error) { + reject(new Error(msg.error.message)); + } else { + resolve(msg.result); + } + } + } catch (e) { + // ignore parsing errors + } + }); + }); + } + + async close(): Promise { + if (this._ws) { + this._ws.close(); + this._ws = null; + } + for (const p of this._pending.values()) { + p.reject(new Error('CDP connection closed')); + } + this._pending.clear(); + } + + async send(method: string, params: any = {}): Promise { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) { + throw new Error('CDP connection is not open'); + } + const id = ++this._idCounter; + return new Promise((resolve, reject) => { + this._pending.set(id, { resolve, reject }); + this._ws!.send(JSON.stringify({ id, method, params })); + }); + } +} + +class CDPPage implements IPage { + constructor(private bridge: CDPBridge) {} + + async goto(url: string): Promise { + await this.bridge.send('Page.navigate', { url }); + await new Promise(r => setTimeout(r, 1000)); + } + + async evaluate(js: string): Promise { + const expression = wrapForEval(js); + const result = await this.bridge.send('Runtime.evaluate', { + expression, + returnByValue: true, + awaitPromise: true + }); + if (result.exceptionDetails) { + throw new Error('Evaluate error: ' + (result.exceptionDetails.exception?.description || 'Unknown exception')); + } + return result.result?.value; + } + + async snapshot(opts?: any): Promise { + throw new Error('Method not implemented.'); + } + async click(ref: string): Promise { + const safeRef = JSON.stringify(ref); + const code = ` + (() => { + const ref = ${safeRef}; + const el = document.querySelector('[data-ref="' + ref + '"]') + || document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0]; + if (!el) throw new Error('Element not found: ' + ref); + el.scrollIntoView({ behavior: 'instant', block: 'center' }); + el.click(); + return 'clicked'; + })() + `; + await this.evaluate(code); + } + async typeText(ref: string, text: string): Promise { + const safeRef = JSON.stringify(ref); + const safeText = JSON.stringify(text); + const code = ` + (() => { + const ref = ${safeRef}; + const el = document.querySelector('[data-ref="' + ref + '"]') + || document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0]; + if (!el) throw new Error('Element not found: ' + ref); + el.focus(); + el.value = ${safeText}; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return 'typed'; + })() + `; + await this.evaluate(code); + } + async pressKey(key: string): Promise { + const code = ` + (() => { + const el = document.activeElement || document.body; + el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true })); + el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true })); + return 'pressed'; + })() + `; + await this.evaluate(code); + } + async wait(options: any): Promise { + if (typeof options === 'number') { + await new Promise(resolve => setTimeout(resolve, options * 1000)); + return; + } + if (options.time) { + await new Promise(resolve => setTimeout(resolve, options.time * 1000)); + return; + } + if (options.text) { + const timeout = (options.timeout ?? 30) * 1000; + const code = ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${timeout}; + const check = () => { + if (document.body.innerText.includes(${JSON.stringify(options.text)})) return resolve('found'); + if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(options.text)})); + setTimeout(check, 200); + }; + check(); + }) + `; + await this.evaluate(code); + } + } + async tabs(): Promise { + throw new Error('Method not implemented.'); + } + async closeTab(index?: number): Promise { + throw new Error('Method not implemented.'); + } + async newTab(): Promise { + throw new Error('Method not implemented.'); + } + async selectTab(index: number): Promise { + throw new Error('Method not implemented.'); + } + async networkRequests(includeStatic?: boolean): Promise { + throw new Error('Method not implemented.'); + } + async consoleMessages(level?: string): Promise { + throw new Error('Method not implemented.'); + } + async scroll(direction?: string, amount?: number): Promise { + throw new Error('Method not implemented.'); + } + async autoScroll(options?: any): Promise { + throw new Error('Method not implemented.'); + } + async installInterceptor(pattern: string): Promise { + throw new Error('Method not implemented.'); + } + async getInterceptedRequests(): Promise { + throw new Error('Method not implemented.'); + } + async screenshot(options?: any): Promise { + throw new Error('Method not implemented.'); + } +} + +function wrapForEval(js: string): string { + const code = js.trim(); + if (!code) return 'undefined'; + if (/^\([\s\S]*\)\s*\(.*\)\s*$/.test(code)) return code; + if (/^(async\s+)?(\([^)]*\)|[A-Za-z_]\w*)\s*=>/.test(code)) return `(${code})()`; + if (/^(async\s+)?function[\s(]/.test(code)) return `(${code})()`; + return code; +} + +function selectCDPTarget(targets: CDPTarget[]): CDPTarget | undefined { + const preferredPattern = compilePreferredPattern(process.env.OPENCLI_CDP_TARGET); + + const ranked = targets + .map((target, index) => ({ target, index, score: scoreCDPTarget(target, preferredPattern) })) + .filter(({ score }) => Number.isFinite(score)) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return a.index - b.index; + }); + + return ranked[0]?.target; +} + +function scoreCDPTarget(target: CDPTarget, preferredPattern?: RegExp): number { + if (!target.webSocketDebuggerUrl) return Number.NEGATIVE_INFINITY; + + const type = (target.type ?? '').toLowerCase(); + const url = (target.url ?? '').toLowerCase(); + const title = (target.title ?? '').toLowerCase(); + const haystack = `${title} ${url}`; + + if (!haystack.trim() && !type) return Number.NEGATIVE_INFINITY; + if (haystack.includes('devtools')) return Number.NEGATIVE_INFINITY; + + let score = 0; + + if (preferredPattern && preferredPattern.test(haystack)) score += 1000; + + if (type === 'app') score += 120; + else if (type === 'webview') score += 100; + else if (type === 'page') score += 80; + else if (type === 'iframe') score += 20; + + if (url.startsWith('http://localhost') || url.startsWith('https://localhost')) score += 90; + if (url.startsWith('file://')) score += 60; + if (url.startsWith('http://127.0.0.1') || url.startsWith('https://127.0.0.1')) score += 50; + if (url.startsWith('about:blank')) score -= 120; + if (url === '' || url === 'about:blank') score -= 40; + + if (title && title !== 'devtools') score += 25; + if (title.includes('antigravity')) score += 120; + if (title.includes('codex')) score += 120; + if (title.includes('cursor')) score += 120; + if (title.includes('chatwise')) score += 120; + if (title.includes('notion')) score += 120; + if (title.includes('discord')) score += 120; + if (title.includes('netease')) score += 120; + + if (url.includes('antigravity')) score += 100; + if (url.includes('codex')) score += 100; + if (url.includes('cursor')) score += 100; + if (url.includes('chatwise')) score += 100; + if (url.includes('notion')) score += 100; + if (url.includes('discord')) score += 100; + if (url.includes('netease')) score += 100; + + return score; +} + +function compilePreferredPattern(raw: string | undefined): RegExp | undefined { + const value = raw?.trim(); + if (!value) return undefined; + return new RegExp(escapeRegExp(value.toLowerCase())); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +export const __test__ = { + selectCDPTarget, + scoreCDPTarget, +}; diff --git a/src/browser/index.ts b/src/browser/index.ts index 11a6278..4ee3a93 100644 --- a/src/browser/index.ts +++ b/src/browser/index.ts @@ -7,9 +7,11 @@ export { Page } from './page.js'; export { BrowserBridge, BrowserBridge as PlaywrightMCP } from './mcp.js'; +export { CDPBridge } from './cdp.js'; export { isDaemonRunning } from './daemon-client.js'; import { extractTabEntries, diffTabIndexes, appendLimited } from './tabs.js'; +import { __test__ as cdpTest } from './cdp.js'; import { withTimeoutMs } from '../runtime.js'; export const __test__ = { @@ -17,4 +19,6 @@ export const __test__ = { diffTabIndexes, appendLimited, withTimeoutMs, + selectCDPTarget: cdpTest.selectCDPTarget, + scoreCDPTarget: cdpTest.scoreCDPTarget, }; diff --git a/src/main.ts b/src/main.ts index e59588a..d7d3306 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,7 +11,7 @@ import chalk from 'chalk'; import { discoverClis, executeCommand } from './engine.js'; import { Strategy, type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js'; import { render as renderOutput } from './output.js'; -import { BrowserBridge } from './browser/index.js'; +import { BrowserBridge, CDPBridge } from './browser/index.js'; import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js'; import { PKG_VERSION } from './version.js'; import { getCompletions, printCompletionScript } from './completion.js'; @@ -99,18 +99,19 @@ program.command('verify').description('Validate + smoke test').argument('[target }); program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('').option('--site ').option('--goal ').option('--wait ', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click ', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")') - .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserBridge, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); }); + .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserFactory as any, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); }); program.command('synthesize').description('Synthesize CLIs from explore').argument('').option('--top ', '', '3') .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); }); program.command('generate').description('One-shot: explore → synthesize → register').argument('').option('--goal ').option('--site ') - .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: BrowserBridge, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; }); + .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; const r = await generateCliFromUrl({ url, BrowserFactory: BrowserFactory as any, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; }); program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('').option('--site ') .action(async (url, opts) => { const { cascadeProbe, renderCascadeResult } = await import('./cascade.js'); - const result = await browserSession(BrowserBridge, async (page) => { + const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; + const result = await browserSession(BrowserFactory as any, async (page) => { // Navigate to the site first for cookie context try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {} return cascadeProbe(page, url); @@ -192,7 +193,8 @@ for (const [, cmd] of registry) { if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1'; let result: any; if (cmd.browser) { - result = await browserSession(BrowserBridge, async (page) => { + const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; + result = await browserSession(BrowserFactory as any, async (page) => { return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }); }); } else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }