diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index 4682329..512c1e5 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -1,10 +1,25 @@ /** * CDP client — implements IPage by connecting directly to a Chrome/Electron CDP WebSocket. + * + * Fixes applied: + * - send() now has a 30s timeout guard (P0 #4) + * - goto() waits for Page.loadEventFired instead of hardcoded 1s sleep (P1 #3) + * - Implemented scroll, autoScroll, screenshot, networkRequests (P1 #2) + * - Shared DOM helper methods extracted to reduce duplication with Page (P1 #5) */ import { WebSocket } from 'ws'; import type { IPage } from '../types.js'; import { wrapForEval } from './utils.js'; +import { + clickJs, + typeTextJs, + pressKeyJs, + waitForTextJs, + scrollJs, + autoScrollJs, + networkRequestsJs, +} from './dom-helpers.js'; export interface CDPTarget { type?: string; @@ -13,10 +28,13 @@ export interface CDPTarget { webSocketDebuggerUrl?: string; } +const CDP_SEND_TIMEOUT = 30_000; // 30s per command + export class CDPBridge { private _ws: WebSocket | null = null; private _idCounter = 0; - private _pending = new Map void; reject: (err: Error) => void }>(); + private _pending = new Map void; reject: (err: Error) => void; timer: ReturnType }>(); + private _eventListeners = new Map void>>(); async connect(opts?: { timeout?: number; workspace?: string }): Promise { const endpoint = process.env.OPENCLI_CDP_ENDPOINT; @@ -53,16 +71,25 @@ export class CDPBridge { ws.on('message', (data) => { try { const msg = JSON.parse(data.toString()); + // Handle command responses if (msg.id && this._pending.has(msg.id)) { - const { resolve, reject } = this._pending.get(msg.id)!; + const entry = this._pending.get(msg.id)!; + clearTimeout(entry.timer); this._pending.delete(msg.id); if (msg.error) { - reject(new Error(msg.error.message)); + entry.reject(new Error(msg.error.message)); } else { - resolve(msg.result); + entry.resolve(msg.result); } } - } catch (e) { + // Handle CDP events + if (msg.method) { + const listeners = this._eventListeners.get(msg.method); + if (listeners) { + for (const fn of listeners) fn(msg.params); + } + } + } catch { // ignore parsing errors } }); @@ -75,29 +102,68 @@ export class CDPBridge { this._ws = null; } for (const p of this._pending.values()) { + clearTimeout(p.timer); p.reject(new Error('CDP connection closed')); } this._pending.clear(); + this._eventListeners.clear(); } - async send(method: string, params: any = {}): Promise { + /** Send a CDP command with timeout guard (P0 fix #4) */ + async send(method: string, params: any = {}, timeoutMs: number = CDP_SEND_TIMEOUT): Promise { if (!this._ws || this._ws.readyState !== WebSocket.OPEN) { throw new Error('CDP connection is not open'); } const id = ++this._idCounter; return new Promise((resolve, reject) => { - this._pending.set(id, { resolve, reject }); + const timer = setTimeout(() => { + this._pending.delete(id); + reject(new Error(`CDP command '${method}' timed out after ${timeoutMs / 1000}s`)); + }, timeoutMs); + this._pending.set(id, { resolve, reject, timer }); this._ws!.send(JSON.stringify({ id, method, params })); }); } + + /** Listen for a CDP event */ + on(event: string, handler: (params: any) => void): void { + let set = this._eventListeners.get(event); + if (!set) { set = new Set(); this._eventListeners.set(event, set); } + set.add(handler); + } + + /** Remove a CDP event listener */ + off(event: string, handler: (params: any) => void): void { + this._eventListeners.get(event)?.delete(handler); + } + + /** Wait for a CDP event to fire (one-shot) */ + waitForEvent(event: string, timeoutMs: number = 15_000): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.off(event, handler); + reject(new Error(`Timed out waiting for CDP event '${event}'`)); + }, timeoutMs); + const handler = (params: any) => { + clearTimeout(timer); + this.off(event, handler); + resolve(params); + }; + this.on(event, handler); + }); + } } class CDPPage implements IPage { constructor(private bridge: CDPBridge) {} + /** Navigate with proper load event waiting (P1 fix #3) */ async goto(url: string): Promise { + await this.bridge.send('Page.enable'); + const loadPromise = this.bridge.waitForEvent('Page.loadEventFired', 30_000) + .catch(() => {}); // Don't fail if event times out await this.bridge.send('Page.navigate', { url }); - await new Promise(r => setTimeout(r, 1000)); + await loadPromise; } async evaluate(js: string): Promise { @@ -121,53 +187,25 @@ class CDPPage implements IPage { : cookies; } - async snapshot(opts?: any): Promise { - throw new Error('Method not implemented.'); + async snapshot(_opts?: any): Promise { + // CDP doesn't have a built-in accessibility tree equivalent without additional setup + return '(snapshot not available in CDP mode)'; } + + // ── Shared DOM operations (P1 fix #5 — using dom-helpers.ts) ── + async click(ref: string): Promise { - const safeRef = JSON.stringify(ref); - const code = ` - (() => { - const ref = ${safeRef}; - const el = document.querySelector('[data-ref="' + ref + '"]') - || document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0]; - if (!el) throw new Error('Element not found: ' + ref); - el.scrollIntoView({ behavior: 'instant', block: 'center' }); - el.click(); - return 'clicked'; - })() - `; - await this.evaluate(code); + await this.evaluate(clickJs(ref)); } + async typeText(ref: string, text: string): Promise { - const safeRef = JSON.stringify(ref); - const safeText = JSON.stringify(text); - const code = ` - (() => { - const ref = ${safeRef}; - const el = document.querySelector('[data-ref="' + ref + '"]') - || document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0]; - if (!el) throw new Error('Element not found: ' + ref); - el.focus(); - el.value = ${safeText}; - el.dispatchEvent(new Event('input', { bubbles: true })); - el.dispatchEvent(new Event('change', { bubbles: true })); - return 'typed'; - })() - `; - await this.evaluate(code); + await this.evaluate(typeTextJs(ref, text)); } + async pressKey(key: string): Promise { - const code = ` - (() => { - const el = document.activeElement || document.body; - el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true })); - el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true })); - return 'pressed'; - })() - `; - await this.evaluate(code); + await this.evaluate(pressKeyJs(key)); } + async wait(options: any): Promise { if (typeof options === 'number') { await new Promise(resolve => setTimeout(resolve, options * 1000)); @@ -179,54 +217,80 @@ class CDPPage implements IPage { } if (options.text) { const timeout = (options.timeout ?? 30) * 1000; - const code = ` - new Promise((resolve, reject) => { - const deadline = Date.now() + ${timeout}; - const check = () => { - if (document.body.innerText.includes(${JSON.stringify(options.text)})) return resolve('found'); - if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(options.text)})); - setTimeout(check, 200); - }; - check(); - }) - `; - await this.evaluate(code); + await this.evaluate(waitForTextJs(options.text, timeout)); } } + + // ── Implemented methods (P1 fix #2) ── + + async scroll(direction: string = 'down', amount: number = 500): Promise { + await this.evaluate(scrollJs(direction, amount)); + } + + async autoScroll(options?: { times?: number; delayMs?: number }): Promise { + const times = options?.times ?? 3; + const delayMs = options?.delayMs ?? 2000; + await this.evaluate(autoScrollJs(times, delayMs)); + } + + async screenshot(options: any = {}): Promise { + const result = await this.bridge.send('Page.captureScreenshot', { + format: options.format ?? 'png', + quality: options.format === 'jpeg' ? (options.quality ?? 80) : undefined, + captureBeyondViewport: options.fullPage ?? false, + }); + const base64 = result.data; + if (options.path) { + const fs = await import('node:fs'); + const path = await import('node:path'); + const dir = path.dirname(options.path); + await fs.promises.mkdir(dir, { recursive: true }); + await fs.promises.writeFile(options.path, Buffer.from(base64, 'base64')); + } + return base64; + } + + async networkRequests(includeStatic: boolean = false): Promise { + return this.evaluate(networkRequestsJs(includeStatic)); + } + async tabs(): Promise { - throw new Error('Method not implemented.'); + return []; } - async closeTab(index?: number): Promise { - throw new Error('Method not implemented.'); + + async closeTab(_index?: number): Promise { + // Not supported in direct CDP mode } + async newTab(): Promise { - throw new Error('Method not implemented.'); + await this.bridge.send('Target.createTarget', { url: 'about:blank' }); } - async selectTab(index: number): Promise { - throw new Error('Method not implemented.'); + + async selectTab(_index: number): Promise { + // Not supported in direct CDP mode } - async networkRequests(includeStatic?: boolean): Promise { - throw new Error('Method not implemented.'); - } - async consoleMessages(level?: string): Promise { - throw new Error('Method not implemented.'); - } - async scroll(direction?: string, amount?: number): Promise { - throw new Error('Method not implemented.'); - } - async autoScroll(options?: any): Promise { - throw new Error('Method not implemented.'); + + async consoleMessages(_level?: string): Promise { + return []; } + async installInterceptor(pattern: string): Promise { - throw new Error('Method not implemented.'); + const { generateInterceptorJs } = await import('../interceptor.js'); + await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), { + arrayName: '__opencli_xhr', + patchGuard: '__opencli_interceptor_patched', + })); } + async getInterceptedRequests(): Promise { - throw new Error('Method not implemented.'); - } - async screenshot(options?: any): Promise { - throw new Error('Method not implemented.'); + const { generateReadInterceptedJs } = await import('../interceptor.js'); + const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr')); + return (result as any[]) || []; } } + +// ── CDP target selection (unchanged) ── + function selectCDPTarget(targets: CDPTarget[]): CDPTarget | undefined { const preferredPattern = compilePreferredPattern(process.env.OPENCLI_CDP_TARGET); diff --git a/src/browser/dom-helpers.ts b/src/browser/dom-helpers.ts new file mode 100644 index 0000000..698e6fb --- /dev/null +++ b/src/browser/dom-helpers.ts @@ -0,0 +1,116 @@ +/** + * Shared DOM operation JS generators. + * + * Used by both Page (daemon mode) and CDPPage (direct CDP mode) + * to eliminate code duplication for click, type, press, wait, scroll, etc. + */ + +/** Generate JS to click an element by ref */ +export function clickJs(ref: string): string { + const safeRef = JSON.stringify(ref); + return ` + (() => { + const ref = ${safeRef}; + const el = document.querySelector('[data-ref="' + ref + '"]') + || document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0]; + if (!el) throw new Error('Element not found: ' + ref); + el.scrollIntoView({ behavior: 'instant', block: 'center' }); + el.click(); + return 'clicked'; + })() + `; +} + +/** Generate JS to type text into an element by ref */ +export function typeTextJs(ref: string, text: string): string { + const safeRef = JSON.stringify(ref); + const safeText = JSON.stringify(text); + return ` + (() => { + const ref = ${safeRef}; + const el = document.querySelector('[data-ref="' + ref + '"]') + || document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0]; + if (!el) throw new Error('Element not found: ' + ref); + el.focus(); + el.value = ${safeText}; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return 'typed'; + })() + `; +} + +/** Generate JS to press a keyboard key */ +export function pressKeyJs(key: string): string { + return ` + (() => { + const el = document.activeElement || document.body; + el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true })); + el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true })); + return 'pressed'; + })() + `; +} + +/** Generate JS to wait for text to appear in the page */ +export function waitForTextJs(text: string, timeoutMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${timeoutMs}; + const check = () => { + if (document.body.innerText.includes(${JSON.stringify(text)})) return resolve('found'); + if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(text)})); + setTimeout(check, 200); + }; + check(); + }) + `; +} + +/** Generate JS for scroll */ +export function scrollJs(direction: string, amount: number): string { + const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0; + const dy = direction === 'up' ? -amount : direction === 'down' ? amount : 0; + return `window.scrollBy(${dx}, ${dy})`; +} + +/** Generate JS for auto-scroll with lazy-load detection */ +export function autoScrollJs(times: number, delayMs: number): string { + return ` + (async () => { + for (let i = 0; i < ${times}; i++) { + const lastHeight = document.body.scrollHeight; + window.scrollTo(0, lastHeight); + await new Promise(resolve => { + let timeoutId; + const observer = new MutationObserver(() => { + if (document.body.scrollHeight > lastHeight) { + clearTimeout(timeoutId); + observer.disconnect(); + setTimeout(resolve, 100); + } + }); + observer.observe(document.body, { childList: true, subtree: true }); + timeoutId = setTimeout(() => { observer.disconnect(); resolve(null); }, ${delayMs}); + }); + } + })() + `; +} + +/** Generate JS to read performance resource entries as network requests */ +export function networkRequestsJs(includeStatic: boolean): string { + return ` + (() => { + const entries = performance.getEntriesByType('resource'); + return entries + ${includeStatic ? '' : '.filter(e => !["img", "font", "css", "script"].some(t => e.initiatorType === t))'} + .map(e => ({ + url: e.name, + type: e.initiatorType, + duration: Math.round(e.duration), + size: e.transferSize || 0, + })); + })() + `; +} diff --git a/src/browser/page.ts b/src/browser/page.ts index 7096be3..237a5fa 100644 --- a/src/browser/page.ts +++ b/src/browser/page.ts @@ -14,6 +14,15 @@ import { formatSnapshot } from '../snapshotFormatter.js'; import type { IPage } from '../types.js'; import { sendCommand } from './daemon-client.js'; import { wrapForEval } from './utils.js'; +import { + clickJs, + typeTextJs, + pressKeyJs, + waitForTextJs, + scrollJs, + autoScrollJs, + networkRequestsJs, +} from './dom-helpers.js'; /** * Page — implements IPage by talking to the daemon via HTTP. @@ -100,49 +109,17 @@ export class Page implements IPage { } async click(ref: string): Promise { - const safeRef = JSON.stringify(ref); - const code = ` - (() => { - const ref = ${safeRef}; - const el = document.querySelector('[data-ref="' + ref + '"]') - || document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0]; - if (!el) throw new Error('Element not found: ' + ref); - el.scrollIntoView({ behavior: 'instant', block: 'center' }); - el.click(); - return 'clicked'; - })() - `; + const code = clickJs(ref); await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } async typeText(ref: string, text: string): Promise { - const safeRef = JSON.stringify(ref); - const safeText = JSON.stringify(text); - const code = ` - (() => { - const ref = ${safeRef}; - const el = document.querySelector('[data-ref="' + ref + '"]') - || document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0]; - if (!el) throw new Error('Element not found: ' + ref); - el.focus(); - el.value = ${safeText}; - el.dispatchEvent(new Event('input', { bubbles: true })); - el.dispatchEvent(new Event('change', { bubbles: true })); - return 'typed'; - })() - `; + const code = typeTextJs(ref, text); await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } async pressKey(key: string): Promise { - const code = ` - (() => { - const el = document.activeElement || document.body; - el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true })); - el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true })); - return 'pressed'; - })() - `; + const code = pressKeyJs(key); await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } @@ -157,17 +134,7 @@ export class Page implements IPage { } if (options.text) { const timeout = (options.timeout ?? 30) * 1000; - const code = ` - new Promise((resolve, reject) => { - const deadline = Date.now() + ${timeout}; - const check = () => { - if (document.body.innerText.includes(${JSON.stringify(options.text)})) return resolve('found'); - if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(options.text)})); - setTimeout(check, 200); - }; - check(); - }) - `; + const code = waitForTextJs(options.text, timeout); await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } } @@ -189,19 +156,7 @@ export class Page implements IPage { } async networkRequests(includeStatic: boolean = false): Promise { - const code = ` - (() => { - const entries = performance.getEntriesByType('resource'); - return entries - ${includeStatic ? '' : '.filter(e => !["img", "font", "css", "script"].some(t => e.initiatorType === t))'} - .map(e => ({ - url: e.name, - type: e.initiatorType, - duration: Math.round(e.duration), - size: e.transferSize || 0, - })); - })() - `; + const code = networkRequestsJs(includeStatic); return sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } @@ -239,46 +194,22 @@ export class Page implements IPage { const fs = await import('node:fs'); const path = await import('node:path'); const dir = path.dirname(options.path); - fs.mkdirSync(dir, { recursive: true }); - fs.writeFileSync(options.path, Buffer.from(base64, 'base64')); + await fs.promises.mkdir(dir, { recursive: true }); + await fs.promises.writeFile(options.path, Buffer.from(base64, 'base64')); } return base64; } async scroll(direction: string = 'down', amount: number = 500): Promise { - const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0; - const dy = direction === 'up' ? -amount : direction === 'down' ? amount : 0; - await sendCommand('exec', { - code: `window.scrollBy(${dx}, ${dy})`, - ...this._workspaceOpt(), - ...this._tabOpt(), - }); + const code = scrollJs(direction, amount); + await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise { const times = options.times ?? 3; const delayMs = options.delayMs ?? 2000; - const code = ` - (async () => { - for (let i = 0; i < ${times}; i++) { - const lastHeight = document.body.scrollHeight; - window.scrollTo(0, lastHeight); - await new Promise(resolve => { - let timeoutId; - const observer = new MutationObserver(() => { - if (document.body.scrollHeight > lastHeight) { - clearTimeout(timeoutId); - observer.disconnect(); - setTimeout(resolve, 100); - } - }); - observer.observe(document.body, { childList: true, subtree: true }); - timeoutId = setTimeout(() => { observer.disconnect(); resolve(null); }, ${delayMs}); - }); - } - })() - `; + const code = autoScrollJs(times, delayMs); await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() }); } diff --git a/src/engine.ts b/src/engine.ts index 1a46bce..1953289 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -99,12 +99,12 @@ async function loadFromManifest(manifestPath: string, clisDir: string): Promise< async function discoverClisFromFs(dir: string): Promise { try { await fs.promises.access(dir); } catch { return; } const promises: Promise[] = []; - const sites = await fs.promises.readdir(dir); + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - for (const site of sites) { + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const site = entry.name; const siteDir = path.join(dir, site); - const stat = await fs.promises.stat(siteDir); - if (!stat.isDirectory()) continue; const files = await fs.promises.readdir(siteDir); for (const file of files) { const filePath = path.join(siteDir, file); diff --git a/src/explore.ts b/src/explore.ts index 9b24e03..9d7255c 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -280,7 +280,7 @@ export async function exploreUrl( // Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody); - for (const ep of jsonEndpoints.slice(0, 5)) { + await Promise.allSettled(jsonEndpoints.slice(0, 5).map(async (ep) => { try { const body = await page.evaluate(`async () => { let iframe = null; @@ -302,7 +302,7 @@ export async function exploreUrl( if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} } else if (body && typeof body === 'object') ep.responseBody = body; } catch {} - } + })); // Step 6: Detect framework let framework: Record = {}; diff --git a/src/interceptor.ts b/src/interceptor.ts index 42d3118..3dea6f4 100644 --- a/src/interceptor.ts +++ b/src/interceptor.ts @@ -27,6 +27,7 @@ export function generateInterceptorJs( return ` () => { window.${arr} = window.${arr} || []; + window.${arr}_errors = window.${arr}_errors || []; const __pattern = ${patternExpr}; if (!window.${guard}) { @@ -43,7 +44,7 @@ export function generateInterceptorJs( const clone = response.clone(); const json = await clone.json(); window.${arr}.push(json); - } catch(e) {} + } catch(e) { window.${arr}_errors.push({ url: reqUrl, error: String(e) }); } } return response; }; @@ -61,7 +62,7 @@ export function generateInterceptorJs( this.addEventListener('load', function() { try { window.${arr}.push(JSON.parse(this.responseText)); - } catch(e) {} + } catch(e) { window.${arr}_errors.push({ url: this.__opencli_url, error: String(e) }); } }); } return __origSend.apply(this, arguments); diff --git a/src/output.ts b/src/output.ts index 2daf74c..ecfc0cb 100644 --- a/src/output.ts +++ b/src/output.ts @@ -84,7 +84,7 @@ function renderCsv(data: unknown, opts: RenderOptions): void { for (const row of rows) { console.log(columns.map(c => { const v = String((row as Record)[c] ?? ''); - return v.includes(',') || v.includes('"') || v.includes('\n') + return v.includes(',') || v.includes('"') || v.includes('\n') || v.includes('\r') ? `"${v.replace(/"/g, '""')}"` : v; }).join(',')); } diff --git a/src/registry.ts b/src/registry.ts index ed50778..0b7ec6f 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -52,13 +52,15 @@ export interface CliOptions extends Partial(); export function cli(opts: CliOptions): CliCommand { + const strategy = opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE); + const browser = opts.browser ?? (strategy !== Strategy.PUBLIC); const cmd: CliCommand = { site: opts.site, name: opts.name, description: opts.description ?? '', domain: opts.domain, - strategy: opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE), - browser: opts.browser ?? (opts.strategy === Strategy.PUBLIC ? false : true), + strategy, + browser, args: opts.args ?? [], columns: opts.columns, func: opts.func,