refactor: deep audit fixes P0-P3

P0: page.ts screenshot async I/O, cdp.ts send() 30s timeout
P1: cdp.ts event-based goto, implement scroll/screenshot/networkRequests,
    extract dom-helpers.ts shared module for Page/CDPPage
P2: engine.ts readdir withFileTypes, explore.ts parallel refetch
P3: registry.ts strategy ordering, output.ts CSV \r escape,
    interceptor.ts error tracking array
This commit is contained in:
jackwener committed 2026-03-20 15:27:07 +08:00
1 parent 9d8b6441be
commit dff8f1e9c4
8 files changed
+295 -181

No files matched your search

+146 -82
View File
@@ -1,10 +1,25 @@
/**
* CDP client — implements IPage by connecting directly to a Chrome/Electron CDP WebSocket.
*
* Fixes applied:
* - send() now has a 30s timeout guard (P0 #4)
* - goto() waits for Page.loadEventFired instead of hardcoded 1s sleep (P1 #3)
* - Implemented scroll, autoScroll, screenshot, networkRequests (P1 #2)
* - Shared DOM helper methods extracted to reduce duplication with Page (P1 #5)
*/
import { WebSocket } from 'ws';
import type { IPage } from '../types.js';
import { wrapForEval } from './utils.js';
import {
clickJs,
typeTextJs,
pressKeyJs,
waitForTextJs,
scrollJs,
autoScrollJs,
networkRequestsJs,
} from './dom-helpers.js';
export interface CDPTarget {
type?: string;
@@ -13,10 +28,13 @@ export interface CDPTarget {
webSocketDebuggerUrl?: string;
}
const CDP_SEND_TIMEOUT = 30_000; // 30s per command
export class CDPBridge {
private _ws: WebSocket | null = null;
private _idCounter = 0;
private _pending = new Map<number, { resolve: (val: any) => void; reject: (err: Error) => void }>();
private _pending = new Map<number, { resolve: (val: any) => void; reject: (err: Error) => void; timer: ReturnType<typeof setTimeout> }>();
private _eventListeners = new Map<string, Set<(params: any) => void>>();
async connect(opts?: { timeout?: number; workspace?: string }): Promise<IPage> {
const endpoint = process.env.OPENCLI_CDP_ENDPOINT;
@@ -53,16 +71,25 @@ export class CDPBridge {
ws.on('message', (data) => {
try {
const msg = JSON.parse(data.toString());
// Handle command responses
if (msg.id && this._pending.has(msg.id)) {
const { resolve, reject } = this._pending.get(msg.id)!;
const entry = this._pending.get(msg.id)!;
clearTimeout(entry.timer);
this._pending.delete(msg.id);
if (msg.error) {
reject(new Error(msg.error.message));
entry.reject(new Error(msg.error.message));
} else {
resolve(msg.result);
entry.resolve(msg.result);
}
}
} catch (e) {
// Handle CDP events
if (msg.method) {
const listeners = this._eventListeners.get(msg.method);
if (listeners) {
for (const fn of listeners) fn(msg.params);
}
}
} catch {
// ignore parsing errors
}
});
@@ -75,29 +102,68 @@ export class CDPBridge {
this._ws = null;
}
for (const p of this._pending.values()) {
clearTimeout(p.timer);
p.reject(new Error('CDP connection closed'));
}
this._pending.clear();
this._eventListeners.clear();
}
async send(method: string, params: any = {}): Promise<any> {
/** Send a CDP command with timeout guard (P0 fix #4) */
async send(method: string, params: any = {}, timeoutMs: number = CDP_SEND_TIMEOUT): Promise<any> {
if (!this._ws || this._ws.readyState !== WebSocket.OPEN) {
throw new Error('CDP connection is not open');
}
const id = ++this._idCounter;
return new Promise((resolve, reject) => {
this._pending.set(id, { resolve, reject });
const timer = setTimeout(() => {
this._pending.delete(id);
reject(new Error(`CDP command '${method}' timed out after ${timeoutMs / 1000}s`));
}, timeoutMs);
this._pending.set(id, { resolve, reject, timer });
this._ws!.send(JSON.stringify({ id, method, params }));
});
}
/** Listen for a CDP event */
on(event: string, handler: (params: any) => void): void {
let set = this._eventListeners.get(event);
if (!set) { set = new Set(); this._eventListeners.set(event, set); }
set.add(handler);
}
/** Remove a CDP event listener */
off(event: string, handler: (params: any) => void): void {
this._eventListeners.get(event)?.delete(handler);
}
/** Wait for a CDP event to fire (one-shot) */
waitForEvent(event: string, timeoutMs: number = 15_000): Promise<any> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
this.off(event, handler);
reject(new Error(`Timed out waiting for CDP event '${event}'`));
}, timeoutMs);
const handler = (params: any) => {
clearTimeout(timer);
this.off(event, handler);
resolve(params);
};
this.on(event, handler);
});
}
}
class CDPPage implements IPage {
constructor(private bridge: CDPBridge) {}
/** Navigate with proper load event waiting (P1 fix #3) */
async goto(url: string): Promise<void> {
await this.bridge.send('Page.enable');
const loadPromise = this.bridge.waitForEvent('Page.loadEventFired', 30_000)
.catch(() => {}); // Don't fail if event times out
await this.bridge.send('Page.navigate', { url });
await new Promise(r => setTimeout(r, 1000));
await loadPromise;
}
async evaluate(js: string): Promise<any> {
@@ -121,53 +187,25 @@ class CDPPage implements IPage {
: cookies;
}
async snapshot(opts?: any): Promise<any> {
throw new Error('Method not implemented.');
async snapshot(_opts?: any): Promise<any> {
// CDP doesn't have a built-in accessibility tree equivalent without additional setup
return '(snapshot not available in CDP mode)';
}
// ── Shared DOM operations (P1 fix #5 — using dom-helpers.ts) ──
async click(ref: string): Promise<void> {
const safeRef = JSON.stringify(ref);
const code = `
(() => {
const ref = ${safeRef};
const el = document.querySelector('[data-ref="' + ref + '"]')
|| document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0];
if (!el) throw new Error('Element not found: ' + ref);
el.scrollIntoView({ behavior: 'instant', block: 'center' });
el.click();
return 'clicked';
})()
`;
await this.evaluate(code);
await this.evaluate(clickJs(ref));
}
async typeText(ref: string, text: string): Promise<void> {
const safeRef = JSON.stringify(ref);
const safeText = JSON.stringify(text);
const code = `
(() => {
const ref = ${safeRef};
const el = document.querySelector('[data-ref="' + ref + '"]')
|| document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0];
if (!el) throw new Error('Element not found: ' + ref);
el.focus();
el.value = ${safeText};
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
return 'typed';
})()
`;
await this.evaluate(code);
await this.evaluate(typeTextJs(ref, text));
}
async pressKey(key: string): Promise<void> {
const code = `
(() => {
const el = document.activeElement || document.body;
el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true }));
el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true }));
return 'pressed';
})()
`;
await this.evaluate(code);
await this.evaluate(pressKeyJs(key));
}
async wait(options: any): Promise<void> {
if (typeof options === 'number') {
await new Promise(resolve => setTimeout(resolve, options * 1000));
@@ -179,54 +217,80 @@ class CDPPage implements IPage {
}
if (options.text) {
const timeout = (options.timeout ?? 30) * 1000;
const code = `
new Promise((resolve, reject) => {
const deadline = Date.now() + ${timeout};
const check = () => {
if (document.body.innerText.includes(${JSON.stringify(options.text)})) return resolve('found');
if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(options.text)}));
setTimeout(check, 200);
};
check();
})
`;
await this.evaluate(code);
await this.evaluate(waitForTextJs(options.text, timeout));
}
}
// ── Implemented methods (P1 fix #2) ──
async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
await this.evaluate(scrollJs(direction, amount));
}
async autoScroll(options?: { times?: number; delayMs?: number }): Promise<void> {
const times = options?.times ?? 3;
const delayMs = options?.delayMs ?? 2000;
await this.evaluate(autoScrollJs(times, delayMs));
}
async screenshot(options: any = {}): Promise<string> {
const result = await this.bridge.send('Page.captureScreenshot', {
format: options.format ?? 'png',
quality: options.format === 'jpeg' ? (options.quality ?? 80) : undefined,
captureBeyondViewport: options.fullPage ?? false,
});
const base64 = result.data;
if (options.path) {
const fs = await import('node:fs');
const path = await import('node:path');
const dir = path.dirname(options.path);
await fs.promises.mkdir(dir, { recursive: true });
await fs.promises.writeFile(options.path, Buffer.from(base64, 'base64'));
}
return base64;
}
async networkRequests(includeStatic: boolean = false): Promise<any> {
return this.evaluate(networkRequestsJs(includeStatic));
}
async tabs(): Promise<any> {
throw new Error('Method not implemented.');
return [];
}
async closeTab(index?: number): Promise<void> {
throw new Error('Method not implemented.');
async closeTab(_index?: number): Promise<void> {
// Not supported in direct CDP mode
}
async newTab(): Promise<void> {
throw new Error('Method not implemented.');
await this.bridge.send('Target.createTarget', { url: 'about:blank' });
}
async selectTab(index: number): Promise<void> {
throw new Error('Method not implemented.');
async selectTab(_index: number): Promise<void> {
// Not supported in direct CDP mode
}
async networkRequests(includeStatic?: boolean): Promise<any> {
throw new Error('Method not implemented.');
}
async consoleMessages(level?: string): Promise<any> {
throw new Error('Method not implemented.');
}
async scroll(direction?: string, amount?: number): Promise<void> {
throw new Error('Method not implemented.');
}
async autoScroll(options?: any): Promise<void> {
throw new Error('Method not implemented.');
async consoleMessages(_level?: string): Promise<any> {
return [];
}
async installInterceptor(pattern: string): Promise<void> {
throw new Error('Method not implemented.');
const { generateInterceptorJs } = await import('../interceptor.js');
await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
arrayName: '__opencli_xhr',
patchGuard: '__opencli_interceptor_patched',
}));
}
async getInterceptedRequests(): Promise<any[]> {
throw new Error('Method not implemented.');
}
async screenshot(options?: any): Promise<string> {
throw new Error('Method not implemented.');
const { generateReadInterceptedJs } = await import('../interceptor.js');
const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
return (result as any[]) || [];
}
}
// ── CDP target selection (unchanged) ──
function selectCDPTarget(targets: CDPTarget[]): CDPTarget | undefined {
const preferredPattern = compilePreferredPattern(process.env.OPENCLI_CDP_TARGET);
+116
View File
@@ -0,0 +1,116 @@
/**
* Shared DOM operation JS generators.
*
* Used by both Page (daemon mode) and CDPPage (direct CDP mode)
* to eliminate code duplication for click, type, press, wait, scroll, etc.
*/
/** Generate JS to click an element by ref */
export function clickJs(ref: string): string {
const safeRef = JSON.stringify(ref);
return `
(() => {
const ref = ${safeRef};
const el = document.querySelector('[data-ref="' + ref + '"]')
|| document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0];
if (!el) throw new Error('Element not found: ' + ref);
el.scrollIntoView({ behavior: 'instant', block: 'center' });
el.click();
return 'clicked';
})()
`;
}
/** Generate JS to type text into an element by ref */
export function typeTextJs(ref: string, text: string): string {
const safeRef = JSON.stringify(ref);
const safeText = JSON.stringify(text);
return `
(() => {
const ref = ${safeRef};
const el = document.querySelector('[data-ref="' + ref + '"]')
|| document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0];
if (!el) throw new Error('Element not found: ' + ref);
el.focus();
el.value = ${safeText};
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
return 'typed';
})()
`;
}
/** Generate JS to press a keyboard key */
export function pressKeyJs(key: string): string {
return `
(() => {
const el = document.activeElement || document.body;
el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true }));
el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true }));
return 'pressed';
})()
`;
}
/** Generate JS to wait for text to appear in the page */
export function waitForTextJs(text: string, timeoutMs: number): string {
return `
new Promise((resolve, reject) => {
const deadline = Date.now() + ${timeoutMs};
const check = () => {
if (document.body.innerText.includes(${JSON.stringify(text)})) return resolve('found');
if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(text)}));
setTimeout(check, 200);
};
check();
})
`;
}
/** Generate JS for scroll */
export function scrollJs(direction: string, amount: number): string {
const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0;
const dy = direction === 'up' ? -amount : direction === 'down' ? amount : 0;
return `window.scrollBy(${dx}, ${dy})`;
}
/** Generate JS for auto-scroll with lazy-load detection */
export function autoScrollJs(times: number, delayMs: number): string {
return `
(async () => {
for (let i = 0; i < ${times}; i++) {
const lastHeight = document.body.scrollHeight;
window.scrollTo(0, lastHeight);
await new Promise(resolve => {
let timeoutId;
const observer = new MutationObserver(() => {
if (document.body.scrollHeight > lastHeight) {
clearTimeout(timeoutId);
observer.disconnect();
setTimeout(resolve, 100);
}
});
observer.observe(document.body, { childList: true, subtree: true });
timeoutId = setTimeout(() => { observer.disconnect(); resolve(null); }, ${delayMs});
});
}
})()
`;
}
/** Generate JS to read performance resource entries as network requests */
export function networkRequestsJs(includeStatic: boolean): string {
return `
(() => {
const entries = performance.getEntriesByType('resource');
return entries
${includeStatic ? '' : '.filter(e => !["img", "font", "css", "script"].some(t => e.initiatorType === t))'}
.map(e => ({
url: e.name,
type: e.initiatorType,
duration: Math.round(e.duration),
size: e.transferSize || 0,
}));
})()
`;
}
+19 -88
View File
@@ -14,6 +14,15 @@ import { formatSnapshot } from '../snapshotFormatter.js';
import type { IPage } from '../types.js';
import { sendCommand } from './daemon-client.js';
import { wrapForEval } from './utils.js';
import {
clickJs,
typeTextJs,
pressKeyJs,
waitForTextJs,
scrollJs,
autoScrollJs,
networkRequestsJs,
} from './dom-helpers.js';
/**
* Page — implements IPage by talking to the daemon via HTTP.
@@ -100,49 +109,17 @@ export class Page implements IPage {
}
async click(ref: string): Promise<void> {
const safeRef = JSON.stringify(ref);
const code = `
(() => {
const ref = ${safeRef};
const el = document.querySelector('[data-ref="' + ref + '"]')
|| document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0];
if (!el) throw new Error('Element not found: ' + ref);
el.scrollIntoView({ behavior: 'instant', block: 'center' });
el.click();
return 'clicked';
})()
`;
const code = clickJs(ref);
await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
async typeText(ref: string, text: string): Promise<void> {
const safeRef = JSON.stringify(ref);
const safeText = JSON.stringify(text);
const code = `
(() => {
const ref = ${safeRef};
const el = document.querySelector('[data-ref="' + ref + '"]')
|| document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0];
if (!el) throw new Error('Element not found: ' + ref);
el.focus();
el.value = ${safeText};
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
return 'typed';
})()
`;
const code = typeTextJs(ref, text);
await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
async pressKey(key: string): Promise<void> {
const code = `
(() => {
const el = document.activeElement || document.body;
el.dispatchEvent(new KeyboardEvent('keydown', { key: ${JSON.stringify(key)}, bubbles: true }));
el.dispatchEvent(new KeyboardEvent('keyup', { key: ${JSON.stringify(key)}, bubbles: true }));
return 'pressed';
})()
`;
const code = pressKeyJs(key);
await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
@@ -157,17 +134,7 @@ export class Page implements IPage {
}
if (options.text) {
const timeout = (options.timeout ?? 30) * 1000;
const code = `
new Promise((resolve, reject) => {
const deadline = Date.now() + ${timeout};
const check = () => {
if (document.body.innerText.includes(${JSON.stringify(options.text)})) return resolve('found');
if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(options.text)}));
setTimeout(check, 200);
};
check();
})
`;
const code = waitForTextJs(options.text, timeout);
await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
}
@@ -189,19 +156,7 @@ export class Page implements IPage {
}
async networkRequests(includeStatic: boolean = false): Promise<any> {
const code = `
(() => {
const entries = performance.getEntriesByType('resource');
return entries
${includeStatic ? '' : '.filter(e => !["img", "font", "css", "script"].some(t => e.initiatorType === t))'}
.map(e => ({
url: e.name,
type: e.initiatorType,
duration: Math.round(e.duration),
size: e.transferSize || 0,
}));
})()
`;
const code = networkRequestsJs(includeStatic);
return sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
@@ -239,46 +194,22 @@ export class Page implements IPage {
const fs = await import('node:fs');
const path = await import('node:path');
const dir = path.dirname(options.path);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(options.path, Buffer.from(base64, 'base64'));
await fs.promises.mkdir(dir, { recursive: true });
await fs.promises.writeFile(options.path, Buffer.from(base64, 'base64'));
}
return base64;
}
async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0;
const dy = direction === 'up' ? -amount : direction === 'down' ? amount : 0;
await sendCommand('exec', {
code: `window.scrollBy(${dx}, ${dy})`,
...this._workspaceOpt(),
...this._tabOpt(),
});
const code = scrollJs(direction, amount);
await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise<void> {
const times = options.times ?? 3;
const delayMs = options.delayMs ?? 2000;
const code = `
(async () => {
for (let i = 0; i < ${times}; i++) {
const lastHeight = document.body.scrollHeight;
window.scrollTo(0, lastHeight);
await new Promise(resolve => {
let timeoutId;
const observer = new MutationObserver(() => {
if (document.body.scrollHeight > lastHeight) {
clearTimeout(timeoutId);
observer.disconnect();
setTimeout(resolve, 100);
}
});
observer.observe(document.body, { childList: true, subtree: true });
timeoutId = setTimeout(() => { observer.disconnect(); resolve(null); }, ${delayMs});
});
}
})()
`;
const code = autoScrollJs(times, delayMs);
await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
}
+4 -4
View File
@@ -99,12 +99,12 @@ async function loadFromManifest(manifestPath: string, clisDir: string): Promise<
async function discoverClisFromFs(dir: string): Promise<void> {
try { await fs.promises.access(dir); } catch { return; }
const promises: Promise<any>[] = [];
const sites = await fs.promises.readdir(dir);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const site of sites) {
for (const entry of entries) {
if (!entry.isDirectory()) continue;
const site = entry.name;
const siteDir = path.join(dir, site);
const stat = await fs.promises.stat(siteDir);
if (!stat.isDirectory()) continue;
const files = await fs.promises.readdir(siteDir);
for (const file of files) {
const filePath = path.join(siteDir, file);
+2 -2
View File
@@ -280,7 +280,7 @@ export async function exploreUrl(
// Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe
const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody);
for (const ep of jsonEndpoints.slice(0, 5)) {
await Promise.allSettled(jsonEndpoints.slice(0, 5).map(async (ep) => {
try {
const body = await page.evaluate(`async () => {
let iframe = null;
@@ -302,7 +302,7 @@ export async function exploreUrl(
if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} }
else if (body && typeof body === 'object') ep.responseBody = body;
} catch {}
}
}));
// Step 6: Detect framework
let framework: Record<string, boolean> = {};
+3 -2
View File
@@ -27,6 +27,7 @@ export function generateInterceptorJs(
return `
() => {
window.${arr} = window.${arr} || [];
window.${arr}_errors = window.${arr}_errors || [];
const __pattern = ${patternExpr};
if (!window.${guard}) {
@@ -43,7 +44,7 @@ export function generateInterceptorJs(
const clone = response.clone();
const json = await clone.json();
window.${arr}.push(json);
} catch(e) {}
} catch(e) { window.${arr}_errors.push({ url: reqUrl, error: String(e) }); }
}
return response;
};
@@ -61,7 +62,7 @@ export function generateInterceptorJs(
this.addEventListener('load', function() {
try {
window.${arr}.push(JSON.parse(this.responseText));
} catch(e) {}
} catch(e) { window.${arr}_errors.push({ url: this.__opencli_url, error: String(e) }); }
});
}
return __origSend.apply(this, arguments);
+1 -1
View File
@@ -84,7 +84,7 @@ function renderCsv(data: unknown, opts: RenderOptions): void {
for (const row of rows) {
console.log(columns.map(c => {
const v = String((row as Record<string, unknown>)[c] ?? '');
return v.includes(',') || v.includes('"') || v.includes('\n')
return v.includes(',') || v.includes('"') || v.includes('\n') || v.includes('\r')
? `"${v.replace(/"/g, '""')}"` : v;
}).join(','));
}
+4 -2
View File
@@ -52,13 +52,15 @@ export interface CliOptions extends Partial<Omit<CliCommand, 'args' | 'descripti
const _registry = new Map<string, CliCommand>();
export function cli(opts: CliOptions): CliCommand {
const strategy = opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE);
const browser = opts.browser ?? (strategy !== Strategy.PUBLIC);
const cmd: CliCommand = {
site: opts.site,
name: opts.name,
description: opts.description ?? '',
domain: opts.domain,
strategy: opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE),
browser: opts.browser ?? (opts.strategy === Strategy.PUBLIC ? false : true),
strategy,
browser,
args: opts.args ?? [],
columns: opts.columns,
func: opts.func,