okooooooo/opencli
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: eliminate code duplication, improve type safety, add tests

Browse Source 
- NEW: src/interceptor.ts — unified XHR/Fetch interceptor (was duplicated 3x)
- NEW: src/version.ts — centralized PKG_VERSION (was duplicated 2x)
- NEW: src/constants.ts — shared VOLATILE_PARAMS, FIELD_ROLES etc.
- NEW: src/engine.test.ts, src/registry.test.ts — 14 new unit tests

- browser.ts: use shared normalizeEval, interceptor, withTimeoutMs, PKG_VERSION
- intercept.ts, tap.ts: use shared interceptor generators
- cascade.ts: extract shared buildFetchProbeJs (90% dedup)
- engine.ts: use InternalCliCommand (no more 'as any' casts)
- executor.ts: remove all 15 'as StepHandler' type casts
- runtime.ts: add withTimeoutMs, IBrowserFactory interface
- registry.ts: add InternalCliCommand type for internal fields
- validate.ts: add pipeline step name validation
- output.ts: remove 'null as any' and '.filter(() => true)' hacks
- explore.ts, synthesize.ts: use shared constants
- docs: fix V2EX commands (3→6), SKILL.md version, verify example

Tests: 88 passed (was 74), tsc --noEmit: 0 errors
This commit is contained in:
jackwener committed 2026-03-16 13:22:12 +08:00
1 parent b0c51ddf19
commit 34a9bff2b3
23 files changed
+562 -332
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+1 -1
View File
@@ -132,7 +132,7 @@ npm install -g @jackwener/opencli@latest
| **smzdm** | `search` | 🔐 Browser |
| **ctrip** | `search` | 🔐 Browser |
| **github** | `search` | 🌐 Public |
| **v2ex** | `hot` `latest` `topic` | 🌐 Public |
| **v2ex** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 Public / 🔐 Browser |
| **hackernews** | `top` | 🌐 Public |
| **bbc** | `news` | 🌐 Public |
README.zh-CN.md
+1 -1
View File
@@ -132,7 +132,7 @@ npm install -g @jackwener/opencli@latest
| **smzdm** | `search` | 🔐 浏览器 |
| **ctrip** | `search` | 🔐 浏览器 |
| **github** | `search` | 🌐 公共 API |
| **v2ex** | `hot` `latest` `topic` | 🌐 公共 API |
| **v2ex** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 公共 API / 🔐 浏览器 |
| **hackernews** | `top` | 🌐 公共 API |
| **bbc** | `news` | 🌐 公共 API |
SKILL.md
+7 -4
View File
@@ -1,7 +1,7 @@
---
name: opencli
description: "OpenCLI — Make any website your CLI. Zero risk, AI-powered, reuse Chrome login."
version: 0.5.0
version: 0.5.1
author: jackwener
tags: [cli, browser, web, mcp, playwright, bilibili, zhihu, twitter, github, v2ex, hackernews, reddit, xiaohongshu, xueqiu, AI, agent]
---
@@ -95,10 +95,13 @@ opencli reddit frontpage --limit 10 # 首页
opencli reddit search --keyword "AI" # 搜索
opencli reddit subreddit --name rust # 子版块浏览
# V2EX (public)
# V2EX (public + browser)
opencli v2ex hot --limit 10 # 热门话题
opencli v2ex latest --limit 10 # 最新话题
opencli v2ex topic --id 1024 # 主题详情
opencli v2ex daily # 每日签到 (browser)
opencli v2ex me # 我的信息 (browser)
opencli v2ex notifications --limit 10 # 通知 (browser)
# Hacker News (public)
opencli hackernews top --limit 10 # Top stories
@@ -156,8 +159,8 @@ opencli cascade <api-url>
# Explore with interactive fuzzing (click buttons to trigger lazy APIs)
opencli explore <url> --auto --click "字幕,CC,评论"
# Verify: smoke-test a generated adapter
opencli verify <site/name> --smoke
# Verify: validate adapter definitions
opencli verify
```
## Output Formats
src/browser.test.ts
+1 -1
View File
@@ -69,7 +69,7 @@ describe('browser helpers', () => {
});
it('times out slow promises', async () => {
await expect(__test__.withTimeout(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
await expect(__test__.withTimeoutMs(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
});
});
src/browser.ts
+14 -85
View File
@@ -10,10 +10,10 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { formatSnapshot } from './snapshotFormatter.js';
// Read version from package.json (single source of truth)
const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
import { PKG_VERSION } from './version.js';
import { normalizeEvaluateSource } from './pipeline/template.js';
import { generateInterceptorJs, generateReadInterceptedJs } from './interceptor.js';
import { withTimeoutMs } from './runtime.js';
const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
const STDERR_BUFFER_LIMIT = 16 * 1024;
@@ -158,23 +158,10 @@ export class Page implements IPage {
async evaluate(js: string): Promise<any> {
// Normalize IIFE format to function format expected by MCP browser_evaluate
const normalized = this.normalizeEval(js);
const normalized = normalizeEvaluateSource(js);
return this.call('tools/call', { name: 'browser_evaluate', arguments: { function: normalized } });
}
private normalizeEval(source: string): string {
const s = source.trim();
if (!s) return '() => undefined';
// IIFE: (async () => {...})() → wrap as () => (...)
if (s.startsWith('(') && s.endsWith(')()')) return `() => (${s})`;
// Already a function/arrow
if (/^(async\s+)?\([^)]*\)\s*=>/.test(s)) return s;
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(s)) return s;
if (s.startsWith('function ') || s.startsWith('async function ')) return s;
// Raw expression → wrap
return `() => (${s})`;
}
async snapshot(opts: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean } = {}): Promise<any> {
const raw = await this.call('tools/call', { name: 'browser_snapshot', arguments: {} });
if (opts.raw) return raw;
@@ -263,57 +250,15 @@ export class Page implements IPage {
}
async installInterceptor(pattern: string): Promise<void> {
const js = `
() => {
window.__opencli_xhr = window.__opencli_xhr || [];
window.__opencli_patterns = window.__opencli_patterns || [];
if (!window.__opencli_patterns.includes('${pattern}')) {
window.__opencli_patterns.push('${pattern}');
}
if (!window.__patched_xhr) {
const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
const XHR = XMLHttpRequest.prototype;
const open = XHR.open;
const send = XHR.send;
XHR.open = function(method, url) {
this._url = url;
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
};
XHR.send = function() {
this.addEventListener('load', function() {
if (checkMatch(this._url)) {
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
}
});
return send.apply(this, arguments);
};
const origFetch = window.fetch;
window.fetch = async function(...args) {
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
const res = await origFetch.apply(this, args);
setTimeout(async () => {
try {
if (checkMatch(u)) {
const clone = res.clone();
const j = await clone.json();
window.__opencli_xhr.push({url: u, data: j});
}
} catch(e) {}
}, 0);
return res;
};
window.__patched_xhr = true;
}
}
`;
await this.evaluate(js);
await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
arrayName: '__opencli_xhr',
patchGuard: '__opencli_interceptor_patched',
}));
}
async getInterceptedRequests(): Promise<any[]> {
return (await this.evaluate('() => window.__opencli_xhr')) || [];
const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
return result || [];
}
}
@@ -530,7 +475,7 @@ export class PlaywrightMCP {
// Use tabs as a readiness probe and for tab cleanup bookkeeping.
debugLog('Fetching initial tabs count...');
withTimeout(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
withTimeoutMs(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
debugLog(`Tabs response: ${typeof tabs === 'string' ? tabs : JSON.stringify(tabs)}`);
this._initialTabIdentities = extractTabIdentities(tabs);
settleSuccess(page);
@@ -555,7 +500,7 @@ export class PlaywrightMCP {
// Extension mode opens bridge/session tabs that we can clean up best-effort.
if (this._page && this._proc && !this._proc.killed) {
try {
const tabs = await withTimeout(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
const tabs = await withTimeoutMs(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
const tabEntries = extractTabEntries(tabs);
const tabsToClose = diffTabIndexes(this._initialTabIdentities, tabEntries);
for (const index of tabsToClose) {
@@ -672,29 +617,13 @@ function buildMcpArgs(input: { mcpPath: string; executablePath?: string | null }
return args;
}
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
promise.then(
(value) => {
clearTimeout(timer);
resolve(value);
},
(error) => {
clearTimeout(timer);
reject(error);
},
);
});
}
export const __test__ = {
createJsonRpcRequest,
extractTabEntries,
diffTabIndexes,
appendLimited,
buildMcpArgs,
withTimeout,
withTimeoutMs,
};
function findMcpServerPath(): string | null {
src/cascade.ts
+47 -75
View File
@@ -37,6 +37,49 @@ interface CascadeResult {
confidence: number;
}
/**
* Build the JavaScript source for a fetch probe.
* Shared logic for PUBLIC, COOKIE, and HEADER strategies.
*/
function buildFetchProbeJs(url: string, opts: {
credentials?: boolean;
extractCsrf?: boolean;
}): string {
const credentialsLine = opts.credentials ? `credentials: 'include',` : '';
const headerSetup = opts.extractCsrf
? `
const cookies = document.cookie.split(';').map(c => c.trim());
const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
const headers = {};
if (csrf) { headers['X-Csrf-Token'] = csrf; headers['X-XSRF-Token'] = csrf; }
`
: 'const headers = {};';
return `
async () => {
try {
${headerSetup}
const resp = await fetch(${JSON.stringify(url)}, {
${credentialsLine}
headers
});
const status = resp.status;
if (!resp.ok) return { status, ok: false };
const text = await resp.text();
let hasData = false;
try {
const json = JSON.parse(text);
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
typeof json === 'object' && Object.keys(json).length > 0);
// Check for API-level error codes (common in Chinese sites)
if (json.code !== undefined && json.code !== 0) hasData = false;
} catch {}
return { status, ok: true, hasData, preview: text.slice(0, 200) };
} catch (e) { return { ok: false, error: e.message }; }
}
`;
}
/**
* Probe an endpoint with a specific strategy.
* Returns whether the probe succeeded and basic response info.
@@ -45,32 +88,14 @@ export async function probeEndpoint(
page: IPage,
url: string,
strategy: Strategy,
opts: { timeout?: number } = {},
_opts: { timeout?: number } = {},
): Promise<ProbeResult> {
const result: ProbeResult = { strategy, success: false };
try {
switch (strategy) {
case Strategy.PUBLIC: {
// Try direct fetch without browser (no credentials)
const js = `
async () => {
try {
const resp = await fetch(${JSON.stringify(url)});
const status = resp.status;
if (!resp.ok) return { status, ok: false };
const text = await resp.text();
let hasData = false;
try {
const json = JSON.parse(text);
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
typeof json === 'object' && Object.keys(json).length > 0);
} catch {}
return { status, ok: true, hasData, preview: text.slice(0, 200) };
} catch (e) { return { ok: false, error: e.message }; }
}
`;
const resp = await page.evaluate(js);
const resp = await page.evaluate(buildFetchProbeJs(url, {}));
result.statusCode = resp?.status;
result.success = resp?.ok && resp?.hasData;
result.hasData = resp?.hasData;
@@ -79,27 +104,7 @@ export async function probeEndpoint(
}
case Strategy.COOKIE: {
// Fetch with credentials: 'include' (uses browser cookies)
const js = `
async () => {
try {
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
const status = resp.status;
if (!resp.ok) return { status, ok: false };
const text = await resp.text();
let hasData = false;
try {
const json = JSON.parse(text);
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
typeof json === 'object' && Object.keys(json).length > 0);
// Check for API-level error codes (common in Chinese sites)
if (json.code !== undefined && json.code !== 0) hasData = false;
} catch {}
return { status, ok: true, hasData, preview: text.slice(0, 200) };
} catch (e) { return { ok: false, error: e.message }; }
}
`;
const resp = await page.evaluate(js);
const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true }));
result.statusCode = resp?.status;
result.success = resp?.ok && resp?.hasData;
result.hasData = resp?.hasData;
@@ -108,39 +113,7 @@ export async function probeEndpoint(
}
case Strategy.HEADER: {
// Fetch with credentials + try to extract common auth headers
const js = `
async () => {
try {
// Try to extract CSRF tokens from cookies
const cookies = document.cookie.split(';').map(c => c.trim());
const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
const headers = {};
if (csrf) {
headers['X-Csrf-Token'] = csrf;
headers['X-XSRF-Token'] = csrf;
}
const resp = await fetch(${JSON.stringify(url)}, {
credentials: 'include',
headers
});
const status = resp.status;
if (!resp.ok) return { status, ok: false };
const text = await resp.text();
let hasData = false;
try {
const json = JSON.parse(text);
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
typeof json === 'object' && Object.keys(json).length > 0);
if (json.code !== undefined && json.code !== 0) hasData = false;
} catch {}
return { status, ok: true, hasData, preview: text.slice(0, 200) };
} catch (e) { return { ok: false, error: e.message }; }
}
`;
const resp = await page.evaluate(js);
const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true, extractCsrf: true }));
result.statusCode = resp?.status;
result.success = resp?.ok && resp?.hasData;
result.hasData = resp?.hasData;
@@ -151,7 +124,6 @@ export async function probeEndpoint(
case Strategy.INTERCEPT:
case Strategy.UI:
// These require specific implementation per-site
// Mark as needing manual implementation
result.success = false;
result.error = `Strategy ${strategy} requires site-specific implementation`;
break;
src/constants.ts
+35
View File
@@ -0,0 +1,35 @@
/**
* Shared constants used across explore, synthesize, and pipeline modules.
*/
/** URL query params that are volatile/ephemeral and should be stripped from patterns */
export const VOLATILE_PARAMS = new Set([
'w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign',
]);
/** Search-related query parameter names */
export const SEARCH_PARAMS = new Set([
'q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w',
]);
/** Pagination-related query parameter names */
export const PAGINATION_PARAMS = new Set([
'page', 'pn', 'offset', 'cursor', 'next', 'page_num',
]);
/** Limit/page-size query parameter names */
export const LIMIT_PARAMS = new Set([
'limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num',
]);
/** Field role → common API field names mapping */
export const FIELD_ROLES: Record<string, string[]> = {
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
};
src/engine.test.ts
+77
View File
@@ -0,0 +1,77 @@
/**
* Tests for engine.ts: CLI discovery and command execution.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { discoverClis, executeCommand } from './engine.js';
import { getRegistry, cli, Strategy } from './registry.js';
describe('discoverClis', () => {
it('handles non-existent directories gracefully', async () => {
// Should not throw for missing directories
await expect(discoverClis('/tmp/nonexistent-opencli-test-dir')).resolves.not.toThrow();
});
});
describe('executeCommand', () => {
it('executes a command with func', async () => {
const cmd = cli({
site: 'test-engine',
name: 'func-test',
description: 'test command with func',
browser: false,
strategy: Strategy.PUBLIC,
func: async (_page, kwargs) => {
return [{ title: kwargs.query ?? 'default' }];
},
});
const result = await executeCommand(cmd, null, { query: 'hello' });
expect(result).toEqual([{ title: 'hello' }]);
});
it('executes a command with pipeline', async () => {
const cmd = cli({
site: 'test-engine',
name: 'pipe-test',
description: 'test command with pipeline',
browser: false,
strategy: Strategy.PUBLIC,
pipeline: [
{ evaluate: '() => [{ n: 1 }, { n: 2 }, { n: 3 }]' },
{ limit: '2' },
],
});
// Pipeline commands require page for evaluate step, so we'll test the error path
await expect(executeCommand(cmd, null, {})).rejects.toThrow();
});
it('throws for command with no func or pipeline', async () => {
const cmd = cli({
site: 'test-engine',
name: 'empty-test',
description: 'empty command',
browser: false,
});
await expect(executeCommand(cmd, null, {})).rejects.toThrow('has no func or pipeline');
});
it('passes debug flag to func', async () => {
let receivedDebug = false;
const cmd = cli({
site: 'test-engine',
name: 'debug-test',
description: 'debug test',
browser: false,
func: async (_page, _kwargs, debug) => {
receivedDebug = debug ?? false;
return [];
},
});
await executeCommand(cmd, null, {}, true);
expect(receivedDebug).toBe(true);
});
});
src/engine.ts
+5 -5
View File
@@ -11,7 +11,7 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import yaml from 'js-yaml';
import { type CliCommand, type Arg, Strategy, registerCommand } from './registry.js';
import { type CliCommand, type InternalCliCommand, type Arg, Strategy, registerCommand } from './registry.js';
import type { IPage } from './types.js';
import { executePipeline } from './pipeline.js';
@@ -66,7 +66,7 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
// The actual module is loaded lazily on first executeCommand().
const strategy = (Strategy as any)[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE;
const modulePath = path.resolve(clisDir, entry.modulePath);
const cmd: CliCommand = {
const cmd: InternalCliCommand = {
site: entry.site,
name: entry.name,
description: entry.description ?? '',
@@ -77,7 +77,6 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
columns: entry.columns,
timeoutSeconds: entry.timeout,
source: modulePath,
// Mark as lazy — executeCommand will load the module before running
_lazy: true,
_modulePath: modulePath,
};
@@ -170,8 +169,9 @@ export async function executeCommand(
debug: boolean = false,
): Promise<any> {
// Lazy-load TS module on first execution
if ((cmd as any)._lazy && (cmd as any)._modulePath) {
const modulePath = (cmd as any)._modulePath;
const internal = cmd as InternalCliCommand;
if (internal._lazy && internal._modulePath) {
const modulePath = internal._modulePath;
if (!_loadedModules.has(modulePath)) {
try {
await import(`file://${modulePath}`);
src/explore.ts
+2 -15
View File
@@ -9,6 +9,7 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
// ── Site name detection ────────────────────────────────────────────────────
@@ -43,21 +44,7 @@ export function slugify(value: string): string {
// ── Field & capability inference ───────────────────────────────────────────
const FIELD_ROLES: Record<string, string[]> = {
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
};
const SEARCH_PARAMS = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w']);
const PAGINATION_PARAMS = new Set(['page', 'pn', 'offset', 'cursor', 'next', 'page_num']);
const LIMIT_PARAMS = new Set(['limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num']);
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign']);
// (constants now imported from constants.ts)
// ── Network analysis ───────────────────────────────────────────────────────
src/interceptor.ts
+153
View File
@@ -0,0 +1,153 @@
/**
* Shared XHR/Fetch interceptor JavaScript generators.
*
* Provides a single source of truth for monkey-patching browser
* fetch() and XMLHttpRequest to capture API responses matching
* a URL pattern. Used by:
* - Page.installInterceptor() (browser.ts)
* - stepIntercept (pipeline/steps/intercept.ts)
* - stepTap (pipeline/steps/tap.ts)
*/
/**
* Generate JavaScript source that installs a fetch/XHR interceptor.
* Captured responses are pushed to `window.__opencli_intercepted`.
*
* @param patternExpr - JS expression resolving to a URL substring to match (e.g. a JSON.stringify'd string)
* @param opts.arrayName - Global array name for captured data (default: '__opencli_intercepted')
* @param opts.patchGuard - Global boolean name to prevent double-patching (default: '__opencli_interceptor_patched')
*/
export function generateInterceptorJs(
patternExpr: string,
opts: { arrayName?: string; patchGuard?: string } = {},
): string {
const arr = opts.arrayName ?? '__opencli_intercepted';
const guard = opts.patchGuard ?? '__opencli_interceptor_patched';
return `
() => {
window.${arr} = window.${arr} || [];
const __pattern = ${patternExpr};
if (!window.${guard}) {
const __checkMatch = (url) => __pattern && url.includes(__pattern);
// ── Patch fetch ──
const __origFetch = window.fetch;
window.fetch = async function(...args) {
const reqUrl = typeof args[0] === 'string' ? args[0]
: (args[0] && args[0].url) || '';
const response = await __origFetch.apply(this, args);
if (__checkMatch(reqUrl)) {
try {
const clone = response.clone();
const json = await clone.json();
window.${arr}.push(json);
} catch(e) {}
}
return response;
};
// ── Patch XMLHttpRequest ──
const __XHR = XMLHttpRequest.prototype;
const __origOpen = __XHR.open;
const __origSend = __XHR.send;
__XHR.open = function(method, url) {
this.__opencli_url = String(url);
return __origOpen.apply(this, arguments);
};
__XHR.send = function() {
if (__checkMatch(this.__opencli_url)) {
this.addEventListener('load', function() {
try {
window.${arr}.push(JSON.parse(this.responseText));
} catch(e) {}
});
}
return __origSend.apply(this, arguments);
};
window.${guard} = true;
}
}
`;
}
/**
* Generate JavaScript source to read and clear intercepted data.
*/
export function generateReadInterceptedJs(arrayName: string = '__opencli_intercepted'): string {
return `
() => {
const data = window.${arrayName} || [];
window.${arrayName} = [];
return data;
}
`;
}
/**
* Generate a self-contained tap interceptor for store-action bridge.
* Unlike the global interceptor, this one:
* - Installs temporarily, restores originals in finally block
* - Resolves a promise on first capture (for immediate await)
* - Returns captured data directly
*/
export function generateTapInterceptorJs(patternExpr: string): {
setupVar: string;
capturedVar: string;
promiseVar: string;
resolveVar: string;
fetchPatch: string;
xhrPatch: string;
restorePatch: string;
} {
return {
setupVar: `
let captured = null;
let captureResolve;
const capturePromise = new Promise(r => { captureResolve = r; });
const capturePattern = ${patternExpr};
`,
capturedVar: 'captured',
promiseVar: 'capturePromise',
resolveVar: 'captureResolve',
fetchPatch: `
const origFetch = window.fetch;
window.fetch = async function(...fetchArgs) {
const resp = await origFetch.apply(this, fetchArgs);
try {
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
if (capturePattern && url.includes(capturePattern) && !captured) {
try { captured = await resp.clone().json(); captureResolve(); } catch {}
}
} catch {}
return resp;
};
`,
xhrPatch: `
const origXhrOpen = XMLHttpRequest.prototype.open;
const origXhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.open = function(method, url) {
this.__tapUrl = String(url);
return origXhrOpen.apply(this, arguments);
};
XMLHttpRequest.prototype.send = function(body) {
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
this.addEventListener('load', function() {
if (!captured) {
try { captured = JSON.parse(this.responseText); captureResolve(); } catch {}
}
});
}
return origXhrSend.apply(this, arguments);
};
`,
restorePatch: `
window.fetch = origFetch;
XMLHttpRequest.prototype.open = origXhrOpen;
XMLHttpRequest.prototype.send = origXhrSend;
`,
};
}
src/main.ts
+1 -5
View File
@@ -3,7 +3,6 @@
* opencli — Make any website your CLI. AI-powered.
*/
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -14,16 +13,13 @@ import { type CliCommand, fullName, getRegistry, strategyLabel } from './registr
import { render as renderOutput } from './output.js';
import { PlaywrightMCP } from './browser.js';
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
import { PKG_VERSION } from './version.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
// Read version from package.json (single source of truth)
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
await discoverClis(BUILTIN_CLIS, USER_CLIS);
const program = new Command();
src/output.ts
-4
View File
@@ -40,10 +40,6 @@ function renderTable(data: any, opts: RenderOptions): void {
style: { head: [], border: [] },
wordWrap: true,
wrapOnWordBoundary: true,
colWidths: columns.map((_c, i) => {
if (i === 0) return 6;
return null as any;
}).filter(() => true),
});
for (const row of rows) {
src/pipeline/executor.ts
+15 -15
View File
@@ -15,26 +15,26 @@ export interface PipelineContext {
debug?: boolean;
}
/** Step handler signature */
/** Step handler: all steps conform to (page, params, data, args) => Promise<any> */
type StepHandler = (page: IPage | null, params: any, data: any, args: Record<string, any>) => Promise<any>;
/** Registry of all available step handlers */
const STEP_HANDLERS: Record<string, StepHandler> = {
navigate: stepNavigate as StepHandler,
navigate: stepNavigate,
fetch: stepFetch,
select: stepSelect as StepHandler,
evaluate: stepEvaluate as StepHandler,
snapshot: stepSnapshot as StepHandler,
click: stepClick as StepHandler,
type: stepType as StepHandler,
wait: stepWait as StepHandler,
press: stepPress as StepHandler,
map: stepMap as StepHandler,
filter: stepFilter as StepHandler,
sort: stepSort as StepHandler,
limit: stepLimit as StepHandler,
intercept: stepIntercept as StepHandler,
tap: stepTap as StepHandler,
select: stepSelect,
evaluate: stepEvaluate,
snapshot: stepSnapshot,
click: stepClick,
type: stepType,
wait: stepWait,
press: stepPress,
map: stepMap,
filter: stepFilter,
sort: stepSort,
limit: stepLimit,
intercept: stepIntercept,
tap: stepTap,
};
export async function executePipeline(
src/pipeline/steps/intercept.ts
+4 -55
View File
@@ -4,6 +4,7 @@
import type { IPage } from '../../types.js';
import { render } from '../template.js';
import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
export async function stepIntercept(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any> {
const cfg = typeof params === 'object' ? params : {};
@@ -15,52 +16,7 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
if (!capturePattern) return data;
// Step 1: Inject fetch/XHR interceptor BEFORE trigger
await page.evaluate(`
() => {
window.__opencli_intercepted = window.__opencli_intercepted || [];
const pattern = ${JSON.stringify(capturePattern)};
if (!window.__opencli_fetch_patched) {
const origFetch = window.fetch;
window.fetch = async function(...args) {
const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
const response = await origFetch.apply(this, args);
setTimeout(async () => {
try {
if (reqUrl.includes(pattern)) {
const clone = response.clone();
const json = await clone.json();
window.__opencli_intercepted.push(json);
}
} catch(e) {}
}, 0);
return response;
};
window.__opencli_fetch_patched = true;
}
if (!window.__opencli_xhr_patched) {
const XHR = XMLHttpRequest.prototype;
const open = XHR.open;
const send = XHR.send;
XHR.open = function(method, url, ...args) {
this._reqUrl = url;
return open.call(this, method, url, ...args);
};
XHR.send = function(...args) {
this.addEventListener('load', function() {
try {
if (this._reqUrl && this._reqUrl.includes(pattern)) {
window.__opencli_intercepted.push(JSON.parse(this.responseText));
}
} catch(e) {}
});
return send.apply(this, args);
};
window.__opencli_xhr_patched = true;
}
}
`);
await page.evaluate(generateInterceptorJs(JSON.stringify(capturePattern)));
// Step 2: Execute the trigger action
if (trigger.startsWith('navigate:')) {
@@ -81,16 +37,9 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
await page.wait(Math.min(timeout, 3));
// Step 4: Retrieve captured data
const matchingResponses = await page.evaluate(`
() => {
const data = window.__opencli_intercepted || [];
window.__opencli_intercepted = []; // clear after reading
return data;
}
`);
const matchingResponses = await page.evaluate(generateReadInterceptedJs());
// Step 4: Select from response if specified
// Step 5: Select from response if specified
let result = matchingResponses.length === 1 ? matchingResponses[0] :
matchingResponses.length > 1 ? matchingResponses : data;
src/pipeline/steps/tap.ts
+12 -51
View File
@@ -11,6 +11,7 @@
import type { IPage } from '../../types.js';
import { render } from '../template.js';
import { generateTapInterceptorJs } from '../../interceptor.js';
export async function stepTap(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any> {
const cfg = typeof params === 'object' ? params : {};
@@ -38,53 +39,15 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
: `store[${JSON.stringify(actionName)}]()`;
// Use shared interceptor generator for fetch/XHR patching
const tap = generateTapInterceptorJs(JSON.stringify(capturePattern));
const js = `
async () => {
// ── 1. Setup capture proxy (fetch + XHR dual interception) ──
let captured = null;
let captureResolve;
const capturePromise = new Promise(r => { captureResolve = r; });
const capturePattern = ${JSON.stringify(capturePattern)};
// Intercept fetch API
const origFetch = window.fetch;
window.fetch = async function(...fetchArgs) {
const resp = await origFetch.apply(this, fetchArgs);
try {
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
if (capturePattern && url.includes(capturePattern) && !captured) {
try { captured = await resp.clone().json(); captureResolve(); } catch {}
}
} catch {}
return resp;
};
// Intercept XMLHttpRequest
const origXhrOpen = XMLHttpRequest.prototype.open;
const origXhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.open = function(method, url) {
this.__tapUrl = String(url);
return origXhrOpen.apply(this, arguments);
};
XMLHttpRequest.prototype.send = function(body) {
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
const xhr = this;
const origHandler = xhr.onreadystatechange;
xhr.onreadystatechange = function() {
if (xhr.readyState === 4 && !captured) {
try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {}
}
if (origHandler) origHandler.apply(this, arguments);
};
const origOnload = xhr.onload;
xhr.onload = function() {
if (!captured) { try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {} }
if (origOnload) origOnload.apply(this, arguments);
};
}
return origXhrSend.apply(this, arguments);
};
${tap.setupVar}
${tap.fetchPatch}
${tap.xhrPatch}
try {
// ── 2. Find store ──
@@ -119,19 +82,17 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
await ${actionCall};
// ── 4. Wait for network response ──
if (!captured) {
if (!${tap.capturedVar}) {
const timeoutPromise = new Promise(r => setTimeout(r, ${timeout} * 1000));
await Promise.race([capturePromise, timeoutPromise]);
await Promise.race([${tap.promiseVar}, timeoutPromise]);
}
} finally {
// ── 5. Always restore originals ──
window.fetch = origFetch;
XMLHttpRequest.prototype.open = origXhrOpen;
XMLHttpRequest.prototype.send = origXhrSend;
${tap.restorePatch}
}
if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
return captured${selectChain} ?? captured;
if (!${tap.capturedVar}) return { error: 'No matching response captured for pattern: ' + capturePattern };
return ${tap.capturedVar}${selectChain} ?? ${tap.capturedVar};
}
`;
src/registry.test.ts
+106
View File
@@ -0,0 +1,106 @@
/**
* Tests for registry.ts: Strategy enum, cli() registration, helpers.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { cli, getRegistry, fullName, strategyLabel, registerCommand, Strategy, type CliCommand } from './registry.js';
describe('cli() registration', () => {
it('registers a command and returns it', () => {
const cmd = cli({
site: 'test-registry',
name: 'hello',
description: 'A test command',
strategy: Strategy.PUBLIC,
browser: false,
});
expect(cmd.site).toBe('test-registry');
expect(cmd.name).toBe('hello');
expect(cmd.strategy).toBe(Strategy.PUBLIC);
expect(cmd.browser).toBe(false);
expect(cmd.args).toEqual([]);
});
it('puts registered command in the registry', () => {
cli({
site: 'test-registry',
name: 'registered',
description: 'test',
});
const registry = getRegistry();
expect(registry.has('test-registry/registered')).toBe(true);
});
it('defaults strategy to COOKIE when browser is true', () => {
const cmd = cli({
site: 'test-registry',
name: 'default-strategy',
});
expect(cmd.strategy).toBe(Strategy.COOKIE);
expect(cmd.browser).toBe(true);
});
it('defaults strategy to PUBLIC when browser is false', () => {
const cmd = cli({
site: 'test-registry',
name: 'no-browser',
browser: false,
});
expect(cmd.strategy).toBe(Strategy.PUBLIC);
});
it('overwrites existing command on re-registration', () => {
cli({ site: 'test-registry', name: 'overwrite', description: 'v1' });
cli({ site: 'test-registry', name: 'overwrite', description: 'v2' });
const reg = getRegistry();
expect(reg.get('test-registry/overwrite')?.description).toBe('v2');
});
});
describe('fullName', () => {
it('returns site/name', () => {
const cmd: CliCommand = {
site: 'bilibili', name: 'hot', description: '', args: [],
};
expect(fullName(cmd)).toBe('bilibili/hot');
});
});
describe('strategyLabel', () => {
it('returns strategy string', () => {
const cmd: CliCommand = {
site: 'test', name: 'test', description: '', args: [],
strategy: Strategy.INTERCEPT,
};
expect(strategyLabel(cmd)).toBe('intercept');
});
it('returns public when no strategy set', () => {
const cmd: CliCommand = {
site: 'test', name: 'test', description: '', args: [],
};
expect(strategyLabel(cmd)).toBe('public');
});
});
describe('registerCommand', () => {
it('registers a pre-built command', () => {
const cmd: CliCommand = {
site: 'test-registry',
name: 'direct-reg',
description: 'directly registered',
args: [],
strategy: Strategy.HEADER,
browser: true,
};
registerCommand(cmd);
const reg = getRegistry();
expect(reg.get('test-registry/direct-reg')?.strategy).toBe(Strategy.HEADER);
});
});
src/registry.ts
+4 -1
View File
@@ -34,7 +34,10 @@ export interface CliCommand {
pipeline?: any[];
timeoutSeconds?: number;
source?: string;
/** Internal: lazy-loaded TS module support */
}
/** Internal extension for lazy-loaded TS modules (not exposed in public API) */
export interface InternalCliCommand extends CliCommand {
_lazy?: boolean;
_modulePath?: string;
}
src/runtime.ts
+22 -8
View File
@@ -9,23 +9,37 @@ export const DEFAULT_BROWSER_COMMAND_TIMEOUT = parseInt(process.env.OPENCLI_BROW
export const DEFAULT_BROWSER_EXPLORE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_EXPLORE_TIMEOUT ?? '120', 10);
export const DEFAULT_BROWSER_SMOKE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_SMOKE_TIMEOUT ?? '60', 10);
/**
* Timeout with seconds unit. Used for high-level command timeouts.
*/
export async function runWithTimeout<T>(
promise: Promise<T>,
opts: { timeout: number; label?: string },
): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
reject(new Error(`${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`));
}, opts.timeout * 1000);
return withTimeoutMs(promise, opts.timeout * 1000, `${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`);
}
promise
.then((result) => { clearTimeout(timer); resolve(result); })
.catch((err) => { clearTimeout(timer); reject(err); });
/**
* Timeout with milliseconds unit. Used for low-level internal timeouts.
*/
export function withTimeoutMs<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
promise.then(
(value) => { clearTimeout(timer); resolve(value); },
(error) => { clearTimeout(timer); reject(error); },
);
});
}
/** Interface for browser factory (PlaywrightMCP or test mocks) */
export interface IBrowserFactory {
connect(opts?: { timeout?: number }): Promise<IPage>;
close(): Promise<void>;
}
export async function browserSession<T>(
BrowserFactory: new () => any,
BrowserFactory: new () => IBrowserFactory,
fn: (page: IPage) => Promise<T>,
): Promise<T> {
const mcp = new BrowserFactory();
src/synthesize.ts
+5 -5
View File
@@ -6,12 +6,12 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import yaml from 'js-yaml';
import { VOLATILE_PARAMS, SEARCH_PARAMS, LIMIT_PARAMS, PAGINATION_PARAMS } from './constants.js';
/** Volatile params to strip from generated URLs */
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
/** Renamed aliases for backward compatibility with local references */
const SEARCH_PARAM_NAMES = SEARCH_PARAMS;
const LIMIT_PARAM_NAMES = LIMIT_PARAMS;
const PAGE_PARAM_NAMES = PAGINATION_PARAMS;
export function synthesizeFromExplore(
target: string,
src/validate.ts
+22
View File
@@ -3,6 +3,14 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import yaml from 'js-yaml';
/** All recognized pipeline step names */
const KNOWN_STEP_NAMES = new Set([
'navigate', 'click', 'type', 'wait', 'press', 'snapshot', 'scroll',
'fetch', 'evaluate',
'select', 'map', 'filter', 'sort', 'limit',
'intercept', 'tap',
]);
export function validateClisWithTarget(dirs: string[], target?: string): any {
const results: any[] = [];
let errors = 0; let warnings = 0; let files = 0;
@@ -38,6 +46,20 @@ function validateYamlFile(filePath: string): any {
if (def.pipeline && !Array.isArray(def.pipeline)) errors.push('"pipeline" must be an array');
if (def.columns && !Array.isArray(def.columns)) errors.push('"columns" must be an array');
if (def.args && typeof def.args !== 'object') errors.push('"args" must be an object');
// Validate pipeline step names (catch typos like 'navaigate')
if (Array.isArray(def.pipeline)) {
for (let i = 0; i < def.pipeline.length; i++) {
const step = def.pipeline[i];
if (step && typeof step === 'object') {
const stepKeys = Object.keys(step);
for (const key of stepKeys) {
if (!KNOWN_STEP_NAMES.has(key)) {
warnings.push(`Pipeline step ${i}: unknown step name "${key}" (did you mean one of: ${[...KNOWN_STEP_NAMES].join(', ')}?)`);
}
}
}
}
}
} catch (e: any) { errors.push(`YAML parse error: ${e.message}`); }
return { path: filePath, errors, warnings };
}
src/verify.ts
+10 -1
View File
@@ -1,9 +1,18 @@
/** Verification: validate + smoke. */
/**
* Verification: runs validation and optional smoke test.
*
* The smoke test is intentionally kept as a stub — full browser-based
* smoke testing requires a running browser session and is better suited
* to the `opencli test` command or CI pipelines.
*/
import { validateClisWithTarget, renderValidationReport } from './validate.js';
export async function verifyClis(opts: any): Promise<any> {
const report = validateClisWithTarget([opts.builtinClis, opts.userClis], opts.target);
return { ok: report.ok, validation: report, smoke: null };
}
export function renderVerifyReport(report: any): string {
return renderValidationReport(report.validation);
}
src/version.ts
+18
View File
@@ -0,0 +1,18 @@
/**
* Single source of truth for package version.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
export const PKG_VERSION: string = (() => {
try {
return JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version;
} catch {
return '0.0.0';
}
})();