feat: introduce browser interception and auto-scroll utilities, enable dynamic loading of TypeScript CLIs, and add a JSON template filter

This commit is contained in:
jackwener committed 2026-03-15 15:42:19 +08:00
1 parent 20f7d58dd6
commit 829172fc59
12 files changed
+164 -244

No files matched your search

+43 -68
View File
@@ -51,7 +51,16 @@ opencli bilibili hot -v # 查看已有命令的 pipeline 每步数据流
- **Request Headers**: Cookie? Bearer? 自定义签名头(X-s、X-t)?
- **Response Body**: JSON 结构,特别是数据在哪个路径(`data.items`、`data.list`)
### 1c. 框架检测
### 1c. 高阶 API 发现捷径法则 (Heuristics)
在开始死磕复杂的抓包拦截之前,按照以下优先级进行尝试:
1. **后缀爆破法 (`.json`)**: 像 Reddit 这样复杂的网站,只要在其 URL 后加上 `.json`(例如 `/r/all.json`),就能在带 Cookie 的情况下直接利用 `fetch` 拿到极其干净的 REST 数据(Tier 2 Cookie 策略极速秒杀)。
2. **全局状态查找法 (`__INITIAL_STATE__`)**: 许多服务端渲染 (SSR) 的网站(如小红书、Bilibili)会将首页或详情页的完整数据挂载到全局 window 对象上。与其去拦截网络请求,不如直接 `page.evaluate('() => window.__INITIAL_STATE__')` 获取整个数据树。
3. **框架探测与 Store Action 截断**: 如果站点使用 Vue + Pinia,可以使用 `tap` 步骤调用 action,让前端框架代替你完成复杂的鉴权签名封装。
4. **底层 XHR/Fetch 拦截**: 最后手段,当上述都不行时,使用 TypeScript 适配器进行无侵入式的请求抓取。
### 1d. 框架检测
Explore 自动检测前端框架。如果需要手动确认:
@@ -110,9 +119,9 @@ opencli cascade https://api.example.com/hot
```
你的 pipeline 里有 evaluate 步骤(内嵌 JS 代码)?
→ ✅ 用 TypeScript (src/clis/<site>/<name>.ts),需在 index.ts 注册
→ ✅ 用 TypeScript (src/clis/<site>/<name>.ts),保存即自动动态注册
→ ❌ 纯声明式(navigate + tap + map + limit)?
→ ✅ 用 YAML (src/clis/<site>/<name>.yaml),放入即自动注册
→ ✅ 用 YAML (src/clis/<site>/<name>.yaml),保存即自动注册
```
| 场景 | 选择 | 示例 |
@@ -310,7 +319,7 @@ pipeline:
适用于需要嵌入 JS 代码读取 Pinia state、XHR 拦截、GraphQL、分页、复杂数据转换等场景。
文件路径: `src/clis/<site>/<name>.ts`,还需要在 `src/clis/index.ts` 中 import 注册。
文件路径: `src/clis/<site>/<name>.ts`。文件将会在运行时被动态扫描并注册(切勿在 `index.ts` 中手动 `import`)。
#### Tier 3 — Header 认证(Twitter)
@@ -353,84 +362,54 @@ cli({
});
```
#### Tier 4 — Store Action + XHR 拦截(小红书)
#### Tier 4 — XHR/Fetch 双重拦截 (Twitter/小红书 通用模式)
```typescript
// src/clis/xiaohongshu/search.ts
// src/clis/xiaohongshu/user.ts
import { cli, Strategy } from '../../registry.js';
cli({
site: 'xiaohongshu',
name: 'search',
description: '搜索小红书笔记',
strategy: Strategy.COOKIE, // 实际是 intercept 模式
args: [{ name: 'keyword', required: true }],
columns: ['rank', 'title', 'author', 'likes', 'type'],
name: 'user',
description: '获取用户笔记',
strategy: Strategy.INTERCEPT,
args: [{ name: 'id', required: true }],
columns: ['rank', 'title', 'likes', 'url'],
func: async (page, kwargs) => {
await page.goto('https://www.xiaohongshu.com');
await page.wait(2);
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
await page.wait(5);
const data = await page.evaluate(`
(async () => {
const app = document.querySelector('#app')?.__vue_app__;
const pinia = app?.config?.globalProperties?.$pinia;
if (!pinia?._s) return { error: 'Page not ready' };
// XHR/Fetch 底层拦截:捕获所有包含 'v1/user/posted' 的请求
await page.installInterceptor('v1/user/posted');
const searchStore = pinia._s.get('search');
if (!searchStore) return { error: 'Search store not found' };
// 触发后端 API:模拟人类用户向底部滚动2次
await page.autoScroll({ times: 2, delayMs: 2000 });
// XHR 拦截:捕获 store action 发出的请求
let captured = null;
const origOpen = XMLHttpRequest.prototype.open;
const origSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.open = function(m, u) {
this.__url = u;
return origOpen.apply(this, arguments);
};
XMLHttpRequest.prototype.send = function(b) {
if (this.__url?.includes('search/notes')) {
const x = this;
const orig = x.onreadystatechange;
x.onreadystatechange = function() {
if (x.readyState === 4 && !captured) {
try { captured = JSON.parse(x.responseText); } catch {}
}
if (orig) orig.apply(this, arguments);
};
}
return origSend.apply(this, arguments);
};
// 提取所有被拦截捕获的 JSON 响应体
const requests = await page.getInterceptedRequests();
if (!requests || requests.length === 0) return [];
try {
// 触发 Store Action,让网站自己签名发请求
searchStore.mutateSearchValue('${kwargs.keyword}');
await searchStore.loadMore();
await new Promise(r => setTimeout(r, 800));
} finally {
// 恢复原始 XHR
XMLHttpRequest.prototype.open = origOpen;
XMLHttpRequest.prototype.send = origSend;
let results = [];
for (const req of requests) {
if (req.data?.data?.notes) {
for (const note of req.data.data.notes) {
results.push({
title: note.display_title || '',
likes: note.interact_info?.liked_count || '0',
url: `https://explore/${note.note_id || note.id}`
});
}
}
}
if (!captured?.success) return { error: captured?.msg || 'Search failed' };
return (captured.data?.items || []).map(i => ({
title: i.note_card?.display_title || '',
author: i.note_card?.user?.nickname || '',
likes: i.note_card?.interact_info?.liked_count || '0',
type: i.note_card?.type || '',
}));
})()
`);
if (!Array.isArray(data)) return [];
return data.slice(0, kwargs.limit || 20).map((item, i) => ({
return results.slice(0, 20).map((item, i) => ({
rank: i + 1, ...item,
}));
},
});
```
> **XHR 拦截核心思路**:不自己构造签名,而是劫持网站自己的 `XMLHttpRequest`,让网站的 Store Action 发出正确签名的请求,我们只是"窃听"响应。用完后必须恢复原始方法。
> **拦截核心思路**:不自己构造签名,而是利用 `installInterceptor` 劫持网站自己的 `XMLHttpRequest` 和 `fetch`,让网站发请求,我们直接在底层取出解析好的 `response.json()`。
---
@@ -537,11 +516,7 @@ opencli mysite hot -f csv > data.csv
### TS 适配器
在 `src/clis/index.ts` 添加 import:
```typescript
import './mysite/search.js';
```
放入 `src/clis/<site>/<name>.ts` 即自动加载模块,无需在 `index.ts` 中写入 `import`。
### 验证注册
+4 -2
View File
@@ -13,8 +13,10 @@ A CLI tool that turns **any website** into a command-line interface. **28+ comma
- 🔐 **Account-safe** — Reuses Chrome's logged-in state; your credentials never leave the browser
- 🤖 **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies
- 📝 **Declarative YAML** — Most adapters are ~30 lines of YAML pipeline
- 🔌 **TypeScript escape hatch** — Complex adapters (XHR interception, GraphQL) in TS
- 🚀 **Dynamic Loader** — Simply drop `.ts` or `.yaml` adapters into the `clis/` folder for auto-registration
- 📝 **Dual-Engine Architecture**:
- **YAML Declarative Engine**: Most adapters are minimal ~30 lines of YAML pipeline
- **Native Browser Injection Engine**: Advanced TypeScript utilities (`installInterceptor`, `autoScroll`) for XHR hijacking, GraphQL unwrapping, and store mutation
## 🚀 Quick Start
+4 -2
View File
@@ -14,8 +14,10 @@ OpenCLI 通过 Chrome 浏览器 + [Playwright MCP Bridge](https://github.com/nic
- 🌐 **28+ 命令,16 个站点** — B站、知乎、小红书、Twitter、Reddit、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube
- 🔐 **零风控** — 复用 Chrome 登录态,无需存储任何凭证
- 🤖 **AI 原生** — `explore` 自动发现 API,`synthesize` 生成适配器,`cascade` 探测认证策略
- 📝 **声明式 YAML** — 大部分适配器只需 ~30 行 YAML
- 🔌 **TypeScript 扩展** — 复杂场景(XHR 拦截、GraphQL)可用 TS 编写
- 🚀 **动态加载引擎** — 只需将 `.ts` 或 `.yaml` 适配器放入 `clis/` 文件夹即可自动注册生效
- 📝 **双引擎架构设计**:
- **YAML 声明式引擎**:大部分适配器只需极简的 ~30 行 YAML 声明
- **原生浏览器注入引擎**:提供高级 TS API(`installInterceptor`、`autoScroll`)轻松实现 XHR 劫持、GraphQL 解包及状态库注入
## 🚀 快速开始
+19 -13
View File
@@ -209,7 +209,7 @@ pipeline:
### TypeScript Adapter (programmatic)
Create `src/clis/<site>/<name>.ts` and import in `clis/index.ts`:
Create `src/clis/<site>/<name>.ts`. It will be automatically dynamically loaded (DO NOT manually import it in `index.ts`):
```typescript
import { cli, Strategy } from '../../registry.js';
@@ -217,27 +217,33 @@ import { cli, Strategy } from '../../registry.js';
cli({
site: 'mysite',
name: 'search',
strategy: Strategy.COOKIE,
strategy: Strategy.INTERCEPT, // Or COOKIE
args: [{ name: 'keyword', required: true }],
columns: ['rank', 'title', 'url'],
func: async (page, kwargs) => {
await page.goto('https://www.mysite.com');
const data = await page.evaluate(`
(async () => {
const res = await fetch('/api/search?q=${kwargs.keyword}', {
credentials: 'include'
});
return await res.json();
})()
`);
return data.items.map((item, i) => ({
await page.goto('https://www.mysite.com/search');
// Inject native XHR/Fetch interceptor hook
await page.installInterceptor('/api/search');
// Auto scroll down to trigger lazy loading
await page.autoScroll({ times: 3, delayMs: 2000 });
// Retrieve intercepted JSON payloads
const requests = await page.getInterceptedRequests();
let results = [];
for (const req of requests) {
results.push(...req.data.items);
}
return results.map((item, i) => ({
rank: i + 1, title: item.title, url: item.url,
}));
},
});
```
**When to use TS**: XHR interception (小红书), cookie extraction (Twitter ct0), Wbi signing (Bilibili), auto-pagination, complex data transforms.
**When to use TS**: XHR interception (`page.installInterceptor`), infinite scrolling (`page.autoScroll`), cookie extraction, complex data transforms (like GraphQL unwrapping).
## Pipeline Steps
+63
View File
@@ -135,6 +135,69 @@ export class Page implements IPage {
async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
await this.call('tools/call', { name: 'browser_press_key', arguments: { key: direction === 'down' ? 'PageDown' : 'PageUp' } });
}
async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise<void> {
const times = options.times ?? 3;
const delayMs = options.delayMs ?? 2000;
for (let i = 0; i < times; i++) {
await this.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
await this.wait(delayMs / 1000);
}
}
async installInterceptor(pattern: string): Promise<void> {
const js = `
() => {
window.__opencli_xhr = window.__opencli_xhr || [];
window.__opencli_patterns = window.__opencli_patterns || [];
if (!window.__opencli_patterns.includes('${pattern}')) {
window.__opencli_patterns.push('${pattern}');
}
if (!window.__patched_xhr) {
const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
const XHR = XMLHttpRequest.prototype;
const open = XHR.open;
const send = XHR.send;
XHR.open = function(method, url) {
this._url = url;
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
};
XHR.send = function() {
this.addEventListener('load', function() {
if (checkMatch(this._url)) {
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
}
});
return send.apply(this, arguments);
};
const origFetch = window.fetch;
window.fetch = async function(...args) {
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
const res = await origFetch.apply(this, args);
setTimeout(async () => {
try {
if (checkMatch(u)) {
const clone = res.clone();
const j = await clone.json();
window.__opencli_xhr.push({url: u, data: j});
}
} catch(e) {}
}, 0);
return res;
};
window.__patched_xhr = true;
}
}
`;
await this.evaluate(js);
}
async getInterceptedRequests(): Promise<any[]> {
return (await this.evaluate('() => window.__opencli_xhr')) || [];
}
}
/**
-55
View File
@@ -1,55 +0,0 @@
/**
* Import all TypeScript CLI adapters so they self-register.
*
* Each TS adapter calls cli() on import, which adds itself to the global registry.
*/
// bilibili
import './bilibili/search.js';
import './bilibili/me.js';
import './bilibili/favorite.js';
import './bilibili/history.js';
import './bilibili/feed.js';
import './bilibili/user-videos.js';
import './bilibili/ranking.js';
import './bilibili/dynamic.js';
// github
import './github/search.js';
// zhihu
import './zhihu/question.js';
// xiaohongshu
import './xiaohongshu/search.js';
import './xiaohongshu/user.js';
// bbc
import './bbc/news.js';
// weibo
import './weibo/hot.js';
// boss
import './boss/search.js';
// yahoo-finance
import './yahoo-finance/quote.js';
// reuters
import './reuters/search.js';
// smzdm
import './smzdm/search.js';
// ctrip
import './ctrip/search.js';
// youtube
import './youtube/search.js';
// twitter
import './twitter/search.js';
import './twitter/profile.js';
// reddit
+3 -28
View File
@@ -18,38 +18,13 @@ cli({
await page.wait(5);
// Inject XHR interceptor
await page.evaluate(`
() => {
window.__opencli_xhr = [];
if (!window.__patched_xhr) {
const XHR = XMLHttpRequest.prototype;
const open = XHR.open;
const send = XHR.send;
XHR.open = function(method, url) {
this._url = url;
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
};
XHR.send = function() {
this.addEventListener('load', function() {
if(this._url.includes('SearchTimeline')) {
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
}
});
return send.apply(this, arguments);
};
window.__patched_xhr = true;
}
}
`);
await page.installInterceptor('SearchTimeline');
// Trigger API by scrolling
for (let i = 0; i < 3; i++) {
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
await page.wait(2);
}
await page.autoScroll({ times: 3, delayMs: 2000 });
// Retrieve data
const requests = await page.evaluate('() => window.__opencli_xhr');
const requests = await page.getInterceptedRequests();
if (!requests || requests.length === 0) return [];
let results: any[] = [];
+3 -28
View File
@@ -19,38 +19,13 @@ cli({
await page.wait(5);
// 2. Inject XHR interceptor
await page.evaluate(`
() => {
window.__opencli_xhr = [];
if (!window.__patched_xhr) {
const XHR = XMLHttpRequest.prototype;
const open = XHR.open;
const send = XHR.send;
XHR.open = function(method, url) {
this._url = url;
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
};
XHR.send = function() {
this.addEventListener('load', function() {
if(this._url.includes('SearchTimeline')) {
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
}
});
return send.apply(this, arguments);
};
window.__patched_xhr = true;
}
}
`);
await page.installInterceptor('SearchTimeline');
// 3. Trigger API by scrolling
for (let i = 0; i < 3; i++) {
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
await page.wait(2);
}
await page.autoScroll({ times: 3, delayMs: 2000 });
// 4. Retrieve data
const requests = await page.evaluate('() => window.__opencli_xhr');
const requests = await page.getInterceptedRequests();
if (!requests || requests.length === 0) return [];
let results: any[] = [];
+3 -44
View File
@@ -16,54 +16,13 @@ cli({
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
await page.wait(5);
await page.evaluate(`
() => {
window.__opencli_xhr = [];
if (!window.__patched_xhr) {
const origFetch = window.fetch;
window.fetch = async function(...args) {
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
const res = await origFetch.apply(this, args);
setTimeout(async () => {
try {
if (u.includes('v1/user/posted')) {
const clone = res.clone();
const j = await clone.json();
window.__opencli_xhr.push({url: u, data: j});
}
} catch(e) {}
}, 0);
return res;
};
const XHR = XMLHttpRequest.prototype;
const open = XHR.open;
const send = XHR.send;
XHR.open = function(method, url) {
this._url = url;
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
};
XHR.send = function() {
this.addEventListener('load', function() {
if(this._url.includes('v1/user/posted')) {
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
}
});
return send.apply(this, arguments);
};
window.__patched_xhr = true;
}
}
`);
await page.installInterceptor('v1/user/posted');
// Trigger API by scrolling
for (let i = 0; i < 2; i++) {
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
await page.wait(2);
}
await page.autoScroll({ times: 2, delayMs: 2000 });
// Retrieve data
const requests = await page.evaluate('() => window.__opencli_xhr');
const requests = await page.getInterceptedRequests();
if (!requests || requests.length === 0) return [];
let results: any[] = [];
+10 -1
View File
@@ -9,7 +9,8 @@ import { type CliCommand, type Arg, Strategy, registerCommand } from './registry
import type { IPage } from './types.js';
import { executePipeline } from './pipeline.js';
export function discoverClis(...dirs: string[]): void {
export async function discoverClis(...dirs: string[]): Promise<void> {
const promises: Promise<any>[] = [];
for (const dir of dirs) {
if (!fs.existsSync(dir)) continue;
for (const site of fs.readdirSync(dir)) {
@@ -19,10 +20,18 @@ export function discoverClis(...dirs: string[]): void {
const filePath = path.join(siteDir, file);
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
registerYamlCli(filePath, site);
} else if (file.endsWith('.js')) {
// Dynamic import of compiled adapter modules
promises.push(
import(`file://${filePath}`).catch((err: any) => {
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
})
);
}
}
}
}
await Promise.all(promises);
}
function registerYamlCli(filePath: string, defaultSite: string): void {
+9 -3
View File
@@ -12,7 +12,6 @@ import chalk from 'chalk';
import { discoverClis, executeCommand } from './engine.js';
import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
import { render as renderOutput } from './output.js';
import './clis/index.js';
import { PlaywrightMCP } from './browser.js';
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
@@ -25,7 +24,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
discoverClis(BUILTIN_CLIS, USER_CLIS);
await discoverClis(BUILTIN_CLIS, USER_CLIS);
const program = new Command();
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
@@ -104,8 +103,15 @@ for (const [, cmd] of registry) {
if (cmd.browser) {
result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
} else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
}
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
} catch (err: any) { console.error(chalk.red(`Error: ${err.message ?? err}`)); process.exitCode = 1; }
} catch (err: any) {
if (actionOpts.verbose && err.stack) { console.error(chalk.red(err.stack)); }
else { console.error(chalk.red(`Error: ${err.message ?? err}`)); }
process.exitCode = 1;
}
});
}
+3
View File
@@ -20,4 +20,7 @@ export interface IPage {
networkRequests(includeStatic?: boolean): Promise<any>;
consoleMessages(level?: string): Promise<any>;
scroll(direction?: string, amount?: number): Promise<void>;
autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
installInterceptor(pattern: string): Promise<void>;
getInterceptedRequests(): Promise<any[]>;
}