feat: introduce browser interception and auto-scroll utilities, enable dynamic loading of TypeScript CLIs, and add a JSON template filter
This commit is contained in:
1 parent
20f7d58dd6
commit
829172fc59
12 files changed
+164
-244
No files matched your search
+43
-68
@@ -51,7 +51,16 @@ opencli bilibili hot -v # 查看已有命令的 pipeline 每步数据流
|
||||
- **Request Headers**: Cookie? Bearer? 自定义签名头(X-s、X-t)?
|
||||
- **Response Body**: JSON 结构,特别是数据在哪个路径(`data.items`、`data.list`)
|
||||
|
||||
### 1c. 框架检测
|
||||
### 1c. 高阶 API 发现捷径法则 (Heuristics)
|
||||
|
||||
在开始死磕复杂的抓包拦截之前,按照以下优先级进行尝试:
|
||||
|
||||
1. **后缀爆破法 (`.json`)**: 像 Reddit 这样复杂的网站,只要在其 URL 后加上 `.json`(例如 `/r/all.json`),就能在带 Cookie 的情况下直接利用 `fetch` 拿到极其干净的 REST 数据(Tier 2 Cookie 策略极速秒杀)。
|
||||
2. **全局状态查找法 (`__INITIAL_STATE__`)**: 许多服务端渲染 (SSR) 的网站(如小红书、Bilibili)会将首页或详情页的完整数据挂载到全局 window 对象上。与其去拦截网络请求,不如直接 `page.evaluate('() => window.__INITIAL_STATE__')` 获取整个数据树。
|
||||
3. **框架探测与 Store Action 截断**: 如果站点使用 Vue + Pinia,可以使用 `tap` 步骤调用 action,让前端框架代替你完成复杂的鉴权签名封装。
|
||||
4. **底层 XHR/Fetch 拦截**: 最后手段,当上述都不行时,使用 TypeScript 适配器进行无侵入式的请求抓取。
|
||||
|
||||
### 1d. 框架检测
|
||||
|
||||
Explore 自动检测前端框架。如果需要手动确认:
|
||||
|
||||
@@ -110,9 +119,9 @@ opencli cascade https://api.example.com/hot
|
||||
|
||||
```
|
||||
你的 pipeline 里有 evaluate 步骤(内嵌 JS 代码)?
|
||||
→ ✅ 用 TypeScript (src/clis/<site>/<name>.ts),需在 index.ts 注册
|
||||
→ ✅ 用 TypeScript (src/clis/<site>/<name>.ts),保存即自动动态注册
|
||||
→ ❌ 纯声明式(navigate + tap + map + limit)?
|
||||
→ ✅ 用 YAML (src/clis/<site>/<name>.yaml),放入即自动注册
|
||||
→ ✅ 用 YAML (src/clis/<site>/<name>.yaml),保存即自动注册
|
||||
```
|
||||
|
||||
| 场景 | 选择 | 示例 |
|
||||
@@ -310,7 +319,7 @@ pipeline:
|
||||
|
||||
适用于需要嵌入 JS 代码读取 Pinia state、XHR 拦截、GraphQL、分页、复杂数据转换等场景。
|
||||
|
||||
文件路径: `src/clis/<site>/<name>.ts`,还需要在 `src/clis/index.ts` 中 import 注册。
|
||||
文件路径: `src/clis/<site>/<name>.ts`。文件将会在运行时被动态扫描并注册(切勿在 `index.ts` 中手动 `import`)。
|
||||
|
||||
#### Tier 3 — Header 认证(Twitter)
|
||||
|
||||
@@ -353,84 +362,54 @@ cli({
|
||||
});
|
||||
```
|
||||
|
||||
#### Tier 4 — Store Action + XHR 拦截(小红书)
|
||||
#### Tier 4 — XHR/Fetch 双重拦截 (Twitter/小红书 通用模式)
|
||||
|
||||
```typescript
|
||||
// src/clis/xiaohongshu/search.ts
|
||||
// src/clis/xiaohongshu/user.ts
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'xiaohongshu',
|
||||
name: 'search',
|
||||
description: '搜索小红书笔记',
|
||||
strategy: Strategy.COOKIE, // 实际是 intercept 模式
|
||||
args: [{ name: 'keyword', required: true }],
|
||||
columns: ['rank', 'title', 'author', 'likes', 'type'],
|
||||
name: 'user',
|
||||
description: '获取用户笔记',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
args: [{ name: 'id', required: true }],
|
||||
columns: ['rank', 'title', 'likes', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.xiaohongshu.com');
|
||||
await page.wait(2);
|
||||
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
|
||||
await page.wait(5);
|
||||
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const app = document.querySelector('#app')?.__vue_app__;
|
||||
const pinia = app?.config?.globalProperties?.$pinia;
|
||||
if (!pinia?._s) return { error: 'Page not ready' };
|
||||
// XHR/Fetch 底层拦截:捕获所有包含 'v1/user/posted' 的请求
|
||||
await page.installInterceptor('v1/user/posted');
|
||||
|
||||
const searchStore = pinia._s.get('search');
|
||||
if (!searchStore) return { error: 'Search store not found' };
|
||||
// 触发后端 API:模拟人类用户向底部滚动2次
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// XHR 拦截:捕获 store action 发出的请求
|
||||
let captured = null;
|
||||
const origOpen = XMLHttpRequest.prototype.open;
|
||||
const origSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(m, u) {
|
||||
this.__url = u;
|
||||
return origOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(b) {
|
||||
if (this.__url?.includes('search/notes')) {
|
||||
const x = this;
|
||||
const orig = x.onreadystatechange;
|
||||
x.onreadystatechange = function() {
|
||||
if (x.readyState === 4 && !captured) {
|
||||
try { captured = JSON.parse(x.responseText); } catch {}
|
||||
}
|
||||
if (orig) orig.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
return origSend.apply(this, arguments);
|
||||
};
|
||||
// 提取所有被拦截捕获的 JSON 响应体
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
try {
|
||||
// 触发 Store Action,让网站自己签名发请求
|
||||
searchStore.mutateSearchValue('${kwargs.keyword}');
|
||||
await searchStore.loadMore();
|
||||
await new Promise(r => setTimeout(r, 800));
|
||||
} finally {
|
||||
// 恢复原始 XHR
|
||||
XMLHttpRequest.prototype.open = origOpen;
|
||||
XMLHttpRequest.prototype.send = origSend;
|
||||
let results = [];
|
||||
for (const req of requests) {
|
||||
if (req.data?.data?.notes) {
|
||||
for (const note of req.data.data.notes) {
|
||||
results.push({
|
||||
title: note.display_title || '',
|
||||
likes: note.interact_info?.liked_count || '0',
|
||||
url: `https://explore/${note.note_id || note.id}`
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!captured?.success) return { error: captured?.msg || 'Search failed' };
|
||||
return (captured.data?.items || []).map(i => ({
|
||||
title: i.note_card?.display_title || '',
|
||||
author: i.note_card?.user?.nickname || '',
|
||||
likes: i.note_card?.interact_info?.liked_count || '0',
|
||||
type: i.note_card?.type || '',
|
||||
}));
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!Array.isArray(data)) return [];
|
||||
return data.slice(0, kwargs.limit || 20).map((item, i) => ({
|
||||
return results.slice(0, 20).map((item, i) => ({
|
||||
rank: i + 1, ...item,
|
||||
}));
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
> **XHR 拦截核心思路**:不自己构造签名,而是劫持网站自己的 `XMLHttpRequest`,让网站的 Store Action 发出正确签名的请求,我们只是"窃听"响应。用完后必须恢复原始方法。
|
||||
> **拦截核心思路**:不自己构造签名,而是利用 `installInterceptor` 劫持网站自己的 `XMLHttpRequest` 和 `fetch`,让网站发请求,我们直接在底层取出解析好的 `response.json()`。
|
||||
|
||||
---
|
||||
|
||||
@@ -537,11 +516,7 @@ opencli mysite hot -f csv > data.csv
|
||||
|
||||
### TS 适配器
|
||||
|
||||
在 `src/clis/index.ts` 添加 import:
|
||||
|
||||
```typescript
|
||||
import './mysite/search.js';
|
||||
```
|
||||
放入 `src/clis/<site>/<name>.ts` 即自动加载模块,无需在 `index.ts` 中写入 `import`。
|
||||
|
||||
### 验证注册
|
||||
|
||||
|
||||
@@ -13,8 +13,10 @@ A CLI tool that turns **any website** into a command-line interface. **28+ comma
|
||||
|
||||
- 🔐 **Account-safe** — Reuses Chrome's logged-in state; your credentials never leave the browser
|
||||
- 🤖 **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies
|
||||
- 📝 **Declarative YAML** — Most adapters are ~30 lines of YAML pipeline
|
||||
- 🔌 **TypeScript escape hatch** — Complex adapters (XHR interception, GraphQL) in TS
|
||||
- 🚀 **Dynamic Loader** — Simply drop `.ts` or `.yaml` adapters into the `clis/` folder for auto-registration
|
||||
- 📝 **Dual-Engine Architecture**:
|
||||
- **YAML Declarative Engine**: Most adapters are minimal ~30 lines of YAML pipeline
|
||||
- **Native Browser Injection Engine**: Advanced TypeScript utilities (`installInterceptor`, `autoScroll`) for XHR hijacking, GraphQL unwrapping, and store mutation
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
|
||||
+4
-2
@@ -14,8 +14,10 @@ OpenCLI 通过 Chrome 浏览器 + [Playwright MCP Bridge](https://github.com/nic
|
||||
- 🌐 **28+ 命令,16 个站点** — B站、知乎、小红书、Twitter、Reddit、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube
|
||||
- 🔐 **零风控** — 复用 Chrome 登录态,无需存储任何凭证
|
||||
- 🤖 **AI 原生** — `explore` 自动发现 API,`synthesize` 生成适配器,`cascade` 探测认证策略
|
||||
- 📝 **声明式 YAML** — 大部分适配器只需 ~30 行 YAML
|
||||
- 🔌 **TypeScript 扩展** — 复杂场景(XHR 拦截、GraphQL)可用 TS 编写
|
||||
- 🚀 **动态加载引擎** — 只需将 `.ts` 或 `.yaml` 适配器放入 `clis/` 文件夹即可自动注册生效
|
||||
- 📝 **双引擎架构设计**:
|
||||
- **YAML 声明式引擎**:大部分适配器只需极简的 ~30 行 YAML 声明
|
||||
- **原生浏览器注入引擎**:提供高级 TS API(`installInterceptor`、`autoScroll`)轻松实现 XHR 劫持、GraphQL 解包及状态库注入
|
||||
|
||||
## 🚀 快速开始
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ pipeline:
|
||||
|
||||
### TypeScript Adapter (programmatic)
|
||||
|
||||
Create `src/clis/<site>/<name>.ts` and import in `clis/index.ts`:
|
||||
Create `src/clis/<site>/<name>.ts`. It will be automatically dynamically loaded (DO NOT manually import it in `index.ts`):
|
||||
|
||||
```typescript
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
@@ -217,27 +217,33 @@ import { cli, Strategy } from '../../registry.js';
|
||||
cli({
|
||||
site: 'mysite',
|
||||
name: 'search',
|
||||
strategy: Strategy.COOKIE,
|
||||
strategy: Strategy.INTERCEPT, // Or COOKIE
|
||||
args: [{ name: 'keyword', required: true }],
|
||||
columns: ['rank', 'title', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.mysite.com');
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const res = await fetch('/api/search?q=${kwargs.keyword}', {
|
||||
credentials: 'include'
|
||||
});
|
||||
return await res.json();
|
||||
})()
|
||||
`);
|
||||
return data.items.map((item, i) => ({
|
||||
await page.goto('https://www.mysite.com/search');
|
||||
|
||||
// Inject native XHR/Fetch interceptor hook
|
||||
await page.installInterceptor('/api/search');
|
||||
|
||||
// Auto scroll down to trigger lazy loading
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// Retrieve intercepted JSON payloads
|
||||
const requests = await page.getInterceptedRequests();
|
||||
|
||||
let results = [];
|
||||
for (const req of requests) {
|
||||
results.push(...req.data.items);
|
||||
}
|
||||
return results.map((item, i) => ({
|
||||
rank: i + 1, title: item.title, url: item.url,
|
||||
}));
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
**When to use TS**: XHR interception (小红书), cookie extraction (Twitter ct0), Wbi signing (Bilibili), auto-pagination, complex data transforms.
|
||||
**When to use TS**: XHR interception (`page.installInterceptor`), infinite scrolling (`page.autoScroll`), cookie extraction, complex data transforms (like GraphQL unwrapping).
|
||||
|
||||
## Pipeline Steps
|
||||
|
||||
|
||||
@@ -135,6 +135,69 @@ export class Page implements IPage {
|
||||
async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
|
||||
await this.call('tools/call', { name: 'browser_press_key', arguments: { key: direction === 'down' ? 'PageDown' : 'PageUp' } });
|
||||
}
|
||||
|
||||
async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise<void> {
|
||||
const times = options.times ?? 3;
|
||||
const delayMs = options.delayMs ?? 2000;
|
||||
for (let i = 0; i < times; i++) {
|
||||
await this.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
||||
await this.wait(delayMs / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
async installInterceptor(pattern: string): Promise<void> {
|
||||
const js = `
|
||||
() => {
|
||||
window.__opencli_xhr = window.__opencli_xhr || [];
|
||||
window.__opencli_patterns = window.__opencli_patterns || [];
|
||||
if (!window.__opencli_patterns.includes('${pattern}')) {
|
||||
window.__opencli_patterns.push('${pattern}');
|
||||
}
|
||||
|
||||
if (!window.__patched_xhr) {
|
||||
const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
|
||||
|
||||
const XHR = XMLHttpRequest.prototype;
|
||||
const open = XHR.open;
|
||||
const send = XHR.send;
|
||||
XHR.open = function(method, url) {
|
||||
this._url = url;
|
||||
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
|
||||
};
|
||||
XHR.send = function() {
|
||||
this.addEventListener('load', function() {
|
||||
if (checkMatch(this._url)) {
|
||||
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
|
||||
}
|
||||
});
|
||||
return send.apply(this, arguments);
|
||||
};
|
||||
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
||||
const res = await origFetch.apply(this, args);
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (checkMatch(u)) {
|
||||
const clone = res.clone();
|
||||
const j = await clone.json();
|
||||
window.__opencli_xhr.push({url: u, data: j});
|
||||
}
|
||||
} catch(e) {}
|
||||
}, 0);
|
||||
return res;
|
||||
};
|
||||
window.__patched_xhr = true;
|
||||
}
|
||||
}
|
||||
`;
|
||||
await this.evaluate(js);
|
||||
}
|
||||
|
||||
async getInterceptedRequests(): Promise<any[]> {
|
||||
return (await this.evaluate('() => window.__opencli_xhr')) || [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
/**
|
||||
* Import all TypeScript CLI adapters so they self-register.
|
||||
*
|
||||
* Each TS adapter calls cli() on import, which adds itself to the global registry.
|
||||
*/
|
||||
|
||||
// bilibili
|
||||
import './bilibili/search.js';
|
||||
import './bilibili/me.js';
|
||||
import './bilibili/favorite.js';
|
||||
import './bilibili/history.js';
|
||||
import './bilibili/feed.js';
|
||||
import './bilibili/user-videos.js';
|
||||
import './bilibili/ranking.js';
|
||||
import './bilibili/dynamic.js';
|
||||
|
||||
// github
|
||||
import './github/search.js';
|
||||
|
||||
// zhihu
|
||||
import './zhihu/question.js';
|
||||
|
||||
// xiaohongshu
|
||||
import './xiaohongshu/search.js';
|
||||
import './xiaohongshu/user.js';
|
||||
|
||||
// bbc
|
||||
import './bbc/news.js';
|
||||
|
||||
// weibo
|
||||
import './weibo/hot.js';
|
||||
|
||||
// boss
|
||||
import './boss/search.js';
|
||||
|
||||
// yahoo-finance
|
||||
import './yahoo-finance/quote.js';
|
||||
|
||||
// reuters
|
||||
import './reuters/search.js';
|
||||
|
||||
// smzdm
|
||||
import './smzdm/search.js';
|
||||
|
||||
// ctrip
|
||||
import './ctrip/search.js';
|
||||
|
||||
// youtube
|
||||
import './youtube/search.js';
|
||||
|
||||
// twitter
|
||||
import './twitter/search.js';
|
||||
import './twitter/profile.js';
|
||||
|
||||
// reddit
|
||||
@@ -18,38 +18,13 @@ cli({
|
||||
await page.wait(5);
|
||||
|
||||
// Inject XHR interceptor
|
||||
await page.evaluate(`
|
||||
() => {
|
||||
window.__opencli_xhr = [];
|
||||
if (!window.__patched_xhr) {
|
||||
const XHR = XMLHttpRequest.prototype;
|
||||
const open = XHR.open;
|
||||
const send = XHR.send;
|
||||
XHR.open = function(method, url) {
|
||||
this._url = url;
|
||||
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
|
||||
};
|
||||
XHR.send = function() {
|
||||
this.addEventListener('load', function() {
|
||||
if(this._url.includes('SearchTimeline')) {
|
||||
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
|
||||
}
|
||||
});
|
||||
return send.apply(this, arguments);
|
||||
};
|
||||
window.__patched_xhr = true;
|
||||
}
|
||||
}
|
||||
`);
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// Trigger API by scrolling
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
||||
await page.wait(2);
|
||||
}
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// Retrieve data
|
||||
const requests = await page.evaluate('() => window.__opencli_xhr');
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
|
||||
@@ -19,38 +19,13 @@ cli({
|
||||
await page.wait(5);
|
||||
|
||||
// 2. Inject XHR interceptor
|
||||
await page.evaluate(`
|
||||
() => {
|
||||
window.__opencli_xhr = [];
|
||||
if (!window.__patched_xhr) {
|
||||
const XHR = XMLHttpRequest.prototype;
|
||||
const open = XHR.open;
|
||||
const send = XHR.send;
|
||||
XHR.open = function(method, url) {
|
||||
this._url = url;
|
||||
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
|
||||
};
|
||||
XHR.send = function() {
|
||||
this.addEventListener('load', function() {
|
||||
if(this._url.includes('SearchTimeline')) {
|
||||
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
|
||||
}
|
||||
});
|
||||
return send.apply(this, arguments);
|
||||
};
|
||||
window.__patched_xhr = true;
|
||||
}
|
||||
}
|
||||
`);
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// 3. Trigger API by scrolling
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
||||
await page.wait(2);
|
||||
}
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data
|
||||
const requests = await page.evaluate('() => window.__opencli_xhr');
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
|
||||
@@ -16,54 +16,13 @@ cli({
|
||||
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
|
||||
await page.wait(5);
|
||||
|
||||
await page.evaluate(`
|
||||
() => {
|
||||
window.__opencli_xhr = [];
|
||||
if (!window.__patched_xhr) {
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
||||
const res = await origFetch.apply(this, args);
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (u.includes('v1/user/posted')) {
|
||||
const clone = res.clone();
|
||||
const j = await clone.json();
|
||||
window.__opencli_xhr.push({url: u, data: j});
|
||||
}
|
||||
} catch(e) {}
|
||||
}, 0);
|
||||
return res;
|
||||
};
|
||||
|
||||
const XHR = XMLHttpRequest.prototype;
|
||||
const open = XHR.open;
|
||||
const send = XHR.send;
|
||||
XHR.open = function(method, url) {
|
||||
this._url = url;
|
||||
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
|
||||
};
|
||||
XHR.send = function() {
|
||||
this.addEventListener('load', function() {
|
||||
if(this._url.includes('v1/user/posted')) {
|
||||
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
|
||||
}
|
||||
});
|
||||
return send.apply(this, arguments);
|
||||
};
|
||||
window.__patched_xhr = true;
|
||||
}
|
||||
}
|
||||
`);
|
||||
await page.installInterceptor('v1/user/posted');
|
||||
|
||||
// Trigger API by scrolling
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
||||
await page.wait(2);
|
||||
}
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// Retrieve data
|
||||
const requests = await page.evaluate('() => window.__opencli_xhr');
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
|
||||
+10
-1
@@ -9,7 +9,8 @@ import { type CliCommand, type Arg, Strategy, registerCommand } from './registry
|
||||
import type { IPage } from './types.js';
|
||||
import { executePipeline } from './pipeline.js';
|
||||
|
||||
export function discoverClis(...dirs: string[]): void {
|
||||
export async function discoverClis(...dirs: string[]): Promise<void> {
|
||||
const promises: Promise<any>[] = [];
|
||||
for (const dir of dirs) {
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
for (const site of fs.readdirSync(dir)) {
|
||||
@@ -19,10 +20,18 @@ export function discoverClis(...dirs: string[]): void {
|
||||
const filePath = path.join(siteDir, file);
|
||||
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
||||
registerYamlCli(filePath, site);
|
||||
} else if (file.endsWith('.js')) {
|
||||
// Dynamic import of compiled adapter modules
|
||||
promises.push(
|
||||
import(`file://${filePath}`).catch((err: any) => {
|
||||
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
function registerYamlCli(filePath: string, defaultSite: string): void {
|
||||
|
||||
+9
-3
@@ -12,7 +12,6 @@ import chalk from 'chalk';
|
||||
import { discoverClis, executeCommand } from './engine.js';
|
||||
import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
|
||||
import { render as renderOutput } from './output.js';
|
||||
import './clis/index.js';
|
||||
import { PlaywrightMCP } from './browser.js';
|
||||
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
||||
|
||||
@@ -25,7 +24,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
|
||||
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
||||
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
|
||||
|
||||
discoverClis(BUILTIN_CLIS, USER_CLIS);
|
||||
await discoverClis(BUILTIN_CLIS, USER_CLIS);
|
||||
|
||||
const program = new Command();
|
||||
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
|
||||
@@ -104,8 +103,15 @@ for (const [, cmd] of registry) {
|
||||
if (cmd.browser) {
|
||||
result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
|
||||
} else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
|
||||
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
|
||||
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
|
||||
}
|
||||
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
|
||||
} catch (err: any) { console.error(chalk.red(`Error: ${err.message ?? err}`)); process.exitCode = 1; }
|
||||
} catch (err: any) {
|
||||
if (actionOpts.verbose && err.stack) { console.error(chalk.red(err.stack)); }
|
||||
else { console.error(chalk.red(`Error: ${err.message ?? err}`)); }
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -20,4 +20,7 @@ export interface IPage {
|
||||
networkRequests(includeStatic?: boolean): Promise<any>;
|
||||
consoleMessages(level?: string): Promise<any>;
|
||||
scroll(direction?: string, amount?: number): Promise<void>;
|
||||
autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
|
||||
installInterceptor(pattern: string): Promise<void>;
|
||||
getInterceptedRequests(): Promise<any[]>;
|
||||
}
|
||||
Reference in new issue
Block a user