diff --git a/README.md b/README.md index e6e9eb5..ea6683f 100644 --- a/README.md +++ b/README.md @@ -1,83 +1,94 @@ # OpenCLI -> **Make any website your CLI.** 操控 Chrome 无风控风险，复用登录，CLI 化全部网站。 +> **Make any website your CLI.** +> Zero risk · Reuse Chrome login · AI-powered discovery -OpenCLI 是一个 AI Native 的 CLI 工具，通过 Chrome 浏览器 + Playwright MCP Bridge 扩展，将任何网站变成命令行工具。 +OpenCLI turns any website into a command-line tool by bridging your Chrome browser through [Playwright MCP](https://github.com/nichochar/playwright-mcp). No passwords stored, no tokens leaked — it just rides your existing browser session. -## ✨ 特性 +## ✨ Highlights -- 🌐 **CLI 化全部网站** — 支持 Bilibili、知乎、GitHub、Twitter、V2EX、Hacker News 等 -- 🔐 **零风控风险** — 复用 Chrome 已登录状态，无需存储密码 -- 🤖 **AI Native** — AI agent 可直接探索网站并自动生成新命令 -- 📝 **声明式 YAML** — 用 YAML 定义 pipeline，无需写代码 -- 🔌 **TypeScript 扩展** — 复杂场景用 TS 编写适配器 +- 🌐 **25+ commands, 13 sites** — Bilibili, 知乎, GitHub, Twitter/X, Reddit, V2EX, 小红书, Hacker News… +- 🔐 **Account-safe** — Reuses Chrome's logged-in state; your credentials never leave the browser +- 🤖 **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds the simplest auth strategy +- 📝 **Declarative YAML** — Most adapters are ~30 lines of YAML pipeline +- 🔌 **TypeScript escape hatch** — Complex adapters (XHR interception, GraphQL) can be written in TS -## 🚀 快速开始 +## 🚀 Quick Start ```bash -# 安装依赖 -cd ~/code/opencli && npm install +npm install +npx tsx src/main.ts list # See all commands -# 列出所有命令 -npx tsx src/main.ts list +# Public APIs (no browser required) +npx tsx src/main.ts hackernews top --limit 5 +npx tsx src/main.ts github search --keyword "rust" +npx tsx src/main.ts v2ex hot --limit 10 -# 公共 API（无需浏览器） -npx tsx src/main.ts hackernews top --limit 10 -npx tsx src/main.ts github search --keyword "typescript" - -# 浏览器命令（需要 Chrome + Playwright MCP Bridge 扩展） -npx tsx src/main.ts bilibili hot --limit 10 -npx tsx src/main.ts zhihu hot --limit 10 -npx tsx src/main.ts twitter trending --limit 10 +# Browser commands (Chrome + MCP Bridge extension required) +npx tsx src/main.ts bilibili hot --limit 5 +npx tsx src/main.ts zhihu hot --limit 5 +npx tsx src/main.ts bilibili search --keyword "AI" --limit 5 ``` -## 📋 前置要求 +## 📋 Prerequisites -浏览器命令需要： -1. Chrome 浏览器正在运行 -2. 安装 [Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm) 扩展 -3. 首次使用时点击扩展图标批准连接 +Browser commands need: +1. **Chrome** running with the target site logged in +2. **[Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm)** extension installed +3. Click the extension icon to approve connection on first use -## 📦 内置命令 +> 💡 Set `PLAYWRIGHT_MCP_EXTENSION_TOKEN` to auto-approve without clicking. -| 站点 | 命令 | 说明 | 模式 | -|------|------|------|------| -| bilibili | hot, search, me, favorite, history, feed, user-videos | 热门 / 搜索 / 个人 / 收藏 / 历史 / 动态 / 投稿 | 🔐 浏览器 | -| zhihu | hot, search, question | 热榜 / 搜索 / 问题详情 | 🔐 浏览器 | -| github | trending, search | Trending / 搜索 | 🔐 / 🌐 公共 | -| twitter | trending | 热门话题 | 🔐 浏览器 | -| v2ex | hot, latest | 热门 / 最新 | 🔐 浏览器 | -| reddit | hot | 热门帖子 | 🔐 浏览器 | -| hackernews | top | 热门故事 | 🌐 公共 API | +## 📦 Built-in Commands -## 🎨 输出格式 +| Site | Commands | Mode | +|------|----------|------| +| **bilibili** | `hot` `search` `me` `favorite` `history` `feed` `user-videos` | 🔐 Browser | +| **zhihu** | `hot` `search` `question` | 🔐 Browser | +| **xiaohongshu** | `search` `feed` | 🔐 Browser | +| **twitter** | `trending` | 🔐 Browser | +| **reddit** | `hot` | 🔐 Browser | +| **github** | `trending` `search` | 🔐 / 🌐 | +| **v2ex** | `hot` `latest` `topic` | 🌐 Public | +| **hackernews** | `top` | 🌐 Public | + +## 🎨 Output Formats ```bash -opencli bilibili hot -f table # 默认表格 -opencli bilibili hot -f json # JSON（适合管道和 AI agent） +opencli bilibili hot -f table # Default: rich table +opencli bilibili hot -f json # JSON (pipe to jq, feed to AI) opencli bilibili hot -f md # Markdown opencli bilibili hot -f csv # CSV +opencli bilibili hot -v # Verbose: show pipeline steps ``` -## 🧠 AI Agent 工作流 +## 🧠 AI Agent Workflow ```bash -# Deep Explore: 自动发现 API 并推理 capabilities -opencli explore --site +# 1. Deep Explore — discover APIs, infer capabilities, detect framework +opencli explore https://example.com --site mysite -# 从探索成果物合成候选 CLI -opencli synthesize +# 2. Synthesize — generate candidate YAML adapters from explore artifacts +opencli synthesize mysite -# 一键：探索 → 合成 → 注册 -opencli generate --goal "hot" +# 3. Generate — one-shot: explore → synthesize → register +opencli generate https://example.com --goal "hot" -# Strategy Cascade: 自动寻找最简可用策略 -opencli cascade +# 4. Strategy Cascade — auto-probe: PUBLIC → COOKIE → HEADER +opencli cascade https://api.example.com/data ``` -## 🔧 创建新命令 +Explore outputs structured artifacts to `.opencli/explore//`: +- `manifest.json` — site metadata, framework detection +- `endpoints.json` — scored API endpoints with response schemas +- `capabilities.json` — inferred capabilities with confidence scores +- `auth.json` — authentication strategy recommendations -参考 [SKILL.md](./SKILL.md) 了解 YAML 和 TypeScript 两种方式创建新的 CLI 适配器。 +## 🔧 Create New Commands + +See **[SKILL.md](./SKILL.md)** for the full adapter development guide: +- **YAML pipeline** — declare navigate → evaluate → map → limit +- **TypeScript adapter** — for XHR interception, GraphQL, pagination ## 📄 License diff --git a/SKILL.md b/SKILL.md index d4eee49..0137e9c 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,47 +1,43 @@ --- name: opencli -description: "OpenCLI — Make any website your CLI. Zero setup, AI-powered. Turn any website into CLI commands via Chrome browser." +description: "OpenCLI — Make any website your CLI. Zero risk, AI-powered, reuse Chrome login." version: 0.1.0 author: jackwener -tags: [cli, browser, web, mcp, playwright, bilibili, zhihu, twitter, github, v2ex, hackernews, 哔哩哔哩, 知乎, AI, agent] +tags: [cli, browser, web, mcp, playwright, bilibili, zhihu, twitter, github, v2ex, hackernews, reddit, xiaohongshu, AI, agent] --- # OpenCLI -> Make any website your CLI. 操控 Chrome 无风控风险，复用登录，CLI 化全部网站。 +> Make any website your CLI. Reuse Chrome login, zero risk, AI-powered discovery. -## 安装 +## Install & Run ```bash cd ~/code/opencli npm install -``` -## 使用方式 - -```bash -# 通过 npx 运行（推荐） +# Run via source npx tsx src/main.ts -# 或者构建后运行 +# Or build first npm run build && node dist/main.js ``` -## 前置要求 +## Prerequisites -浏览器命令需要： -1. Chrome 浏览器正在运行 -2. 安装 [Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm) 扩展 -3. 点击扩展图标批准连接 +Browser commands require: +1. Chrome browser running (logged into target sites) +2. [Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm) extension +3. Click extension icon to approve connection (or set `PLAYWRIGHT_MCP_EXTENSION_TOKEN`) -公共 API 命令（hackernews、github search）无需浏览器。 +Public API commands (`hackernews`, `github search`, `v2ex`) need no browser. -## 内置命令 +## Commands Reference -### 数据查询 +### Data Commands ```bash -# Bilibili +# Bilibili (browser) opencli bilibili hot --limit 10 # B站热门视频 opencli bilibili search --keyword "rust" # 搜索视频 opencli bilibili me # 我的信息 @@ -50,76 +46,89 @@ opencli bilibili history --limit 20 # 观看历史 opencli bilibili feed --limit 10 # 动态时间线 opencli bilibili user-videos --uid 12345 # 用户投稿 -# 知乎 +# 知乎 (browser) opencli zhihu hot --limit 10 # 知乎热榜 opencli zhihu search --keyword "AI" # 搜索 opencli zhihu question --id 34816524 # 问题详情和回答 -# GitHub -opencli github trending --limit 10 # GitHub Trending -opencli github search --keyword "cli" # 搜索仓库（无需浏览器） +# 小红书 (browser) +opencli xiaohongshu search --keyword "美食" # 搜索笔记 -# Twitter/X +# GitHub (trending=browser, search=public) +opencli github trending --limit 10 # GitHub Trending +opencli github search --keyword "cli" # 搜索仓库 + +# Twitter/X (browser) opencli twitter trending --limit 10 # 热门话题 -# V2EX -opencli v2ex hot --limit 10 # 热门话题 -opencli v2ex latest --limit 10 # 最新话题 - -# Hacker News -opencli hackernews top --limit 10 # 热门故事（无需浏览器） - -# Reddit +# Reddit (browser) opencli reddit hot --limit 10 # 热门帖子 opencli reddit hot --subreddit programming # 指定子版块 + +# V2EX (public) +opencli v2ex hot --limit 10 # 热门话题 +opencli v2ex latest --limit 10 # 最新话题 +opencli v2ex topic --id 1024 # 主题详情 + +# Hacker News (public) +opencli hackernews top --limit 10 # Top stories ``` -### 管理命令 +### Management Commands ```bash -opencli list # 列出所有可用命令 -opencli list --json # JSON 格式输出 -opencli validate # 验证所有 CLI 定义 -opencli validate bilibili # 验证指定站点 +opencli list # List all commands +opencli list --json # JSON output +opencli validate # Validate all CLI definitions +opencli validate bilibili # Validate specific site ``` -### AI 工作流（为 AI Agent 设计） +### AI Agent Workflow ```bash -opencli explore # Deep Explore: 自动发现 API 并推理 capabilities -opencli synthesize # 从探索成果物合成候选 CLI -opencli generate --goal "hot" # 一键：探索 → 合成 → 注册 -opencli cascade # Strategy Cascade: 自动寻找最简可用策略 -opencli verify --smoke # 验证 + Smoke 测试 +# Deep Explore: network intercept → response analysis → capability inference +opencli explore --site + +# Synthesize: generate evaluate-based YAML pipelines from explore artifacts +opencli synthesize + +# Generate: one-shot explore → synthesize → register +opencli generate --goal "hot" + +# Strategy Cascade: auto-probe PUBLIC → COOKIE → HEADER +opencli cascade + +# Verify: smoke-test a generated adapter +opencli verify --smoke ``` -## 输出格式 +## Output Formats -所有命令支持 `--format` / `-f` 选项： +All commands support `--format` / `-f`: ```bash -opencli bilibili hot -f table # 默认表格 -opencli bilibili hot -f json # JSON +opencli bilibili hot -f table # Default: rich table +opencli bilibili hot -f json # JSON (pipe to jq, feed to AI agent) opencli bilibili hot -f md # Markdown opencli bilibili hot -f csv # CSV ``` -## 调试 +## Verbose Mode ```bash -opencli bilibili hot -v # 显示 pipeline 每步详情 +opencli bilibili hot -v # Show each pipeline step and data flow ``` -## 创建新的 CLI 适配器 +## Creating Adapters -### YAML 方式（声明式，推荐） +### YAML Pipeline (declarative, recommended) -在 `src/clis//.yaml` 创建文件： +Create `src/clis//.yaml`: ```yaml site: mysite name: hot -description: Hot topics on mysite +description: Hot topics domain: www.mysite.com strategy: cookie # public | cookie | header | intercept | ui browser: true @@ -136,11 +145,13 @@ pipeline: - evaluate: | (async () => { const res = await fetch('/api/hot', { credentials: 'include' }); - return await res.json(); + const d = await res.json(); + return d.data.items.map(item => ({ + title: item.title, + score: item.score, + })); })() - - select: data.items - - map: rank: ${{ index + 1 }} title: ${{ item.title }} @@ -151,9 +162,24 @@ pipeline: columns: [rank, title, score] ``` -### TypeScript 方式（编程式，更灵活） +For public APIs (no browser): -在 `src/clis//.ts` 创建并在 `clis/index.ts` 中 import： +```yaml +strategy: public +browser: false + +pipeline: + - fetch: + url: https://api.example.com/hot.json + - select: data.items + - map: + title: ${{ item.title }} + - limit: ${{ args.limit }} +``` + +### TypeScript Adapter (programmatic) + +Create `src/clis//.ts` and import in `clis/index.ts`: ```typescript import { cli, Strategy } from '../../registry.js'; @@ -165,73 +191,85 @@ cli({ args: [{ name: 'keyword', required: true }], columns: ['rank', 'title', 'url'], func: async (page, kwargs) => { + await page.goto('https://www.mysite.com'); const data = await page.evaluate(` - async () => { - const res = await fetch('/api/search?q=${kwargs.keyword}', { credentials: 'include' }); + (async () => { + const res = await fetch('/api/search?q=${kwargs.keyword}', { + credentials: 'include' + }); return await res.json(); - } + })() `); return data.items.map((item, i) => ({ - rank: i + 1, - title: item.title, - url: item.url, + rank: i + 1, title: item.title, url: item.url, })); }, }); ``` -## Pipeline 步骤参考 +**When to use TS**: XHR interception (小红书), cookie extraction (Twitter ct0), Wbi signing (Bilibili), auto-pagination, complex data transforms. -| 步骤 | 说明 | 示例 | -|------|------|------| -| `navigate` | 导航到 URL | `navigate: https://example.com` | -| `fetch` | HTTP 请求（使用浏览器 cookie） | `fetch: { url: "...", params: { q: "${{ args.keyword }}" } }` | -| `evaluate` | 执行 JavaScript | `evaluate: \| (async () => { ... })()` | -| `select` | 选取 JSON 路径 | `select: data.items` | -| `map` | 映射字段 | `map: { title: "${{ item.title }}" }` | -| `filter` | 过滤 | `filter: item.score > 100` | -| `sort` | 排序 | `sort: { by: score, order: desc }` | -| `limit` | 限制数量 | `limit: ${{ args.limit }}` | -| `snapshot` | 获取页面快照 | `snapshot: { interactive: true }` | -| `click` | 点击元素 | `click: ${{ ref }}` | -| `type` | 输入文本 | `type: { ref: "@1", text: "hello" }` | -| `wait` | 等待 | `wait: 2` 或 `wait: { text: "loaded" }` | -| `press` | 按键 | `press: Enter` | -| `intercept` | 声明式 XHR 拦截 | `intercept: { trigger: "navigate:...", capture: "api/hot", select: "data.items" }` | +## Pipeline Steps -## 模板语法 +| Step | Description | Example | +|------|-------------|---------| +| `navigate` | Go to URL | `navigate: https://example.com` | +| `fetch` | HTTP request (browser cookies) | `fetch: { url: "...", params: { q: "..." } }` | +| `evaluate` | Run JavaScript in page | `evaluate: \| (async () => { ... })()` | +| `select` | Extract JSON path | `select: data.items` | +| `map` | Map fields | `map: { title: "${{ item.title }}" }` | +| `filter` | Filter items | `filter: item.score > 100` | +| `sort` | Sort items | `sort: { by: score, order: desc }` | +| `limit` | Cap result count | `limit: ${{ args.limit }}` | +| `intercept` | Declarative XHR capture | `intercept: { trigger: "navigate:...", capture: "api/hot" }` | +| `snapshot` | Page accessibility tree | `snapshot: { interactive: true }` | +| `click` | Click element | `click: ${{ ref }}` | +| `type` | Type text | `type: { ref: "@1", text: "hello" }` | +| `wait` | Wait for time/text | `wait: 2` or `wait: { text: "loaded" }` | +| `press` | Press key | `press: Enter` | -使用 `${{ expression }}` 进行模板替换： +## Template Syntax ```yaml -# 引用参数 +# Arguments with defaults ${{ args.keyword }} ${{ args.limit | default(20) }} -# 引用当前 item（在 map/filter 中） +# Current item (in map/filter) ${{ item.title }} ${{ item.data.nested.field }} -# 索引（从 0 开始） +# Index (0-based) ${{ index }} ${{ index + 1 }} ``` -## 环境变量 +## 5-Tier Authentication Strategy -| 变量 | 默认值 | 说明 | -|------|--------|------| -| `OPENCLI_BROWSER_CONNECT_TIMEOUT` | 30 | 浏览器连接超时（秒） | -| `OPENCLI_BROWSER_COMMAND_TIMEOUT` | 45 | 命令执行超时（秒） | -| `OPENCLI_BROWSER_EXPLORE_TIMEOUT` | 120 | Explore 超时（秒） | -| `OPENCLI_EXTENSION_LOCK_TIMEOUT` | 120 | 扩展锁超时（秒） | -| `PLAYWRIGHT_MCP_EXTENSION_TOKEN` | — | 自动批准扩展连接 | +| Tier | Name | Method | Example | +|------|------|--------|---------| +| 1 | `public` | No auth, Node.js fetch | Hacker News, V2EX | +| 2 | `cookie` | Browser fetch with `credentials: include` | Bilibili, Zhihu | +| 3 | `header` | Custom headers (ct0, Bearer) | Twitter GraphQL | +| 4 | `intercept` | XHR interception + store mutation | 小红书 Pinia | +| 5 | `ui` | Full UI automation (click/type/scroll) | Last resort | -## 错误排查 +## Environment Variables -| 错误 | 解决方案 | -|------|----------| -| `npx not found` | 安装 Node.js: `brew install node` | -| `Timed out connecting to browser` | 1) 确认 Chrome 已打开 2) 安装 Playwright MCP Bridge 扩展 3) 点击扩展图标批准 | -| `Extension lock timed out` | 等待其他 opencli 命令完成，浏览器命令需串行运行 | -| `Request timed out` | 增大 `OPENCLI_BROWSER_COMMAND_TIMEOUT` 或检查网络 | +| Variable | Default | Description | +|----------|---------|-------------| +| `OPENCLI_BROWSER_CONNECT_TIMEOUT` | 30 | Browser connection timeout (sec) | +| `OPENCLI_BROWSER_COMMAND_TIMEOUT` | 45 | Command execution timeout (sec) | +| `OPENCLI_BROWSER_EXPLORE_TIMEOUT` | 120 | Explore timeout (sec) | +| `OPENCLI_EXTENSION_LOCK_TIMEOUT` | 120 | Extension lock timeout (sec) | +| `PLAYWRIGHT_MCP_EXTENSION_TOKEN` | — | Auto-approve extension connection | + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| `npx not found` | Install Node.js: `brew install node` | +| `Timed out connecting to browser` | 1) Chrome must be open 2) Install MCP Bridge extension 3) Click to approve | +| `Extension lock timed out` | Another opencli command is running; browser commands run serially | +| `Target page context` error | Add `navigate:` step before `evaluate:` in YAML | +| Empty table data | Check if evaluate returns JSON string (MCP parsing) or data path is wrong | diff --git a/package.json b/package.json index 31f73a2..d35f151 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ }, "scripts": { "dev": "tsx src/main.ts", - "build": "tsc", + "build": "tsc && npm run copy-yaml", + "copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done", "start": "node dist/main.js", "typecheck": "tsc --noEmit", "lint": "tsc --noEmit", diff --git a/src/explore.ts b/src/explore.ts index b19e674..3f1e1f3 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -228,6 +228,55 @@ const FRAMEWORK_DETECT_JS = ` } `; +// ── Store discovery ──────────────────────────────────────────────────────── + +const STORE_DISCOVER_JS = ` + () => { + const stores = []; + try { + const app = document.querySelector('#app'); + if (!app?.__vue_app__) return stores; + const gp = app.__vue_app__.config?.globalProperties; + + // Pinia stores + const pinia = gp?.$pinia; + if (pinia?._s) { + pinia._s.forEach((store, id) => { + const actions = []; + const stateKeys = []; + for (const k in store) { + try { + if (k.startsWith('$') || k.startsWith('_')) continue; + if (typeof store[k] === 'function') actions.push(k); + else stateKeys.push(k); + } catch {} + } + stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) }); + }); + } + + // Vuex store modules + const vuex = gp?.$store; + if (vuex?._modules?.root?._children) { + const children = vuex._modules.root._children; + for (const [modName, mod] of Object.entries(children)) { + const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20); + const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15); + stores.push({ type: 'vuex', id: modName, actions, stateKeys }); + } + } + } catch {} + return stores; + } +`; + +export interface DiscoveredStore { + type: 'pinia' | 'vuex'; + id: string; + actions: string[]; + stateKeys: string[]; +} + // ── Main explore function ────────────────────────────────────────────────── export async function exploreUrl( @@ -272,6 +321,15 @@ export async function exploreUrl( let framework: Record = {}; try { const fw = await page.evaluate(FRAMEWORK_DETECT_JS); if (fw && typeof fw === 'object') framework = fw; } catch {} + // Step 6.5: Discover stores (Pinia / Vuex) + let stores: DiscoveredStore[] = []; + if (framework.pinia || framework.vuex) { + try { + const raw = await page.evaluate(STORE_DISCOVER_JS); + if (Array.isArray(raw)) stores = raw; + } catch {} + } + // Step 7: Analyze endpoints const seen = new Map(); for (const entry of networkEntries) { @@ -325,13 +383,31 @@ export async function exploreUrl( args.push({ name: 'limit', type: 'int', required: false, default: 20 }); if (ep.hasPaginationParam) args.push({ name: 'page', type: 'int', required: false, default: 1 }); + // Link store actions to capabilities when store-action strategy is recommended + const epStrategy = inferStrategy(ep.authIndicators); + let storeHint: { store: string; action: string } | undefined; + if ((epStrategy === 'intercept' || ep.authIndicators.includes('signature')) && stores.length > 0) { + // Try to find a store/action that matches this endpoint's purpose + for (const s of stores) { + const matchingAction = s.actions.find(a => + capName.split('_').some(part => a.toLowerCase().includes(part)) || + a.toLowerCase().includes('fetch') || a.toLowerCase().includes('get') + ); + if (matchingAction) { + storeHint = { store: s.id, action: matchingAction }; + break; + } + } + } + capabilities.push({ name: capName, description: `${opts.site ?? detectSiteName(url)} ${capName}`, - strategy: inferStrategy(ep.authIndicators), + strategy: storeHint ? 'store-action' : epStrategy, confidence: Math.min(ep.score / 20, 1.0), endpoint: ep.pattern, itemPath: ep.responseAnalysis?.itemPath ?? null, recommendedColumns: cols.length ? cols : ['title', 'url'], recommendedArgs: args, + ...(storeHint ? { storeHint } : {}), }); } @@ -345,7 +421,7 @@ export async function exploreUrl( const result = { site: siteName, target_url: url, final_url: metadata.url, title: metadata.title, - framework, top_strategy: topStrategy, + framework, stores, top_strategy: topStrategy, endpoint_count: analyzedEndpoints.length + [...seen.values()].filter(ep => ep.score < 5).length, api_endpoint_count: analyzedEndpoints.length, capabilities, auth_indicators: [...allAuth], @@ -354,7 +430,8 @@ export async function exploreUrl( // Write artifacts fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify({ site: siteName, target_url: url, final_url: metadata.url, title: metadata.title, - framework, top_strategy: topStrategy, explored_at: new Date().toISOString(), + framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })), + top_strategy: topStrategy, explored_at: new Date().toISOString(), }, null, 2)); fs.writeFileSync(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({ pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status, @@ -366,6 +443,9 @@ export async function exploreUrl( fs.writeFileSync(path.join(targetDir, 'auth.json'), JSON.stringify({ top_strategy: topStrategy, indicators: [...allAuth], framework, }, null, 2)); + if (stores.length > 0) { + fs.writeFileSync(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)); + } return { ...result, out_dir: targetDir }; })(), { timeout: exploreTimeout, label: `Explore ${url}` }); @@ -374,17 +454,25 @@ export async function exploreUrl( export function renderExploreSummary(result: Record): string { const lines = [ - 'opencli explore: OK', `Site: ${result.site}`, `URL: ${result.target_url}`, + 'opencli probe: OK', `Site: ${result.site}`, `URL: ${result.target_url}`, `Title: ${result.title || '(none)'}`, `Strategy: ${result.top_strategy}`, `Endpoints: ${result.endpoint_count} total, ${result.api_endpoint_count} API`, `Capabilities: ${result.capabilities?.length ?? 0}`, ]; for (const cap of (result.capabilities ?? []).slice(0, 5)) { - lines.push(` • ${cap.name} (${cap.strategy}, ${(cap.confidence * 100).toFixed(0)}%)`); + const storeInfo = cap.storeHint ? ` → ${cap.storeHint.store}.${cap.storeHint.action}()` : ''; + lines.push(` • ${cap.name} (${cap.strategy}, ${(cap.confidence * 100).toFixed(0)}%)${storeInfo}`); } const fw = result.framework ?? {}; const fwNames = Object.entries(fw).filter(([, v]) => v).map(([k]) => k); if (fwNames.length) lines.push(`Framework: ${fwNames.join(', ')}`); + const stores: DiscoveredStore[] = result.stores ?? []; + if (stores.length) { + lines.push(`Stores: ${stores.length}`); + for (const s of stores.slice(0, 5)) { + lines.push(` • ${s.type}/${s.id}: ${s.actions.slice(0, 5).join(', ')}${s.actions.length > 5 ? '...' : ''}`); + } + } lines.push(`Output: ${result.out_dir}`); return lines.join('\n'); } diff --git a/src/main.ts b/src/main.ts index 4ab6f69..44d2027 100644 --- a/src/main.ts +++ b/src/main.ts @@ -52,6 +52,9 @@ program.command('verify').description('Validate + smoke test').argument('[target program.command('explore').description('Explore a website').argument('').option('--site ').option('--goal ').option('--wait ', '', '3') .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); }); +program.command('probe').description('Probe a website: discover APIs, stores, and recommend strategies').argument('').option('--site ').option('--goal ').option('--wait ', '', '3') + .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); }); + program.command('synthesize').description('Synthesize CLIs from explore').argument('').option('--top ', '', '3') .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); }); diff --git a/src/synthesize.ts b/src/synthesize.ts index cae054c..d895cc2 100644 --- a/src/synthesize.ts +++ b/src/synthesize.ts @@ -135,7 +135,28 @@ function buildCandidateYaml(site: string, manifest: any, cap: any, endpoint: any let domain = ''; try { domain = new URL(manifest.target_url).hostname; } catch {} - if (needsBrowser) { + if (cap.strategy === 'store-action' && cap.storeHint) { + // Store Action: navigate + wait + tap (declarative, clean) + pipeline.push({ navigate: manifest.target_url }); + pipeline.push({ wait: 3 }); + const tapStep: Record = { + store: cap.storeHint.store, + action: cap.storeHint.action, + timeout: 8, + }; + // Infer capture pattern from endpoint URL + if (endpoint?.url) { + try { + const epUrl = new URL(endpoint.url); + const pathParts = epUrl.pathname.split('/').filter((p: string) => p); + // Use last meaningful path segment as capture pattern + const capturePart = pathParts.filter((p: string) => !p.match(/^v\d+$/)).pop(); + if (capturePart) tapStep.capture = capturePart; + } catch {} + } + if (cap.itemPath) tapStep.select = cap.itemPath; + pipeline.push({ tap: tapStep }); + } else if (needsBrowser) { // Browser-based: navigate + evaluate (like bilibili/hot.yaml, twitter/trending.yaml) pipeline.push({ navigate: manifest.target_url }); const itemPath = cap.itemPath ?? 'data.data.list';