From e64fc655547f566eae7faa82d7386fb4df1ba389 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sun, 15 Mar 2026 18:03:25 +0800 Subject: [PATCH] feat: add Bilibili subtitle adapter with Wbi signing & explore fuzzing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New: src/clis/bilibili/subtitle.ts - cascading BVID→CID→subtitle fetch using apiGet with automatic Wbi signature (w_rid) - Enhanced: src/explore.ts - added INTERACT_FUZZ_JS for interactive API discovery via simulated clicks (--auto flag) - Enhanced: src/explore.ts - empty-value score penalty for anti-bot detection - Fixed: src/main.ts - restored explore command syntax, added --auto option - Updated: CLI-CREATOR.md - cascading requests & anti-bot sections --- CLI-CREATOR.md | 36 ++++++++++++- src/browser.ts | 39 +++++++++++--- src/clis/bilibili/subtitle.ts | 95 +++++++++++++++++++++++++++++++++++ src/explore.ts | 41 +++++++++++++++ src/main.ts | 4 +- src/pipeline/steps/browser.ts | 11 ++-- src/pipeline/steps/fetch.ts | 28 ++++++++--- src/pipeline/steps/tap.ts | 14 +++--- src/types.ts | 2 +- test_subtitle.ts | 33 ++++++++++++ 10 files changed, 273 insertions(+), 30 deletions(-) create mode 100644 src/clis/bilibili/subtitle.ts create mode 100644 test_subtitle.ts diff --git a/CLI-CREATOR.md b/CLI-CREATOR.md index 7ac5ea7..7a1122b 100644 --- a/CLI-CREATOR.md +++ b/CLI-CREATOR.md @@ -57,8 +57,9 @@ opencli bilibili hot -v # 查看已有命令的 pipeline 每步数据流 1. **后缀爆破法 (`.json`)**: 像 Reddit 这样复杂的网站，只要在其 URL 后加上 `.json`（例如 `/r/all.json`），就能在带 Cookie 的情况下直接利用 `fetch` 拿到极其干净的 REST 数据（Tier 2 Cookie 策略极速秒杀）。另外如功能完备的**雪球 (xueqiu)** 也可以走这种纯 API 的方式极简获取，成为你构建简单 YAML 的黄金标杆。 2. **全局状态查找法 (`__INITIAL_STATE__`)**: 许多服务端渲染 (SSR) 的网站（如小红书、Bilibili）会将首页或详情页的完整数据挂载到全局 window 对象上。与其去拦截网络请求，不如直接 `page.evaluate('() => window.__INITIAL_STATE__')` 获取整个数据树。 -3. **框架探测与 Store Action 截断**: 如果站点使用 Vue + Pinia，可以使用 `tap` 步骤调用 action，让前端框架代替你完成复杂的鉴权签名封装。 -4. **底层 XHR/Fetch 拦截**: 最后手段，当上述都不行时，使用 TypeScript 适配器进行无侵入式的请求抓取。 +3. **主动交互触发法 (Active Interaction)**: 很多深层 API（如视频字幕、评论下的回复）是懒加载的。在静态抓包找不到数据时，尝试在 `evaluate` 步骤或手动打断点时，主动去**点击（Click）页面上的对应按钮**（如"CC"、"展开全部"），从而诱发隐藏的 Network Fetch。 +4. **框架探测与 Store Action 截断**: 如果站点使用 Vue + Pinia，可以使用 `tap` 步骤调用 action，让前端框架代替你完成复杂的鉴权签名封装。 +5. **底层 XHR/Fetch 拦截**: 最后手段，当上述都不行时，使用 TypeScript 适配器进行无侵入式的请求抓取。 ### 1d. 框架检测 @@ -411,6 +412,35 @@ cli({ > **拦截核心思路**：不自己构造签名，而是利用 `installInterceptor` 劫持网站自己的 `XMLHttpRequest` 和 `fetch`，让网站发请求，我们直接在底层取出解析好的 `response.json()`。 +#### 进阶场景 1: 级联请求 (Cascading Requests) 与鉴权绕过 + +部分 API 获取是非常复杂的连环请求（例如 B 站获取视频字幕：先需要 `bvid` 获取核心 `cid`，再通过 `cid` 获取包含签名/Wbi 的字幕列表拉取地址，最后 fetch 真实的 CDN 资源）。在此类场景中，你必须在一个 `evaluate` 块内部或者在 TypeScript Node 端编排整个请求链条： + +```typescript +// 真实场景：B站获取视频字幕的级联获取思路 +const subtitleUrls = await page.evaluate(async (bvid) => { + // Step 1: 拿 CID (通常可以通过页面全局状态极速提取) + const cid = window.__INITIAL_STATE__?.videoData?.cid; + + // Step 2: 依据 BVID 和 CID 拿字幕配置 (可能需要携带 W_RID 签名或依赖浏览器当前登录状态 Cookie) + const res = await fetch(\`/x/player/wbi/v2?bvid=\${bvid}&cid=\${cid}\`, { credentials: 'include' }); + const data = await res.json(); + + // Step 3: 风控拦截/未登录降级空值检测 (Anti-Bot Empty Value Detection) ⚠️ 极其重要 + // 很多大厂 API 只要签名失败或无强登录 Cookie 依然会返回 HTTP 200，但把关键 URL 设为 "" + const firstSubUrl = data.data?.subtitle?.subtitles?.[0]?.subtitle_url; + if (!firstSubUrl) { + throw new Error('被风控降级或需登录：拿不到真实的 subtitle_url，请检查 Cookie 状态 (Tier 2/3)'); + } + + return firstSubUrl; +}, kwargs.bvid); + +// Step 4: 拉取最终的 CDN 静态文件 (无鉴权) +const finalRes = await fetch(subtitleUrls.startsWith('//') ? 'https:' + subtitleUrls : subtitleUrls); +const subtitles = await finalRes.json(); +``` + --- ## Step 4: 测试 @@ -553,6 +583,8 @@ git push | TS evaluate 格式 | `() => {}` 报 `result is not a function` | TS 中 `page.evaluate()` 必须用 IIFE：`(async () => { ... })()` | | 页面异步加载 | evaluate 拿到空数据（store state 还没更新） | 在 evaluate 内用 polling 等待数据出现，或增加 `wait` 时间 | | YAML 内嵌大段 JS | 调试困难，字符串转义问题 | 超过 10 行 JS 的命令改用 TS adapter | +| **风控被拦截(伪200)** | 获取到的 JSON 里核心数据是 `""` (空串) | 极易被误判。必须添加断言！无核心数据立刻要求升级鉴权 Tier 并重新配置 Cookie | +| **API 没找见** | `explore` 工具打分出来的都拿不到深层数据 | 点击页面按钮诱发懒加载数据，再结合 `getInterceptedRequests` 获取 | --- diff --git a/src/browser.ts b/src/browser.ts index efee67e..adbe82d 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -104,8 +104,13 @@ export class Page implements IPage { await this.call('tools/call', { name: 'browser_press_key', arguments: { key } }); } - async wait(seconds: number): Promise { - await this.call('tools/call', { name: 'browser_wait_for', arguments: { time: seconds } }); + async wait(options: number | { text?: string; time?: number; timeout?: number }): Promise { + if (typeof options === 'number') { + await this.call('tools/call', { name: 'browser_wait_for', arguments: { time: options } }); + } else { + // Pass directly to native wait_for, which supports natively awaiting text strings without heavy DOM polling + await this.call('tools/call', { name: 'browser_wait_for', arguments: options }); + } } async tabs(): Promise { @@ -139,10 +144,32 @@ export class Page implements IPage { async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise { const times = options.times ?? 3; const delayMs = options.delayMs ?? 2000; - for (let i = 0; i < times; i++) { - await this.evaluate('() => window.scrollTo(0, document.body.scrollHeight)'); - await this.wait(delayMs / 1000); - } + const js = ` + async () => { + const maxTimes = ${times}; + const maxWaitMs = ${delayMs}; + for (let i = 0; i < maxTimes; i++) { + const lastHeight = document.body.scrollHeight; + window.scrollTo(0, lastHeight); + await new Promise(resolve => { + let timeoutId; + const observer = new MutationObserver(() => { + if (document.body.scrollHeight > lastHeight) { + clearTimeout(timeoutId); + observer.disconnect(); + setTimeout(resolve, 100); // Small debounce for rendering + } + }); + observer.observe(document.body, { childList: true, subtree: true }); + timeoutId = setTimeout(() => { + observer.disconnect(); + resolve(null); + }, maxWaitMs); + }); + } + } + `; + await this.evaluate(js); } async installInterceptor(pattern: string): Promise { diff --git a/src/clis/bilibili/subtitle.ts b/src/clis/bilibili/subtitle.ts new file mode 100644 index 0000000..742a499 --- /dev/null +++ b/src/clis/bilibili/subtitle.ts @@ -0,0 +1,95 @@ +import { cli, Strategy } from '../../registry.js'; +import type { IPage } from '../../types.js'; +import { apiGet } from '../../bilibili.js'; + +cli({ + site: 'bilibili', + name: 'subtitle', + description: '获取 Bilibili 视频的字幕', + strategy: Strategy.COOKIE, + args: [ + { name: 'bvid', required: true }, + ], + columns: ['index', 'from', 'to', 'content'], + func: async (page: IPage | null, kwargs: any) => { + if (!page) throw new Error('Requires browser'); + // 1. 先前往视频详情页 (建立有鉴权的 Session，且这里不需要加载完整个视频) + await page.goto(`https://www.bilibili.com/video/${kwargs.bvid}/`); + + // 2. 利用 __INITIAL_STATE__ 获取基础信息，拿 CID + const cid = await page.evaluate(`(async () => { + const state = window.__INITIAL_STATE__ || {}; + return state?.videoData?.cid; + })()`); + + if (!cid) { + throw new Error('无法在页面中提取到当前视频的 CID，请检查页面是否正常加载。'); + } + + // 3. 在 Node 端使用 apiGet 获取带 Wbi 签名的字幕列表 + // 之前纯靠 evaluate 里的 fetch 会失败，因为 B 站 /wbi/ 开头的接口强校验 w_rid，未签名直接被风控返回 403 HTML + const payload = await apiGet(page, '/x/player/wbi/v2', { + params: { bvid: kwargs.bvid, cid }, + signed: true, // 开启 wbi_sign 自动签名 + }); + + if (payload.code !== 0) { + throw new Error(`获取视频播放信息失败: ${payload.message} (${payload.code})`); + } + + const subtitles = payload.data?.subtitle?.subtitles || []; + if (subtitles.length === 0) { + throw new Error('此视频没有发现外挂或智能字幕。'); + } + + const firstSubUrl = subtitles[0].subtitle_url; + if (!firstSubUrl || firstSubUrl === '') { + throw new Error('[风控拦截/未登录] 获取到的 subtitle_url 为空！请确保 CLI 已成功登录且风控未封锁此账号。'); + } + + const finalUrl = firstSubUrl.startsWith('//') ? 'https:' + firstSubUrl : firstSubUrl; + + + // 4. 解析并拉取 CDN 的 JSON 文件 + const fetchJs = ` + (async () => { + const url = '${finalUrl}'; + const res = await fetch(url); + const text = await res.text(); + + if (text.startsWith(' ({ + index: idx + 1, + from: Number(item.from || 0).toFixed(2) + 's', + to: Number(item.to || 0).toFixed(2) + 's', + content: item.content + })); + }, +}); diff --git a/src/explore.ts b/src/explore.ts index a222f16..ea5ae21 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -277,6 +277,30 @@ export interface DiscoveredStore { stateKeys: string[]; } +// ── Auto-Interaction (Fuzzing) ───────────────────────────────────────────── + +const INTERACT_FUZZ_JS = ` + async () => { + const sleep = ms => new Promise(r => setTimeout(r, ms)); + const clickables = Array.from(document.querySelectorAll( + 'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]' + )).slice(0, 15); // limit to 15 to avoid endless loops + + let clicked = 0; + for (const el of clickables) { + try { + const rect = el.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window })); + clicked++; + await sleep(300); // give it time to trigger network + } + } catch {} + } + return clicked; + } +`; + // ── Main explore function ────────────────────────────────────────────────── export async function exploreUrl( @@ -300,6 +324,18 @@ export async function exploreUrl( // Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist) for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); } + // Step 2.5: Interactive Fuzzing (if requested) + if (opts.auto) { + try { + console.log("🤖 [Explore] Auto-fuzzing interactions enabled. Simulating clicks..."); + const clicks = await page.evaluate(INTERACT_FUZZ_JS); + console.log(`🤖 [Explore] Performed ${clicks} simulated clicks.`); + await page.wait(2); // wait for XHRs to settle + } catch (e) { + console.error("🤖 [Explore] Fuzzing failed:", e); + } + } + // Step 3: Read page metadata const metadata = await readPageMetadata(page); @@ -383,6 +419,11 @@ export async function exploreUrl( args.push({ name: 'limit', type: 'int', required: false, default: 20 }); if (ep.hasPaginationParam) args.push({ name: 'page', type: 'int', required: false, default: 1 }); + // Improve scoring for JSON endpoints that return empty arrays/objects (Anti-Bot Empty Value Detection) + if (ep.score >= 5 && ep.responseAnalysis?.itemCount === 0 && ep.contentType.includes('json')) { + ep.score -= 2; // Penalize empty drops + } + // Link store actions to capabilities when store-action strategy is recommended const epStrategy = inferStrategy(ep.authIndicators); let storeHint: { store: string; action: string } | undefined; diff --git a/src/main.ts b/src/main.ts index 3835f05..a2557a1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -53,8 +53,8 @@ program.command('validate').description('Validate CLI definitions').argument('[t program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false) .action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; }); -program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('').option('--site ').option('--goal ').option('--wait ', '', '3') - .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); }); +program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('').option('--site ').option('--goal ').option('--wait ', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)') + .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto }))); }); program.command('synthesize').description('Synthesize CLIs from explore').argument('').option('--top ', '', '3') .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); }); diff --git a/src/pipeline/steps/browser.ts b/src/pipeline/steps/browser.ts index 6e3fce7..e7b8dbf 100644 --- a/src/pipeline/steps/browser.ts +++ b/src/pipeline/steps/browser.ts @@ -31,13 +31,10 @@ export async function stepWait(page: IPage, params: any, data: any, args: Record if (typeof params === 'number') await page.wait(params); else if (typeof params === 'object' && params) { if ('text' in params) { - const timeout = params.timeout ?? 10; - const start = Date.now(); - while ((Date.now() - start) / 1000 < timeout) { - const snap = await page.snapshot({ raw: true }); - if (typeof snap === 'string' && snap.includes(params.text)) break; - await page.wait(0.5); - } + await page.wait({ + text: String(render(params.text, { args, data })), + timeout: params.timeout + }); } else if ('time' in params) await page.wait(Number(params.time)); } else if (typeof params === 'string') await page.wait(Number(render(params, { args, data }))); return data; diff --git a/src/pipeline/steps/fetch.ts b/src/pipeline/steps/fetch.ts index 0d8f36e..d5d79f5 100644 --- a/src/pipeline/steps/fetch.ts +++ b/src/pipeline/steps/fetch.ts @@ -5,6 +5,23 @@ import type { IPage } from '../../types.js'; import { render } from '../template.js'; +/** Simple async concurrency limiter */ +async function mapConcurrent(items: T[], limit: number, fn: (item: T, index: number) => Promise): Promise { + const results: R[] = new Array(items.length); + let index = 0; + + async function worker() { + while (index < items.length) { + const i = index++; + results[i] = await fn(items[i], i); + } + } + + const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker()); + await Promise.all(workers); + return results; +} + /** Single URL fetch helper */ async function fetchSingle( page: IPage | null, url: string, method: string, @@ -48,12 +65,11 @@ export async function stepFetch(page: IPage | null, params: any, data: any, args // Per-item fetch when data is array and URL references item if (Array.isArray(data) && urlTemplate.includes('item')) { - const results: any[] = []; - for (let i = 0; i < data.length; i++) { - const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i })); - results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data)); - } - return results; + const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5; + return mapConcurrent(data, concurrency, async (item, index) => { + const itemUrl = String(render(urlTemplate, { args, data, item, index })); + return fetchSingle(page, itemUrl, method, queryParams, headers, args, data); + }); } const url = render(urlOrObj, { args, data }); return fetchSingle(page, String(url), method, queryParams, headers, args, data); diff --git a/src/pipeline/steps/tap.ts b/src/pipeline/steps/tap.ts index 6211c0b..6ef4d6e 100644 --- a/src/pipeline/steps/tap.ts +++ b/src/pipeline/steps/tap.ts @@ -42,6 +42,8 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record< async () => { // ── 1. Setup capture proxy (fetch + XHR dual interception) ── let captured = null; + let captureResolve; + const capturePromise = new Promise(r => { captureResolve = r; }); const capturePattern = ${JSON.stringify(capturePattern)}; // Intercept fetch API @@ -52,7 +54,7 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record< const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0] : fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]); if (capturePattern && url.includes(capturePattern) && !captured) { - try { captured = await resp.clone().json(); } catch {} + try { captured = await resp.clone().json(); captureResolve(); } catch {} } } catch {} return resp; @@ -71,13 +73,13 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record< const origHandler = xhr.onreadystatechange; xhr.onreadystatechange = function() { if (xhr.readyState === 4 && !captured) { - try { captured = JSON.parse(xhr.responseText); } catch {} + try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {} } if (origHandler) origHandler.apply(this, arguments); }; const origOnload = xhr.onload; xhr.onload = function() { - if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} } + if (!captured) { try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {} } if (origOnload) origOnload.apply(this, arguments); }; } @@ -117,9 +119,9 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record< await ${actionCall}; // ── 4. Wait for network response ── - const deadline = Date.now() + ${timeout} * 1000; - while (!captured && Date.now() < deadline) { - await new Promise(r => setTimeout(r, 200)); + if (!captured) { + const timeoutPromise = new Promise(r => setTimeout(r, ${timeout} * 1000)); + await Promise.race([capturePromise, timeoutPromise]); } } finally { // ── 5. Always restore originals ── diff --git a/src/types.ts b/src/types.ts index 24016fb..08075d9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,7 +12,7 @@ export interface IPage { click(ref: string): Promise; typeText(ref: string, text: string): Promise; pressKey(key: string): Promise; - wait(seconds: number): Promise; + wait(options: number | { text?: string; time?: number; timeout?: number }): Promise; tabs(): Promise; closeTab(index?: number): Promise; newTab(): Promise; diff --git a/test_subtitle.ts b/test_subtitle.ts new file mode 100644 index 0000000..9f57ca8 --- /dev/null +++ b/test_subtitle.ts @@ -0,0 +1,33 @@ +import { chromium } from 'playwright'; + +async function main() { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + const apis: string[] = []; + + page.on('response', async (res) => { + const url = res.url(); + if (url.includes('subtitle') || url.includes('player/v2') || url.includes('x/player/wbi/v2')) { + console.log("Found subtitle API candidate:", url); + } + }); + + console.log("Navigating to video..."); + await page.goto('https://www.bilibili.com/video/BV1K3411r7F9'); + await page.waitForTimeout(5000); + + // Try to find subtitle info in __INITIAL_STATE__ + const initialState = await page.evaluate(() => { + return (window as any).__INITIAL_STATE__; + }); + + if (initialState) { + console.log("Found __INITIAL_STATE__ exists!"); + // check if there's bvid or aid + console.log("BVID:", initialState.bvid); + console.log("AID:", initialState.aid); + } + + await browser.close(); +} +main();