diff --git a/src/clis/index.ts b/src/clis/index.ts index 15086b3..4165fdd 100644 --- a/src/clis/index.ts +++ b/src/clis/index.ts @@ -17,3 +17,6 @@ import './github/search.js'; // zhihu import './zhihu/question.js'; + +// xiaohongshu +import './xiaohongshu/search.js'; diff --git a/src/clis/v2ex/topic.yaml b/src/clis/v2ex/topic.yaml new file mode 100644 index 0000000..9a07b27 --- /dev/null +++ b/src/clis/v2ex/topic.yaml @@ -0,0 +1,27 @@ +site: v2ex +name: topic +description: V2EX 主题详情和回复 +domain: www.v2ex.com +strategy: public +browser: false + +args: + id: + type: str + required: true + description: Topic ID + +pipeline: + - fetch: + url: https://www.v2ex.com/api/topics/show.json + params: + id: ${{ args.id }} + + - map: + title: ${{ item.title }} + replies: ${{ item.replies }} + url: ${{ item.url }} + + - limit: 1 + +columns: [title, replies, url] diff --git a/src/clis/xiaohongshu/feed.yaml b/src/clis/xiaohongshu/feed.yaml new file mode 100644 index 0000000..bb8fb9b --- /dev/null +++ b/src/clis/xiaohongshu/feed.yaml @@ -0,0 +1,32 @@ +site: xiaohongshu +name: feed +description: "小红书首页推荐 Feed (via Pinia Store Action)" +domain: www.xiaohongshu.com +strategy: intercept +browser: true + +args: + limit: + type: int + default: 20 + description: Number of items to return + +columns: [title, author, likes, type, url] + +pipeline: + - navigate: https://www.xiaohongshu.com/explore + - wait: 3 + - tap: + store: feed + action: fetchFeeds + capture: homefeed + select: data.items + timeout: 8 + - map: + id: ${{ item.id }} + title: ${{ item.note_card.display_title }} + type: ${{ item.note_card.type }} + author: ${{ item.note_card.user.nickname }} + likes: ${{ item.note_card.interact_info.liked_count }} + url: https://www.xiaohongshu.com/explore/${{ item.id }} + - limit: ${{ args.limit | default(20) }} diff --git a/src/clis/xiaohongshu/search.ts b/src/clis/xiaohongshu/search.ts new file mode 100644 index 0000000..7d1a71c --- /dev/null +++ b/src/clis/xiaohongshu/search.ts @@ -0,0 +1,71 @@ +/** + * Xiaohongshu search — trigger search via Pinia store + XHR interception. + * Inspired by bb-sites/xiaohongshu/search.js but adapted for opencli pipeline. + */ + +import { cli, Strategy } from '../../registry.js'; + +cli({ + site: 'xiaohongshu', + name: 'search', + description: '搜索小红书笔记', + domain: 'www.xiaohongshu.com', + strategy: Strategy.COOKIE, + args: [ + { name: 'keyword', required: true, help: 'Search keyword' }, + { name: 'limit', type: 'int', default: 20, help: 'Number of results' }, + ], + columns: ['rank', 'title', 'author', 'likes', 'type'], + func: async (page, kwargs) => { + await page.goto('https://www.xiaohongshu.com'); + await page.wait(2); + + const data = await page.evaluate(` + (async () => { + const app = document.querySelector('#app')?.__vue_app__; + const pinia = app?.config?.globalProperties?.$pinia; + if (!pinia?._s) return {error: 'Page not ready'}; + + const searchStore = pinia._s.get('search'); + if (!searchStore) return {error: 'Search store not found'}; + + let captured = null; + const origOpen = XMLHttpRequest.prototype.open; + const origSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.open = function(m, u) { this.__url = u; return origOpen.apply(this, arguments); }; + XMLHttpRequest.prototype.send = function(b) { + if (this.__url?.includes('search/notes')) { + const x = this; + const orig = x.onreadystatechange; + x.onreadystatechange = function() { if (x.readyState === 4 && !captured) { try { captured = JSON.parse(x.responseText); } catch {} } if (orig) orig.apply(this, arguments); }; + } + return origSend.apply(this, arguments); + }; + + try { + searchStore.mutateSearchValue('${kwargs.keyword}'); + await searchStore.loadMore(); + await new Promise(r => setTimeout(r, 800)); + } finally { + XMLHttpRequest.prototype.open = origOpen; + XMLHttpRequest.prototype.send = origSend; + } + + if (!captured?.success) return {error: captured?.msg || 'Search failed'}; + return (captured.data?.items || []).map(i => ({ + title: i.note_card?.display_title || '', + type: i.note_card?.type || '', + url: 'https://www.xiaohongshu.com/explore/' + i.id, + author: i.note_card?.user?.nickname || '', + likes: i.note_card?.interact_info?.liked_count || '0', + })); + })() + `); + + if (!Array.isArray(data)) return []; + return data.slice(0, kwargs.limit).map((item: any, i: number) => ({ + rank: i + 1, + ...item, + })); + }, +}); diff --git a/src/clis/xiaohongshu/search.yaml b/src/clis/xiaohongshu/search.yaml new file mode 100644 index 0000000..cccee67 --- /dev/null +++ b/src/clis/xiaohongshu/search.yaml @@ -0,0 +1,36 @@ +site: xiaohongshu +name: search +description: "小红书搜索 (via Pinia Store Action)" +domain: www.xiaohongshu.com +strategy: intercept +browser: true + +args: + keyword: + type: str + required: true + description: Search keyword + limit: + type: int + default: 20 + description: Number of items to return + +columns: [title, author, likes, type, url] + +pipeline: + - navigate: https://www.xiaohongshu.com/search_result?keyword=${{ args.keyword }}&source=web_search_result_notes + - wait: 3 + - tap: + store: search + action: fetchResult + capture: search/notes + select: data.items + timeout: 8 + - map: + id: ${{ item.id }} + title: ${{ item.note_card.display_title }} + type: ${{ item.note_card.type }} + author: ${{ item.note_card.user.nickname }} + likes: ${{ item.note_card.interact_info.liked_count }} + url: https://www.xiaohongshu.com/explore/${{ item.id }} + - limit: ${{ args.limit | default(20) }} diff --git a/src/pipeline.ts b/src/pipeline.ts index 0e7c1b2..461a43f 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -295,6 +295,152 @@ async function executeStep(page: any, op: string, params: any, data: any, args: return result; } + case 'tap': { + // ── Declarative Store Action Bridge ────────────────────────────────── + // Usage: + // tap: + // store: feed # Pinia/Vuex store name + // action: fetchFeeds # Store action to call + // args: [] # Optional args to pass to action + // capture: homefeed # URL pattern to capture response + // timeout: 5 # Seconds to wait for network (default: 5) + // select: data.items # Optional: extract sub-path from response + // framework: pinia # Optional: pinia | vuex (auto-detected if omitted) + // + // Generates a self-contained IIFE that: + // 1. Injects fetch + XHR dual interception proxy + // 2. Finds the Pinia/Vuex store and calls the action + // 3. Captures the response matching the URL pattern + // 4. Auto-cleans up interception in finally block + // 5. Returns the captured data (optionally sub-selected) + + const cfg = typeof params === 'object' ? params : {}; + const storeName = String(render(cfg.store ?? '', { args, data })); + const actionName = String(render(cfg.action ?? '', { args, data })); + const capturePattern = String(render(cfg.capture ?? '', { args, data })); + const timeout = cfg.timeout ?? 5; + const selectPath = cfg.select ? String(render(cfg.select, { args, data })) : null; + const framework = cfg.framework ?? null; // auto-detect if null + const actionArgs = cfg.args ?? []; + + if (!storeName || !actionName) throw new Error('tap: store and action are required'); + + // Build select chain for the captured response + const selectChain = selectPath + ? selectPath.split('.').map((p: string) => `?.[${JSON.stringify(p)}]`).join('') + : ''; + + // Serialize action arguments + const actionArgsRendered = actionArgs.map((a: any) => { + const rendered = render(a, { args, data }); + return JSON.stringify(rendered); + }); + const actionCall = actionArgsRendered.length + ? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})` + : `store[${JSON.stringify(actionName)}]()`; + + const js = ` + async () => { + // ── 1. Setup capture proxy (fetch + XHR dual interception) ── + let captured = null; + const capturePattern = ${JSON.stringify(capturePattern)}; + + // Intercept fetch API + const origFetch = window.fetch; + window.fetch = async function(...fetchArgs) { + const resp = await origFetch.apply(this, fetchArgs); + try { + const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0] + : fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]); + if (capturePattern && url.includes(capturePattern) && !captured) { + try { captured = await resp.clone().json(); } catch {} + } + } catch {} + return resp; + }; + + // Intercept XMLHttpRequest + const origXhrOpen = XMLHttpRequest.prototype.open; + const origXhrSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.open = function(method, url) { + this.__tapUrl = String(url); + return origXhrOpen.apply(this, arguments); + }; + XMLHttpRequest.prototype.send = function(body) { + if (capturePattern && this.__tapUrl?.includes(capturePattern)) { + const xhr = this; + const origHandler = xhr.onreadystatechange; + xhr.onreadystatechange = function() { + if (xhr.readyState === 4 && !captured) { + try { captured = JSON.parse(xhr.responseText); } catch {} + } + if (origHandler) origHandler.apply(this, arguments); + }; + // Also handle onload + const origOnload = xhr.onload; + xhr.onload = function() { + if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} } + if (origOnload) origOnload.apply(this, arguments); + }; + } + return origXhrSend.apply(this, arguments); + }; + + try { + // ── 2. Find store ── + let store = null; + const storeName = ${JSON.stringify(storeName)}; + const fw = ${JSON.stringify(framework)}; + + // Auto-detect framework if not specified + const app = document.querySelector('#app'); + if (!fw || fw === 'pinia') { + // Try Pinia (Vue 3) + try { + const pinia = app?.__vue_app__?.config?.globalProperties?.$pinia; + if (pinia?._s) store = pinia._s.get(storeName); + } catch {} + } + if (!store && (!fw || fw === 'vuex')) { + // Try Vuex (Vue 2/3) + try { + const vuexStore = app?.__vue_app__?.config?.globalProperties?.$store + ?? app?.__vue__?.$store; + if (vuexStore) { + // Vuex doesn't have named stores like Pinia, dispatch action + store = { [${JSON.stringify(actionName)}]: (...a) => vuexStore.dispatch(storeName + '/' + ${JSON.stringify(actionName)}, ...a) }; + } + } catch {} + } + + if (!store) return { error: 'Store not found: ' + storeName, hint: 'Page may not be fully loaded or store name may be incorrect' }; + if (typeof store[${JSON.stringify(actionName)}] !== 'function') { + return { error: 'Action not found: ' + ${JSON.stringify(actionName)} + ' on store ' + storeName, + hint: 'Available: ' + Object.keys(store).filter(k => typeof store[k] === 'function' && !k.startsWith('$') && !k.startsWith('_')).join(', ') }; + } + + // ── 3. Call store action ── + await ${actionCall}; + + // ── 4. Wait for network response ── + const deadline = Date.now() + ${timeout} * 1000; + while (!captured && Date.now() < deadline) { + await new Promise(r => setTimeout(r, 200)); + } + } finally { + // ── 5. Always restore originals ── + window.fetch = origFetch; + XMLHttpRequest.prototype.open = origXhrOpen; + XMLHttpRequest.prototype.send = origXhrSend; + } + + if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern }; + return captured${selectChain} ?? captured; + } + `; + + return page.evaluate(js); + } default: return data; } }