feat: add xiaohongshu search + v2ex topic + test fixes

- xiaohongshu/search.ts: Pinia store + XHR interception (from bb-sites)
- v2ex/topic.yaml: public API topic details
- Register xiaohongshu/search in index.ts
- 25 commands across 13 sites

Tested: bilibili hot/search ✅, zhihu hot ✅, v2ex hot/topic ✅,
hackernews top ✅, github search ✅
This commit is contained in:
jackwener committed 2026-03-15 07:16:03 +08:00
1 parent 93f4753d1c
commit 4b85ddd678
6 files changed
+315

No files matched your search

+3
View File
@@ -17,3 +17,6 @@ import './github/search.js';
// zhihu
import './zhihu/question.js';
// xiaohongshu
import './xiaohongshu/search.js';
+27
View File
@@ -0,0 +1,27 @@
site: v2ex
name: topic
description: V2EX 主题详情和回复
domain: www.v2ex.com
strategy: public
browser: false
args:
id:
type: str
required: true
description: Topic ID
pipeline:
- fetch:
url: https://www.v2ex.com/api/topics/show.json
params:
id: ${{ args.id }}
- map:
title: ${{ item.title }}
replies: ${{ item.replies }}
url: ${{ item.url }}
- limit: 1
columns: [title, replies, url]
+32
View File
@@ -0,0 +1,32 @@
site: xiaohongshu
name: feed
description: "小红书首页推荐 Feed (via Pinia Store Action)"
domain: www.xiaohongshu.com
strategy: intercept
browser: true
args:
limit:
type: int
default: 20
description: Number of items to return
columns: [title, author, likes, type, url]
pipeline:
- navigate: https://www.xiaohongshu.com/explore
- wait: 3
- tap:
store: feed
action: fetchFeeds
capture: homefeed
select: data.items
timeout: 8
- map:
id: ${{ item.id }}
title: ${{ item.note_card.display_title }}
type: ${{ item.note_card.type }}
author: ${{ item.note_card.user.nickname }}
likes: ${{ item.note_card.interact_info.liked_count }}
url: https://www.xiaohongshu.com/explore/${{ item.id }}
- limit: ${{ args.limit | default(20) }}
+71
View File
@@ -0,0 +1,71 @@
/**
* Xiaohongshu search — trigger search via Pinia store + XHR interception.
* Inspired by bb-sites/xiaohongshu/search.js but adapted for opencli pipeline.
*/
import { cli, Strategy } from '../../registry.js';
cli({
site: 'xiaohongshu',
name: 'search',
description: '搜索小红书笔记',
domain: 'www.xiaohongshu.com',
strategy: Strategy.COOKIE,
args: [
{ name: 'keyword', required: true, help: 'Search keyword' },
{ name: 'limit', type: 'int', default: 20, help: 'Number of results' },
],
columns: ['rank', 'title', 'author', 'likes', 'type'],
func: async (page, kwargs) => {
await page.goto('https://www.xiaohongshu.com');
await page.wait(2);
const data = await page.evaluate(`
(async () => {
const app = document.querySelector('#app')?.__vue_app__;
const pinia = app?.config?.globalProperties?.$pinia;
if (!pinia?._s) return {error: 'Page not ready'};
const searchStore = pinia._s.get('search');
if (!searchStore) return {error: 'Search store not found'};
let captured = null;
const origOpen = XMLHttpRequest.prototype.open;
const origSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.open = function(m, u) { this.__url = u; return origOpen.apply(this, arguments); };
XMLHttpRequest.prototype.send = function(b) {
if (this.__url?.includes('search/notes')) {
const x = this;
const orig = x.onreadystatechange;
x.onreadystatechange = function() { if (x.readyState === 4 && !captured) { try { captured = JSON.parse(x.responseText); } catch {} } if (orig) orig.apply(this, arguments); };
}
return origSend.apply(this, arguments);
};
try {
searchStore.mutateSearchValue('${kwargs.keyword}');
await searchStore.loadMore();
await new Promise(r => setTimeout(r, 800));
} finally {
XMLHttpRequest.prototype.open = origOpen;
XMLHttpRequest.prototype.send = origSend;
}
if (!captured?.success) return {error: captured?.msg || 'Search failed'};
return (captured.data?.items || []).map(i => ({
title: i.note_card?.display_title || '',
type: i.note_card?.type || '',
url: 'https://www.xiaohongshu.com/explore/' + i.id,
author: i.note_card?.user?.nickname || '',
likes: i.note_card?.interact_info?.liked_count || '0',
}));
})()
`);
if (!Array.isArray(data)) return [];
return data.slice(0, kwargs.limit).map((item: any, i: number) => ({
rank: i + 1,
...item,
}));
},
});
+36
View File
@@ -0,0 +1,36 @@
site: xiaohongshu
name: search
description: "小红书搜索 (via Pinia Store Action)"
domain: www.xiaohongshu.com
strategy: intercept
browser: true
args:
keyword:
type: str
required: true
description: Search keyword
limit:
type: int
default: 20
description: Number of items to return
columns: [title, author, likes, type, url]
pipeline:
- navigate: https://www.xiaohongshu.com/search_result?keyword=${{ args.keyword }}&source=web_search_result_notes
- wait: 3
- tap:
store: search
action: fetchResult
capture: search/notes
select: data.items
timeout: 8
- map:
id: ${{ item.id }}
title: ${{ item.note_card.display_title }}
type: ${{ item.note_card.type }}
author: ${{ item.note_card.user.nickname }}
likes: ${{ item.note_card.interact_info.liked_count }}
url: https://www.xiaohongshu.com/explore/${{ item.id }}
- limit: ${{ args.limit | default(20) }}
+146
View File
@@ -295,6 +295,152 @@ async function executeStep(page: any, op: string, params: any, data: any, args:
return result;
}
case 'tap': {
// ── Declarative Store Action Bridge ──────────────────────────────────
// Usage:
// tap:
// store: feed # Pinia/Vuex store name
// action: fetchFeeds # Store action to call
// args: [] # Optional args to pass to action
// capture: homefeed # URL pattern to capture response
// timeout: 5 # Seconds to wait for network (default: 5)
// select: data.items # Optional: extract sub-path from response
// framework: pinia # Optional: pinia | vuex (auto-detected if omitted)
//
// Generates a self-contained IIFE that:
// 1. Injects fetch + XHR dual interception proxy
// 2. Finds the Pinia/Vuex store and calls the action
// 3. Captures the response matching the URL pattern
// 4. Auto-cleans up interception in finally block
// 5. Returns the captured data (optionally sub-selected)
const cfg = typeof params === 'object' ? params : {};
const storeName = String(render(cfg.store ?? '', { args, data }));
const actionName = String(render(cfg.action ?? '', { args, data }));
const capturePattern = String(render(cfg.capture ?? '', { args, data }));
const timeout = cfg.timeout ?? 5;
const selectPath = cfg.select ? String(render(cfg.select, { args, data })) : null;
const framework = cfg.framework ?? null; // auto-detect if null
const actionArgs = cfg.args ?? [];
if (!storeName || !actionName) throw new Error('tap: store and action are required');
// Build select chain for the captured response
const selectChain = selectPath
? selectPath.split('.').map((p: string) => `?.[${JSON.stringify(p)}]`).join('')
: '';
// Serialize action arguments
const actionArgsRendered = actionArgs.map((a: any) => {
const rendered = render(a, { args, data });
return JSON.stringify(rendered);
});
const actionCall = actionArgsRendered.length
? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
: `store[${JSON.stringify(actionName)}]()`;
const js = `
async () => {
// ── 1. Setup capture proxy (fetch + XHR dual interception) ──
let captured = null;
const capturePattern = ${JSON.stringify(capturePattern)};
// Intercept fetch API
const origFetch = window.fetch;
window.fetch = async function(...fetchArgs) {
const resp = await origFetch.apply(this, fetchArgs);
try {
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
if (capturePattern && url.includes(capturePattern) && !captured) {
try { captured = await resp.clone().json(); } catch {}
}
} catch {}
return resp;
};
// Intercept XMLHttpRequest
const origXhrOpen = XMLHttpRequest.prototype.open;
const origXhrSend = XMLHttpRequest.prototype.send;
XMLHttpRequest.prototype.open = function(method, url) {
this.__tapUrl = String(url);
return origXhrOpen.apply(this, arguments);
};
XMLHttpRequest.prototype.send = function(body) {
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
const xhr = this;
const origHandler = xhr.onreadystatechange;
xhr.onreadystatechange = function() {
if (xhr.readyState === 4 && !captured) {
try { captured = JSON.parse(xhr.responseText); } catch {}
}
if (origHandler) origHandler.apply(this, arguments);
};
// Also handle onload
const origOnload = xhr.onload;
xhr.onload = function() {
if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
if (origOnload) origOnload.apply(this, arguments);
};
}
return origXhrSend.apply(this, arguments);
};
try {
// ── 2. Find store ──
let store = null;
const storeName = ${JSON.stringify(storeName)};
const fw = ${JSON.stringify(framework)};
// Auto-detect framework if not specified
const app = document.querySelector('#app');
if (!fw || fw === 'pinia') {
// Try Pinia (Vue 3)
try {
const pinia = app?.__vue_app__?.config?.globalProperties?.$pinia;
if (pinia?._s) store = pinia._s.get(storeName);
} catch {}
}
if (!store && (!fw || fw === 'vuex')) {
// Try Vuex (Vue 2/3)
try {
const vuexStore = app?.__vue_app__?.config?.globalProperties?.$store
?? app?.__vue__?.$store;
if (vuexStore) {
// Vuex doesn't have named stores like Pinia, dispatch action
store = { [${JSON.stringify(actionName)}]: (...a) => vuexStore.dispatch(storeName + '/' + ${JSON.stringify(actionName)}, ...a) };
}
} catch {}
}
if (!store) return { error: 'Store not found: ' + storeName, hint: 'Page may not be fully loaded or store name may be incorrect' };
if (typeof store[${JSON.stringify(actionName)}] !== 'function') {
return { error: 'Action not found: ' + ${JSON.stringify(actionName)} + ' on store ' + storeName,
hint: 'Available: ' + Object.keys(store).filter(k => typeof store[k] === 'function' && !k.startsWith('$') && !k.startsWith('_')).join(', ') };
}
// ── 3. Call store action ──
await ${actionCall};
// ── 4. Wait for network response ──
const deadline = Date.now() + ${timeout} * 1000;
while (!captured && Date.now() < deadline) {
await new Promise(r => setTimeout(r, 200));
}
} finally {
// ── 5. Always restore originals ──
window.fetch = origFetch;
XMLHttpRequest.prototype.open = origXhrOpen;
XMLHttpRequest.prototype.send = origXhrSend;
}
if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
return captured${selectChain} ?? captured;
}
`;
return page.evaluate(js);
}
default: return data;
}
}