feat: add xiaohongshu search + v2ex topic + test fixes
- xiaohongshu/search.ts: Pinia store + XHR interception (from bb-sites) - v2ex/topic.yaml: public API topic details - Register xiaohongshu/search in index.ts - 25 commands across 13 sites Tested: bilibili hot/search ✅, zhihu hot ✅, v2ex hot/topic ✅, hackernews top ✅, github search ✅
This commit is contained in:
1 parent
93f4753d1c
commit
4b85ddd678
6 files changed
+315
No files matched your search
@@ -17,3 +17,6 @@ import './github/search.js';
|
||||
|
||||
// zhihu
|
||||
import './zhihu/question.js';
|
||||
|
||||
// xiaohongshu
|
||||
import './xiaohongshu/search.js';
|
||||
@@ -0,0 +1,27 @@
|
||||
site: v2ex
|
||||
name: topic
|
||||
description: V2EX 主题详情和回复
|
||||
domain: www.v2ex.com
|
||||
strategy: public
|
||||
browser: false
|
||||
|
||||
args:
|
||||
id:
|
||||
type: str
|
||||
required: true
|
||||
description: Topic ID
|
||||
|
||||
pipeline:
|
||||
- fetch:
|
||||
url: https://www.v2ex.com/api/topics/show.json
|
||||
params:
|
||||
id: ${{ args.id }}
|
||||
|
||||
- map:
|
||||
title: ${{ item.title }}
|
||||
replies: ${{ item.replies }}
|
||||
url: ${{ item.url }}
|
||||
|
||||
- limit: 1
|
||||
|
||||
columns: [title, replies, url]
|
||||
@@ -0,0 +1,32 @@
|
||||
site: xiaohongshu
|
||||
name: feed
|
||||
description: "小红书首页推荐 Feed (via Pinia Store Action)"
|
||||
domain: www.xiaohongshu.com
|
||||
strategy: intercept
|
||||
browser: true
|
||||
|
||||
args:
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: Number of items to return
|
||||
|
||||
columns: [title, author, likes, type, url]
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.xiaohongshu.com/explore
|
||||
- wait: 3
|
||||
- tap:
|
||||
store: feed
|
||||
action: fetchFeeds
|
||||
capture: homefeed
|
||||
select: data.items
|
||||
timeout: 8
|
||||
- map:
|
||||
id: ${{ item.id }}
|
||||
title: ${{ item.note_card.display_title }}
|
||||
type: ${{ item.note_card.type }}
|
||||
author: ${{ item.note_card.user.nickname }}
|
||||
likes: ${{ item.note_card.interact_info.liked_count }}
|
||||
url: https://www.xiaohongshu.com/explore/${{ item.id }}
|
||||
- limit: ${{ args.limit | default(20) }}
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Xiaohongshu search — trigger search via Pinia store + XHR interception.
|
||||
* Inspired by bb-sites/xiaohongshu/search.js but adapted for opencli pipeline.
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'xiaohongshu',
|
||||
name: 'search',
|
||||
description: '搜索小红书笔记',
|
||||
domain: 'www.xiaohongshu.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'keyword', required: true, help: 'Search keyword' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of results' },
|
||||
],
|
||||
columns: ['rank', 'title', 'author', 'likes', 'type'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.xiaohongshu.com');
|
||||
await page.wait(2);
|
||||
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const app = document.querySelector('#app')?.__vue_app__;
|
||||
const pinia = app?.config?.globalProperties?.$pinia;
|
||||
if (!pinia?._s) return {error: 'Page not ready'};
|
||||
|
||||
const searchStore = pinia._s.get('search');
|
||||
if (!searchStore) return {error: 'Search store not found'};
|
||||
|
||||
let captured = null;
|
||||
const origOpen = XMLHttpRequest.prototype.open;
|
||||
const origSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(m, u) { this.__url = u; return origOpen.apply(this, arguments); };
|
||||
XMLHttpRequest.prototype.send = function(b) {
|
||||
if (this.__url?.includes('search/notes')) {
|
||||
const x = this;
|
||||
const orig = x.onreadystatechange;
|
||||
x.onreadystatechange = function() { if (x.readyState === 4 && !captured) { try { captured = JSON.parse(x.responseText); } catch {} } if (orig) orig.apply(this, arguments); };
|
||||
}
|
||||
return origSend.apply(this, arguments);
|
||||
};
|
||||
|
||||
try {
|
||||
searchStore.mutateSearchValue('${kwargs.keyword}');
|
||||
await searchStore.loadMore();
|
||||
await new Promise(r => setTimeout(r, 800));
|
||||
} finally {
|
||||
XMLHttpRequest.prototype.open = origOpen;
|
||||
XMLHttpRequest.prototype.send = origSend;
|
||||
}
|
||||
|
||||
if (!captured?.success) return {error: captured?.msg || 'Search failed'};
|
||||
return (captured.data?.items || []).map(i => ({
|
||||
title: i.note_card?.display_title || '',
|
||||
type: i.note_card?.type || '',
|
||||
url: 'https://www.xiaohongshu.com/explore/' + i.id,
|
||||
author: i.note_card?.user?.nickname || '',
|
||||
likes: i.note_card?.interact_info?.liked_count || '0',
|
||||
}));
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!Array.isArray(data)) return [];
|
||||
return data.slice(0, kwargs.limit).map((item: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
...item,
|
||||
}));
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
site: xiaohongshu
|
||||
name: search
|
||||
description: "小红书搜索 (via Pinia Store Action)"
|
||||
domain: www.xiaohongshu.com
|
||||
strategy: intercept
|
||||
browser: true
|
||||
|
||||
args:
|
||||
keyword:
|
||||
type: str
|
||||
required: true
|
||||
description: Search keyword
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: Number of items to return
|
||||
|
||||
columns: [title, author, likes, type, url]
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.xiaohongshu.com/search_result?keyword=${{ args.keyword }}&source=web_search_result_notes
|
||||
- wait: 3
|
||||
- tap:
|
||||
store: search
|
||||
action: fetchResult
|
||||
capture: search/notes
|
||||
select: data.items
|
||||
timeout: 8
|
||||
- map:
|
||||
id: ${{ item.id }}
|
||||
title: ${{ item.note_card.display_title }}
|
||||
type: ${{ item.note_card.type }}
|
||||
author: ${{ item.note_card.user.nickname }}
|
||||
likes: ${{ item.note_card.interact_info.liked_count }}
|
||||
url: https://www.xiaohongshu.com/explore/${{ item.id }}
|
||||
- limit: ${{ args.limit | default(20) }}
|
||||
+146
@@ -295,6 +295,152 @@ async function executeStep(page: any, op: string, params: any, data: any, args:
|
||||
|
||||
return result;
|
||||
}
|
||||
case 'tap': {
|
||||
// ── Declarative Store Action Bridge ──────────────────────────────────
|
||||
// Usage:
|
||||
// tap:
|
||||
// store: feed # Pinia/Vuex store name
|
||||
// action: fetchFeeds # Store action to call
|
||||
// args: [] # Optional args to pass to action
|
||||
// capture: homefeed # URL pattern to capture response
|
||||
// timeout: 5 # Seconds to wait for network (default: 5)
|
||||
// select: data.items # Optional: extract sub-path from response
|
||||
// framework: pinia # Optional: pinia | vuex (auto-detected if omitted)
|
||||
//
|
||||
// Generates a self-contained IIFE that:
|
||||
// 1. Injects fetch + XHR dual interception proxy
|
||||
// 2. Finds the Pinia/Vuex store and calls the action
|
||||
// 3. Captures the response matching the URL pattern
|
||||
// 4. Auto-cleans up interception in finally block
|
||||
// 5. Returns the captured data (optionally sub-selected)
|
||||
|
||||
const cfg = typeof params === 'object' ? params : {};
|
||||
const storeName = String(render(cfg.store ?? '', { args, data }));
|
||||
const actionName = String(render(cfg.action ?? '', { args, data }));
|
||||
const capturePattern = String(render(cfg.capture ?? '', { args, data }));
|
||||
const timeout = cfg.timeout ?? 5;
|
||||
const selectPath = cfg.select ? String(render(cfg.select, { args, data })) : null;
|
||||
const framework = cfg.framework ?? null; // auto-detect if null
|
||||
const actionArgs = cfg.args ?? [];
|
||||
|
||||
if (!storeName || !actionName) throw new Error('tap: store and action are required');
|
||||
|
||||
// Build select chain for the captured response
|
||||
const selectChain = selectPath
|
||||
? selectPath.split('.').map((p: string) => `?.[${JSON.stringify(p)}]`).join('')
|
||||
: '';
|
||||
|
||||
// Serialize action arguments
|
||||
const actionArgsRendered = actionArgs.map((a: any) => {
|
||||
const rendered = render(a, { args, data });
|
||||
return JSON.stringify(rendered);
|
||||
});
|
||||
const actionCall = actionArgsRendered.length
|
||||
? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
|
||||
: `store[${JSON.stringify(actionName)}]()`;
|
||||
|
||||
const js = `
|
||||
async () => {
|
||||
// ── 1. Setup capture proxy (fetch + XHR dual interception) ──
|
||||
let captured = null;
|
||||
const capturePattern = ${JSON.stringify(capturePattern)};
|
||||
|
||||
// Intercept fetch API
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...fetchArgs) {
|
||||
const resp = await origFetch.apply(this, fetchArgs);
|
||||
try {
|
||||
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
|
||||
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
|
||||
if (capturePattern && url.includes(capturePattern) && !captured) {
|
||||
try { captured = await resp.clone().json(); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
return resp;
|
||||
};
|
||||
|
||||
// Intercept XMLHttpRequest
|
||||
const origXhrOpen = XMLHttpRequest.prototype.open;
|
||||
const origXhrSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(method, url) {
|
||||
this.__tapUrl = String(url);
|
||||
return origXhrOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(body) {
|
||||
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
|
||||
const xhr = this;
|
||||
const origHandler = xhr.onreadystatechange;
|
||||
xhr.onreadystatechange = function() {
|
||||
if (xhr.readyState === 4 && !captured) {
|
||||
try { captured = JSON.parse(xhr.responseText); } catch {}
|
||||
}
|
||||
if (origHandler) origHandler.apply(this, arguments);
|
||||
};
|
||||
// Also handle onload
|
||||
const origOnload = xhr.onload;
|
||||
xhr.onload = function() {
|
||||
if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
|
||||
if (origOnload) origOnload.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
return origXhrSend.apply(this, arguments);
|
||||
};
|
||||
|
||||
try {
|
||||
// ── 2. Find store ──
|
||||
let store = null;
|
||||
const storeName = ${JSON.stringify(storeName)};
|
||||
const fw = ${JSON.stringify(framework)};
|
||||
|
||||
// Auto-detect framework if not specified
|
||||
const app = document.querySelector('#app');
|
||||
if (!fw || fw === 'pinia') {
|
||||
// Try Pinia (Vue 3)
|
||||
try {
|
||||
const pinia = app?.__vue_app__?.config?.globalProperties?.$pinia;
|
||||
if (pinia?._s) store = pinia._s.get(storeName);
|
||||
} catch {}
|
||||
}
|
||||
if (!store && (!fw || fw === 'vuex')) {
|
||||
// Try Vuex (Vue 2/3)
|
||||
try {
|
||||
const vuexStore = app?.__vue_app__?.config?.globalProperties?.$store
|
||||
?? app?.__vue__?.$store;
|
||||
if (vuexStore) {
|
||||
// Vuex doesn't have named stores like Pinia, dispatch action
|
||||
store = { [${JSON.stringify(actionName)}]: (...a) => vuexStore.dispatch(storeName + '/' + ${JSON.stringify(actionName)}, ...a) };
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
if (!store) return { error: 'Store not found: ' + storeName, hint: 'Page may not be fully loaded or store name may be incorrect' };
|
||||
if (typeof store[${JSON.stringify(actionName)}] !== 'function') {
|
||||
return { error: 'Action not found: ' + ${JSON.stringify(actionName)} + ' on store ' + storeName,
|
||||
hint: 'Available: ' + Object.keys(store).filter(k => typeof store[k] === 'function' && !k.startsWith('$') && !k.startsWith('_')).join(', ') };
|
||||
}
|
||||
|
||||
// ── 3. Call store action ──
|
||||
await ${actionCall};
|
||||
|
||||
// ── 4. Wait for network response ──
|
||||
const deadline = Date.now() + ${timeout} * 1000;
|
||||
while (!captured && Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
}
|
||||
} finally {
|
||||
// ── 5. Always restore originals ──
|
||||
window.fetch = origFetch;
|
||||
XMLHttpRequest.prototype.open = origXhrOpen;
|
||||
XMLHttpRequest.prototype.send = origXhrSend;
|
||||
}
|
||||
|
||||
if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
|
||||
return captured${selectChain} ?? captured;
|
||||
}
|
||||
`;
|
||||
|
||||
return page.evaluate(js);
|
||||
}
|
||||
default: return data;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user