diff --git a/src/clis/bilibili/dynamic.ts b/src/clis/bilibili/dynamic.ts new file mode 100644 index 0000000..6c94d13 --- /dev/null +++ b/src/clis/bilibili/dynamic.ts @@ -0,0 +1,34 @@ +import { cli, Strategy } from '../../registry.js'; +import { apiGet } from '../../bilibili.js'; + +cli({ + site: 'bilibili', + name: 'dynamic', + description: 'Get Bilibili user dynamic feed', + domain: 'www.bilibili.com', + strategy: Strategy.COOKIE, + args: [ + { name: 'limit', type: 'int', default: 15 }, + ], + columns: ['id', 'author', 'text', 'likes', 'url'], + func: async (page, kwargs) => { + const payload = await apiGet(page, '/x/polymer/web-dynamic/v1/feed/all', { params: {}, signed: false }); + const results: any[] = payload?.data?.items ?? []; + return results.slice(0, Number(kwargs.limit)).map((item: any) => { + let text = ''; + if (item.modules?.module_dynamic?.desc?.text) { + text = item.modules.module_dynamic.desc.text; + } else if (item.modules?.module_dynamic?.major?.archive?.title) { + text = item.modules.module_dynamic.major.archive.title; + } + + return { + id: item.id_str ?? '', + author: item.modules?.module_author?.name ?? '', + text: text, + likes: item.modules?.module_stat?.like?.count ?? 0, + url: item.id_str ? `https://t.bilibili.com/${item.id_str}` : '' + }; + }); + }, +}); diff --git a/src/clis/bilibili/ranking.ts b/src/clis/bilibili/ranking.ts new file mode 100644 index 0000000..54adc95 --- /dev/null +++ b/src/clis/bilibili/ranking.ts @@ -0,0 +1,25 @@ +import { cli, Strategy } from '../../registry.js'; +import { apiGet } from '../../bilibili.js'; + +cli({ + site: 'bilibili', + name: 'ranking', + description: 'Get Bilibili video ranking board', + domain: 'www.bilibili.com', + strategy: Strategy.COOKIE, + args: [ + { name: 'limit', type: 'int', default: 20 }, + ], + columns: ['rank', 'title', 'author', 'score', 'url'], + func: async (page, kwargs) => { + const payload = await apiGet(page, '/x/web-interface/ranking/v2', { params: { rid: 0, type: 'all' }, signed: false }); + const results: any[] = payload?.data?.list ?? []; + return results.slice(0, Number(kwargs.limit)).map((item: any, i: number) => ({ + rank: i + 1, + title: item.title ?? '', + author: item.owner?.name ?? '', + score: item.stat?.view ?? 0, + url: item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : '' + })); + }, +}); diff --git a/src/clis/index.ts b/src/clis/index.ts index 4f30505..e603c53 100644 --- a/src/clis/index.ts +++ b/src/clis/index.ts @@ -11,6 +11,8 @@ import './bilibili/favorite.js'; import './bilibili/history.js'; import './bilibili/feed.js'; import './bilibili/user-videos.js'; +import './bilibili/ranking.js'; +import './bilibili/dynamic.js'; // github import './github/search.js'; @@ -20,6 +22,7 @@ import './zhihu/question.js'; // xiaohongshu import './xiaohongshu/search.js'; +import './xiaohongshu/user.js'; // bbc import './bbc/news.js'; @@ -44,3 +47,9 @@ import './ctrip/search.js'; // youtube import './youtube/search.js'; + +// twitter +import './twitter/search.js'; +import './twitter/profile.js'; + +// reddit diff --git a/src/clis/reddit/frontpage.yaml b/src/clis/reddit/frontpage.yaml new file mode 100644 index 0000000..32df6ac --- /dev/null +++ b/src/clis/reddit/frontpage.yaml @@ -0,0 +1,30 @@ +site: reddit +name: frontpage +description: Reddit Frontpage / r/all +domain: reddit.com +strategy: cookie +browser: true + +args: + limit: + type: int + default: 15 + +columns: [title, subreddit, author, upvotes, comments, url] + +pipeline: + - navigate: https://www.reddit.com + - evaluate: | + (async () => { + const res = await fetch('/r/all.json?limit=${{ args.limit }}', { credentials: 'include' }); + const j = await res.json(); + return j?.data?.children || []; + })() + - map: + title: ${{ item.data.title }} + subreddit: ${{ item.data.subreddit_name_prefixed }} + author: ${{ item.data.author }} + upvotes: ${{ item.data.score }} + comments: ${{ item.data.num_comments }} + url: https://www.reddit.com${{ item.data.permalink }} + - limit: ${{ args.limit }} diff --git a/src/clis/reddit/search.yaml b/src/clis/reddit/search.yaml new file mode 100644 index 0000000..83dab93 --- /dev/null +++ b/src/clis/reddit/search.yaml @@ -0,0 +1,34 @@ +site: reddit +name: search +description: Search Reddit Posts +domain: reddit.com +strategy: cookie +browser: true + +args: + query: + type: string + required: true + limit: + type: int + default: 15 + +columns: [title, subreddit, author, upvotes, comments, url] + +pipeline: + - navigate: https://www.reddit.com + - evaluate: | + (async () => { + const q = encodeURIComponent('${{ args.query }}'); + const res = await fetch('/search.json?q=' + q + '&limit=${{ args.limit }}', { credentials: 'include' }); + const j = await res.json(); + return j?.data?.children || []; + })() + - map: + title: ${{ item.data.title }} + subreddit: ${{ item.data.subreddit_name_prefixed }} + author: ${{ item.data.author }} + upvotes: ${{ item.data.score }} + comments: ${{ item.data.num_comments }} + url: https://www.reddit.com${{ item.data.permalink }} + - limit: ${{ args.limit }} diff --git a/src/clis/reddit/subreddit.yaml b/src/clis/reddit/subreddit.yaml new file mode 100644 index 0000000..fe2f630 --- /dev/null +++ b/src/clis/reddit/subreddit.yaml @@ -0,0 +1,39 @@ +site: reddit +name: subreddit +description: Get posts from a specific Subreddit +domain: reddit.com +strategy: cookie +browser: true + +args: + name: + type: string + required: true + sort: + type: string + default: hot + description: "Sorting method: hot, new, top, rising" + limit: + type: int + default: 15 + +columns: [title, author, upvotes, comments, url] + +pipeline: + - navigate: https://www.reddit.com + - evaluate: | + (async () => { + let sub = '${{ args.name }}'; + if (sub.startsWith('r/')) sub = sub.slice(2); + const sort = '${{ args.sort }}'; + const res = await fetch('/r/' + sub + '/' + sort + '.json?limit=${{ args.limit }}', { credentials: 'include' }); + const j = await res.json(); + return j?.data?.children || []; + })() + - map: + title: ${{ item.data.title }} + author: ${{ item.data.author }} + upvotes: ${{ item.data.score }} + comments: ${{ item.data.num_comments }} + url: https://www.reddit.com${{ item.data.permalink }} + - limit: ${{ args.limit }} diff --git a/src/clis/twitter/profile.ts b/src/clis/twitter/profile.ts new file mode 100644 index 0000000..0411ebb --- /dev/null +++ b/src/clis/twitter/profile.ts @@ -0,0 +1,86 @@ +import { cli, Strategy } from '../../registry.js'; + +cli({ + site: 'twitter', + name: 'profile', + description: 'Fetch tweets from a user profile', + domain: 'x.com', + strategy: Strategy.INTERCEPT, + browser: true, + args: [ + { name: 'username', type: 'string', required: true }, + { name: 'limit', type: 'int', default: 15 }, + ], + columns: ['id', 'text', 'likes', 'views', 'url'], + func: async (page, kwargs) => { + // Navigate to user profile via search for reliability + await page.goto(`https://x.com/search?q=from:${kwargs.username}&f=live`); + await page.wait(5); + + // Inject XHR interceptor + await page.evaluate(` + () => { + window.__opencli_xhr = []; + if (!window.__patched_xhr) { + const XHR = XMLHttpRequest.prototype; + const open = XHR.open; + const send = XHR.send; + XHR.open = function(method, url) { + this._url = url; + return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2)); + }; + XHR.send = function() { + this.addEventListener('load', function() { + if(this._url.includes('SearchTimeline')) { + try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){} + } + }); + return send.apply(this, arguments); + }; + window.__patched_xhr = true; + } + } + `); + + // Trigger API by scrolling + for (let i = 0; i < 3; i++) { + await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)'); + await page.wait(2); + } + + // Retrieve data + const requests = await page.evaluate('() => window.__opencli_xhr'); + if (!requests || requests.length === 0) return []; + + let results: any[] = []; + for (const req of requests) { + try { + const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions; + const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries'); + if (!addEntries) continue; + + for (const entry of addEntries.entries) { + if (!entry.entryId.startsWith('tweet-')) continue; + + let tweet = entry.content?.itemContent?.tweet_results?.result; + if (!tweet) continue; + + if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) { + tweet = tweet.tweet; + } + + results.push({ + id: tweet.rest_id, + text: tweet.legacy?.full_text || '', + likes: tweet.legacy?.favorite_count || 0, + views: tweet.views?.count || '0', + url: `https://x.com/i/status/${tweet.rest_id}` + }); + } + } catch (e) { + } + } + + return results.slice(0, kwargs.limit); + } +}); diff --git a/src/clis/twitter/search.ts b/src/clis/twitter/search.ts new file mode 100644 index 0000000..725f65d --- /dev/null +++ b/src/clis/twitter/search.ts @@ -0,0 +1,90 @@ +import { cli, Strategy } from '../../registry.js'; + +cli({ + site: 'twitter', + name: 'search', + description: 'Search Twitter/X for tweets', + domain: 'x.com', + strategy: Strategy.INTERCEPT, // Use intercept strategy + browser: true, + args: [ + { name: 'query', type: 'string', required: true }, + { name: 'limit', type: 'int', default: 15 }, + ], + columns: ['id', 'author', 'text', 'likes', 'views', 'url'], + func: async (page, kwargs) => { + // 1. Navigate to the search page + const q = encodeURIComponent(kwargs.query); + await page.goto(`https://x.com/search?q=${q}&f=top`); + await page.wait(5); + + // 2. Inject XHR interceptor + await page.evaluate(` + () => { + window.__opencli_xhr = []; + if (!window.__patched_xhr) { + const XHR = XMLHttpRequest.prototype; + const open = XHR.open; + const send = XHR.send; + XHR.open = function(method, url) { + this._url = url; + return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2)); + }; + XHR.send = function() { + this.addEventListener('load', function() { + if(this._url.includes('SearchTimeline')) { + try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){} + } + }); + return send.apply(this, arguments); + }; + window.__patched_xhr = true; + } + } + `); + + // 3. Trigger API by scrolling + for (let i = 0; i < 3; i++) { + await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)'); + await page.wait(2); + } + + // 4. Retrieve data + const requests = await page.evaluate('() => window.__opencli_xhr'); + if (!requests || requests.length === 0) return []; + + let results: any[] = []; + for (const req of requests) { + try { + const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions; + const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries'); + if (!addEntries) continue; + + for (const entry of addEntries.entries) { + if (!entry.entryId.startsWith('tweet-')) continue; + + let tweet = entry.content?.itemContent?.tweet_results?.result; + if (!tweet) continue; + + // Handle retweet wrapping + if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) { + tweet = tweet.tweet; + } + + results.push({ + id: tweet.rest_id, + author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown', + text: tweet.legacy?.full_text || '', + likes: tweet.legacy?.favorite_count || 0, + views: tweet.views?.count || '0', + url: `https://x.com/i/status/${tweet.rest_id}` + }); + } + } catch (e) { + // ignore parsing errors for individual payloads + } + } + + return results.slice(0, kwargs.limit); + } +}); diff --git a/src/clis/twitter/timeline.ts b/src/clis/twitter/timeline.ts new file mode 100644 index 0000000..b18485c --- /dev/null +++ b/src/clis/twitter/timeline.ts @@ -0,0 +1,50 @@ +import { cli, Strategy } from '../../registry.js'; + +cli({ + site: 'twitter', + name: 'timeline', + description: 'Twitter Home Timeline', + domain: 'x.com', + strategy: Strategy.COOKIE, + args: [ + { name: 'limit', type: 'int', default: 20 }, + ], + columns: ['responseType', 'first'], + func: async (page, kwargs) => { + await page.goto('https://x.com/home'); + await page.wait(5); + // Inject the fetch interceptor manually to see exactly what happens + await page.evaluate(` + () => { + window.__intercept_data = []; + const origFetch = window.fetch; + window.fetch = async function(...args) { + let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || ''; + const res = await origFetch.apply(this, args); + setTimeout(async () => { + try { + if (u.includes('HomeTimeline')) { + const clone = res.clone(); + const j = await clone.json(); + window.__intercept_data.push(j); + } + } catch(e) {} + }, 0); + return res; + }; + } + `); + + // trigger scroll + for(let i=0; i<3; i++) { + await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)'); + await page.wait(2); + } + + // extract + const data = await page.evaluate('() => window.__intercept_data'); + if (!data || data.length === 0) return [{responseType: 'no data captured'}]; + + return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}]; + } +}); diff --git a/src/clis/xiaohongshu/user.ts b/src/clis/xiaohongshu/user.ts new file mode 100644 index 0000000..a014d6a --- /dev/null +++ b/src/clis/xiaohongshu/user.ts @@ -0,0 +1,86 @@ +import { cli, Strategy } from '../../registry.js'; + +cli({ + site: 'xiaohongshu', + name: 'user', + description: 'Get user notes from Xiaohongshu', + domain: 'xiaohongshu.com', + strategy: Strategy.INTERCEPT, + browser: true, + args: [ + { name: 'id', type: 'string', required: true }, + { name: 'limit', type: 'int', default: 15 }, + ], + columns: ['id', 'title', 'type', 'likes', 'url'], + func: async (page, kwargs) => { + await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`); + await page.wait(5); + + await page.evaluate(` + () => { + window.__opencli_xhr = []; + if (!window.__patched_xhr) { + const origFetch = window.fetch; + window.fetch = async function(...args) { + let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || ''; + const res = await origFetch.apply(this, args); + setTimeout(async () => { + try { + if (u.includes('v1/user/posted')) { + const clone = res.clone(); + const j = await clone.json(); + window.__opencli_xhr.push({url: u, data: j}); + } + } catch(e) {} + }, 0); + return res; + }; + + const XHR = XMLHttpRequest.prototype; + const open = XHR.open; + const send = XHR.send; + XHR.open = function(method, url) { + this._url = url; + return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2)); + }; + XHR.send = function() { + this.addEventListener('load', function() { + if(this._url.includes('v1/user/posted')) { + try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){} + } + }); + return send.apply(this, arguments); + }; + window.__patched_xhr = true; + } + } + `); + + // Trigger API by scrolling + for (let i = 0; i < 2; i++) { + await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)'); + await page.wait(2); + } + + // Retrieve data + const requests = await page.evaluate('() => window.__opencli_xhr'); + if (!requests || requests.length === 0) return []; + + let results: any[] = []; + for (const req of requests) { + if (req.data && req.data.data && req.data.data.notes) { + for (const note of req.data.data.notes) { + results.push({ + id: note.note_id || note.id, + title: note.display_title || '', + type: note.type || '', + likes: note.interact_info?.liked_count || '0', + url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}` + }); + } + } + } + + return results.slice(0, kwargs.limit); + } +}); diff --git a/src/pipeline/steps/intercept.ts b/src/pipeline/steps/intercept.ts index 1962f9f..3967cdc 100644 --- a/src/pipeline/steps/intercept.ts +++ b/src/pipeline/steps/intercept.ts @@ -14,7 +14,55 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R if (!capturePattern) return data; - // Step 1: Execute the trigger action + // Step 1: Inject fetch/XHR interceptor BEFORE trigger + await page.evaluate(` + () => { + window.__opencli_intercepted = window.__opencli_intercepted || []; + const pattern = ${JSON.stringify(capturePattern)}; + + if (!window.__opencli_fetch_patched) { + const origFetch = window.fetch; + window.fetch = async function(...args) { + const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || ''; + const response = await origFetch.apply(this, args); + setTimeout(async () => { + try { + if (reqUrl.includes(pattern)) { + const clone = response.clone(); + const json = await clone.json(); + window.__opencli_intercepted.push(json); + } + } catch(e) {} + }, 0); + return response; + }; + window.__opencli_fetch_patched = true; + } + + if (!window.__opencli_xhr_patched) { + const XHR = XMLHttpRequest.prototype; + const open = XHR.open; + const send = XHR.send; + XHR.open = function(method, url, ...args) { + this._reqUrl = url; + return open.call(this, method, url, ...args); + }; + XHR.send = function(...args) { + this.addEventListener('load', function() { + try { + if (this._reqUrl && this._reqUrl.includes(pattern)) { + window.__opencli_intercepted.push(JSON.parse(this.responseText)); + } + } catch(e) {} + }); + return send.apply(this, args); + }; + window.__opencli_xhr_patched = true; + } + } + `); + + // Step 2: Execute the trigger action if (trigger.startsWith('navigate:')) { const url = render(trigger.slice('navigate:'.length), { args, data }); await page.goto(String(url)); @@ -29,36 +77,18 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R await page.scroll('down'); } - // Step 2: Wait a bit for network requests to fire + // Step 3: Wait a bit for network requests to fire await page.wait(Math.min(timeout, 3)); - // Step 3: Get network requests and find matching ones - const rawNetwork = await page.networkRequests(false); - const matchingResponses: any[] = []; - - if (typeof rawNetwork === 'string') { - const lines = rawNetwork.split('\n'); - for (const line of lines) { - const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i); - if (match) { - const [, , url, status] = match; - if (url.includes(capturePattern) && status === '200') { - try { - const body = await page.evaluate(` - async () => { - try { - const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' }); - if (!resp.ok) return null; - return await resp.json(); - } catch { return null; } - } - `); - if (body) matchingResponses.push(body); - } catch {} - } - } + // Step 4: Retrieve captured data + const matchingResponses = await page.evaluate(` + () => { + const data = window.__opencli_intercepted || []; + window.__opencli_intercepted = []; // clear after reading + return data; } - } + `); + // Step 4: Select from response if specified let result = matchingResponses.length === 1 ? matchingResponses[0] :