feat: Add new CLIs for Bilibili, Reddit, Twitter, and Xiaohongshu, incorporating an intercept strategy for Twitter.

src/clis/bilibili/dynamic.ts

@@ -0,0 +1,34 @@
+import { cli, Strategy } from '../../registry.js';
+import { apiGet } from '../../bilibili.js';
+
+cli({
+  site: 'bilibili',
+  name: 'dynamic',
+  description: 'Get Bilibili user dynamic feed',
+  domain: 'www.bilibili.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 15 },
+  ],
+  columns: ['id', 'author', 'text', 'likes', 'url'],
+  func: async (page, kwargs) => {
+    const payload = await apiGet(page, '/x/polymer/web-dynamic/v1/feed/all', { params: {}, signed: false });
+    const results: any[] = payload?.data?.items ?? [];
+    return results.slice(0, Number(kwargs.limit)).map((item: any) => {
+      let text = '';
+      if (item.modules?.module_dynamic?.desc?.text) {
+          text = item.modules.module_dynamic.desc.text;
+      } else if (item.modules?.module_dynamic?.major?.archive?.title) {
+          text = item.modules.module_dynamic.major.archive.title;
+      }
+
+      return {
+        id: item.id_str ?? '',
+        author: item.modules?.module_author?.name ?? '',
+        text: text,
+        likes: item.modules?.module_stat?.like?.count ?? 0,
+        url: item.id_str ? `https://t.bilibili.com/${item.id_str}` : ''
+      };
+    });
+  },
+});

src/clis/bilibili/ranking.ts

@@ -0,0 +1,25 @@
+import { cli, Strategy } from '../../registry.js';
+import { apiGet } from '../../bilibili.js';
+
+cli({
+  site: 'bilibili',
+  name: 'ranking',
+  description: 'Get Bilibili video ranking board',
+  domain: 'www.bilibili.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 20 },
+  ],
+  columns: ['rank', 'title', 'author', 'score', 'url'],
+  func: async (page, kwargs) => {
+    const payload = await apiGet(page, '/x/web-interface/ranking/v2', { params: { rid: 0, type: 'all' }, signed: false });
+    const results: any[] = payload?.data?.list ?? [];
+    return results.slice(0, Number(kwargs.limit)).map((item: any, i: number) => ({
+      rank: i + 1,
+      title: item.title ?? '',
+      author: item.owner?.name ?? '',
+      score: item.stat?.view ?? 0,
+      url: item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : ''
+    }));
+  },
+});

src/clis/index.ts

@@ -11,6 +11,8 @@ import './bilibili/favorite.js';
 import './bilibili/history.js';
 import './bilibili/feed.js';
 import './bilibili/user-videos.js';
+import './bilibili/ranking.js';
+import './bilibili/dynamic.js';
 
 // github
 import './github/search.js';
@@ -20,6 +22,7 @@ import './zhihu/question.js';
 
 // xiaohongshu
 import './xiaohongshu/search.js';
+import './xiaohongshu/user.js';
 
 // bbc
 import './bbc/news.js';
@@ -44,3 +47,9 @@ import './ctrip/search.js';
 
 // youtube
 import './youtube/search.js';
+
+// twitter
+import './twitter/search.js';
+import './twitter/profile.js';
+
+// reddit

src/clis/reddit/frontpage.yaml

@@ -0,0 +1,30 @@
+site: reddit
+name: frontpage
+description: Reddit Frontpage / r/all
+domain: reddit.com
+strategy: cookie
+browser: true
+
+args:
+  limit:
+    type: int
+    default: 15
+
+columns: [title, subreddit, author, upvotes, comments, url]
+
+pipeline:
+  - navigate: https://www.reddit.com
+  - evaluate: |
+      (async () => {
+        const res = await fetch('/r/all.json?limit=${{ args.limit }}', { credentials: 'include' });
+        const j = await res.json();
+        return j?.data?.children || [];
+      })()
+  - map:
+      title: ${{ item.data.title }}
+      subreddit: ${{ item.data.subreddit_name_prefixed }}
+      author: ${{ item.data.author }}
+      upvotes: ${{ item.data.score }}
+      comments: ${{ item.data.num_comments }}
+      url: https://www.reddit.com${{ item.data.permalink }}
+  - limit: ${{ args.limit }}

src/clis/reddit/search.yaml

@@ -0,0 +1,34 @@
+site: reddit
+name: search
+description: Search Reddit Posts
+domain: reddit.com
+strategy: cookie
+browser: true
+
+args:
+  query:
+    type: string
+    required: true
+  limit:
+    type: int
+    default: 15
+
+columns: [title, subreddit, author, upvotes, comments, url]
+
+pipeline:
+  - navigate: https://www.reddit.com
+  - evaluate: |
+      (async () => {
+        const q = encodeURIComponent('${{ args.query }}');
+        const res = await fetch('/search.json?q=' + q + '&limit=${{ args.limit }}', { credentials: 'include' });
+        const j = await res.json();
+        return j?.data?.children || [];
+      })()
+  - map:
+      title: ${{ item.data.title }}
+      subreddit: ${{ item.data.subreddit_name_prefixed }}
+      author: ${{ item.data.author }}
+      upvotes: ${{ item.data.score }}
+      comments: ${{ item.data.num_comments }}
+      url: https://www.reddit.com${{ item.data.permalink }}
+  - limit: ${{ args.limit }}

src/clis/reddit/subreddit.yaml

@@ -0,0 +1,39 @@
+site: reddit
+name: subreddit
+description: Get posts from a specific Subreddit
+domain: reddit.com
+strategy: cookie
+browser: true
+
+args:
+  name:
+    type: string
+    required: true
+  sort:
+    type: string
+    default: hot
+    description: "Sorting method: hot, new, top, rising"
+  limit:
+    type: int
+    default: 15
+
+columns: [title, author, upvotes, comments, url]
+
+pipeline:
+  - navigate: https://www.reddit.com
+  - evaluate: |
+      (async () => {
+        let sub = '${{ args.name }}';
+        if (sub.startsWith('r/')) sub = sub.slice(2);
+        const sort = '${{ args.sort }}';
+        const res = await fetch('/r/' + sub + '/' + sort + '.json?limit=${{ args.limit }}', { credentials: 'include' });
+        const j = await res.json();
+        return j?.data?.children || [];
+      })()
+  - map:
+      title: ${{ item.data.title }}
+      author: ${{ item.data.author }}
+      upvotes: ${{ item.data.score }}
+      comments: ${{ item.data.num_comments }}
+      url: https://www.reddit.com${{ item.data.permalink }}
+  - limit: ${{ args.limit }}

src/clis/twitter/profile.ts

@@ -0,0 +1,86 @@
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'twitter',
+  name: 'profile',
+  description: 'Fetch tweets from a user profile',
+  domain: 'x.com',
+  strategy: Strategy.INTERCEPT,
+  browser: true,
+  args: [
+    { name: 'username', type: 'string', required: true },
+    { name: 'limit', type: 'int', default: 15 },
+  ],
+  columns: ['id', 'text', 'likes', 'views', 'url'],
+  func: async (page, kwargs) => {
+    // Navigate to user profile via search for reliability
+    await page.goto(`https://x.com/search?q=from:${kwargs.username}&f=live`);
+    await page.wait(5);
+
+    // Inject XHR interceptor
+    await page.evaluate(`
+      () => {
+        window.__opencli_xhr = [];
+        if (!window.__patched_xhr) {
+          const XHR = XMLHttpRequest.prototype;
+          const open = XHR.open;
+          const send = XHR.send;
+          XHR.open = function(method, url) {
+            this._url = url;
+            return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
+          };
+          XHR.send = function() {
+            this.addEventListener('load', function() {
+              if(this._url.includes('SearchTimeline')) {
+                try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
+              }
+            });
+            return send.apply(this, arguments);
+          };
+          window.__patched_xhr = true;
+        }
+      }
+    `);
+
+    // Trigger API by scrolling
+    for (let i = 0; i < 3; i++) {
+        await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
+        await page.wait(2);
+    }
+    
+    // Retrieve data
+    const requests = await page.evaluate('() => window.__opencli_xhr');
+    if (!requests || requests.length === 0) return [];
+
+    let results: any[] = [];
+    for (const req of requests) {
+      try {
+        const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
+        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
+        if (!addEntries) continue;
+
+        for (const entry of addEntries.entries) {
+          if (!entry.entryId.startsWith('tweet-')) continue;
+          
+          let tweet = entry.content?.itemContent?.tweet_results?.result;
+          if (!tweet) continue;
+
+          if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
+              tweet = tweet.tweet;
+          }
+
+          results.push({
+            id: tweet.rest_id,
+            text: tweet.legacy?.full_text || '',
+            likes: tweet.legacy?.favorite_count || 0,
+            views: tweet.views?.count || '0',
+            url: `https://x.com/i/status/${tweet.rest_id}`
+          });
+        }
+      } catch (e) {
+      }
+    }
+
+    return results.slice(0, kwargs.limit);
+  }
+});

src/clis/twitter/search.ts

@@ -0,0 +1,90 @@
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'twitter',
+  name: 'search',
+  description: 'Search Twitter/X for tweets',
+  domain: 'x.com',
+  strategy: Strategy.INTERCEPT, // Use intercept strategy
+  browser: true,
+  args: [
+    { name: 'query', type: 'string', required: true },
+    { name: 'limit', type: 'int', default: 15 },
+  ],
+  columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
+  func: async (page, kwargs) => {
+    // 1. Navigate to the search page
+    const q = encodeURIComponent(kwargs.query);
+    await page.goto(`https://x.com/search?q=${q}&f=top`);
+    await page.wait(5);
+
+    // 2. Inject XHR interceptor
+    await page.evaluate(`
+      () => {
+        window.__opencli_xhr = [];
+        if (!window.__patched_xhr) {
+          const XHR = XMLHttpRequest.prototype;
+          const open = XHR.open;
+          const send = XHR.send;
+          XHR.open = function(method, url) {
+            this._url = url;
+            return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
+          };
+          XHR.send = function() {
+            this.addEventListener('load', function() {
+              if(this._url.includes('SearchTimeline')) {
+                try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
+              }
+            });
+            return send.apply(this, arguments);
+          };
+          window.__patched_xhr = true;
+        }
+      }
+    `);
+
+    // 3. Trigger API by scrolling
+    for (let i = 0; i < 3; i++) {
+        await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
+        await page.wait(2);
+    }
+    
+    // 4. Retrieve data
+    const requests = await page.evaluate('() => window.__opencli_xhr');
+    if (!requests || requests.length === 0) return [];
+
+    let results: any[] = [];
+    for (const req of requests) {
+      try {
+        const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
+        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
+        if (!addEntries) continue;
+
+        for (const entry of addEntries.entries) {
+          if (!entry.entryId.startsWith('tweet-')) continue;
+          
+          let tweet = entry.content?.itemContent?.tweet_results?.result;
+          if (!tweet) continue;
+
+          // Handle retweet wrapping
+          if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
+              tweet = tweet.tweet;
+          }
+
+          results.push({
+            id: tweet.rest_id,
+            author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
+            text: tweet.legacy?.full_text || '',
+            likes: tweet.legacy?.favorite_count || 0,
+            views: tweet.views?.count || '0',
+            url: `https://x.com/i/status/${tweet.rest_id}`
+          });
+        }
+      } catch (e) {
+        // ignore parsing errors for individual payloads
+      }
+    }
+
+    return results.slice(0, kwargs.limit);
+  }
+});

src/clis/twitter/timeline.ts

@@ -0,0 +1,50 @@
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'twitter',
+  name: 'timeline',
+  description: 'Twitter Home Timeline',
+  domain: 'x.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 20 },
+  ],
+  columns: ['responseType', 'first'],
+  func: async (page, kwargs) => {
+    await page.goto('https://x.com/home');
+    await page.wait(5);
+    // Inject the fetch interceptor manually to see exactly what happens
+    await page.evaluate(`
+      () => {
+        window.__intercept_data = [];
+        const origFetch = window.fetch;
+        window.fetch = async function(...args) {
+          let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
+          const res = await origFetch.apply(this, args);
+          setTimeout(async () => {
+            try {
+              if (u.includes('HomeTimeline')) {
+                const clone = res.clone();
+                const j = await clone.json();
+                window.__intercept_data.push(j);
+              }
+            } catch(e) {}
+          }, 0);
+          return res;
+        };
+      }
+    `);
+    
+    // trigger scroll
+    for(let i=0; i<3; i++) {
+      await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
+      await page.wait(2);
+    }
+    
+    // extract
+    const data = await page.evaluate('() => window.__intercept_data');
+    if (!data || data.length === 0) return [{responseType: 'no data captured'}];
+    
+    return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}];
+  }
+});

src/clis/xiaohongshu/user.ts

@@ -0,0 +1,86 @@
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'xiaohongshu',
+  name: 'user',
+  description: 'Get user notes from Xiaohongshu',
+  domain: 'xiaohongshu.com',
+  strategy: Strategy.INTERCEPT,
+  browser: true,
+  args: [
+    { name: 'id', type: 'string', required: true },
+    { name: 'limit', type: 'int', default: 15 },
+  ],
+  columns: ['id', 'title', 'type', 'likes', 'url'],
+  func: async (page, kwargs) => {
+    await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
+    await page.wait(5);
+
+    await page.evaluate(`
+      () => {
+        window.__opencli_xhr = [];
+        if (!window.__patched_xhr) {
+          const origFetch = window.fetch;
+          window.fetch = async function(...args) {
+            let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
+            const res = await origFetch.apply(this, args);
+            setTimeout(async () => {
+              try {
+                if (u.includes('v1/user/posted')) {
+                  const clone = res.clone();
+                  const j = await clone.json();
+                  window.__opencli_xhr.push({url: u, data: j});
+                }
+              } catch(e) {}
+            }, 0);
+            return res;
+          };
+          
+          const XHR = XMLHttpRequest.prototype;
+          const open = XHR.open;
+          const send = XHR.send;
+          XHR.open = function(method, url) {
+            this._url = url;
+            return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
+          };
+          XHR.send = function() {
+            this.addEventListener('load', function() {
+              if(this._url.includes('v1/user/posted')) {
+                try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
+              }
+            });
+            return send.apply(this, arguments);
+          };
+          window.__patched_xhr = true;
+        }
+      }
+    `);
+
+    // Trigger API by scrolling
+    for (let i = 0; i < 2; i++) {
+        await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
+        await page.wait(2);
+    }
+    
+    // Retrieve data
+    const requests = await page.evaluate('() => window.__opencli_xhr');
+    if (!requests || requests.length === 0) return [];
+
+    let results: any[] = [];
+    for (const req of requests) {
+      if (req.data && req.data.data && req.data.data.notes) {
+         for (const note of req.data.data.notes) {
+           results.push({
+             id: note.note_id || note.id,
+             title: note.display_title || '',
+             type: note.type || '',
+             likes: note.interact_info?.liked_count || '0',
+             url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}`
+           });
+         }
+      }
+    }
+
+    return results.slice(0, kwargs.limit);
+  }
+});

src/pipeline/steps/intercept.ts

@@ -14,7 +14,55 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
 
   if (!capturePattern) return data;
 
-  // Step 1: Execute the trigger action
+  // Step 1: Inject fetch/XHR interceptor BEFORE trigger
+  await page.evaluate(`
+    () => {
+      window.__opencli_intercepted = window.__opencli_intercepted || [];
+      const pattern = ${JSON.stringify(capturePattern)};
+      
+      if (!window.__opencli_fetch_patched) {
+        const origFetch = window.fetch;
+        window.fetch = async function(...args) {
+          const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
+          const response = await origFetch.apply(this, args);
+          setTimeout(async () => {
+            try {
+              if (reqUrl.includes(pattern)) {
+                const clone = response.clone();
+                const json = await clone.json();
+                window.__opencli_intercepted.push(json);
+              }
+            } catch(e) {}
+          }, 0);
+          return response;
+        };
+        window.__opencli_fetch_patched = true;
+      }
+
+      if (!window.__opencli_xhr_patched) {
+        const XHR = XMLHttpRequest.prototype;
+        const open = XHR.open;
+        const send = XHR.send;
+        XHR.open = function(method, url, ...args) {
+          this._reqUrl = url;
+          return open.call(this, method, url, ...args);
+        };
+        XHR.send = function(...args) {
+          this.addEventListener('load', function() {
+            try {
+              if (this._reqUrl && this._reqUrl.includes(pattern)) {
+                window.__opencli_intercepted.push(JSON.parse(this.responseText));
+              }
+            } catch(e) {}
+          });
+          return send.apply(this, args);
+        };
+        window.__opencli_xhr_patched = true;
+      }
+    }
+  `);
+
+  // Step 2: Execute the trigger action
   if (trigger.startsWith('navigate:')) {
     const url = render(trigger.slice('navigate:'.length), { args, data });
     await page.goto(String(url));
@@ -29,36 +77,18 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
     await page.scroll('down');
   }
 
-  // Step 2: Wait a bit for network requests to fire
+  // Step 3: Wait a bit for network requests to fire
   await page.wait(Math.min(timeout, 3));
 
-  // Step 3: Get network requests and find matching ones
-  const rawNetwork = await page.networkRequests(false);
-  const matchingResponses: any[] = [];
-
-  if (typeof rawNetwork === 'string') {
-    const lines = rawNetwork.split('\n');
-    for (const line of lines) {
-      const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
-      if (match) {
-        const [, , url, status] = match;
-        if (url.includes(capturePattern) && status === '200') {
-          try {
-            const body = await page.evaluate(`
-              async () => {
-                try {
-                  const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
-                  if (!resp.ok) return null;
-                  return await resp.json();
-                } catch { return null; }
-              }
-            `);
-            if (body) matchingResponses.push(body);
-          } catch {}
-        }
-      }
+  // Step 4: Retrieve captured data
+  const matchingResponses = await page.evaluate(`
+    () => {
+      const data = window.__opencli_intercepted || [];
+      window.__opencli_intercepted = []; // clear after reading
+      return data;
     }
-  }
+  `);
+
 
   // Step 4: Select from response if specified
   let result = matchingResponses.length === 1 ? matchingResponses[0] :