okooooooo/opencli
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(intercept): IIFE wrapping for installInterceptor/getInterceptedRequests (#100)

Browse Source 
* fix(intercept): use evaluate() for IIFE wrapping in installInterceptor/getInterceptedRequests

Root cause: daemon migration changed these methods from this.evaluate()
to direct sendCommand('exec'), losing the wrapForEval() IIFE wrapping.
CDP received bare arrow functions that were never invoked.

Fixes #98

* fix(twitter): SPA navigation, data path, and author resolution for INTERCEPT commands

- followers/following: install interceptor on profile page, then click
  followers/following link (SPA navigation preserves JS context).
  Use JSON.stringify for targetUser to prevent injection. Throw on
  navigation failure. Update selector: /verified_followers.
- notifications: install interceptor on home, then pushState+popstate
  to /notifications. Validate navigation URL.
- search: fix author resolution (core.screen_name, not legacy).
- All: fix GraphQL data path (remove extra .data level), update author
  resolution to try core.screen_name before legacy.screen_name.
- followers: remove erroneous .filter(r => r?.url) — interceptor stores
  response body JSON, URL filtering happens at capture time.
This commit is contained in:
AstroHan authored and GitHub committed 2026-03-20 00:13:20 +08:00
1 parent 540f3c677a
commit 65e30b9ac4
5 files changed
+109 -91
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/browser/page.ts
+8 -11
View File
@@ -269,21 +269,18 @@ export class Page implements IPage {
async installInterceptor(pattern: string): Promise<void> {
const { generateInterceptorJs } = await import('../interceptor.js');
await sendCommand('exec', {
code: generateInterceptorJs(JSON.stringify(pattern), {
arrayName: '__opencli_xhr',
patchGuard: '__opencli_interceptor_patched',
}),
...this._tabOpt(),
});
// Must use evaluate() so wrapForEval() converts the arrow function into an IIFE;
// sendCommand('exec') sends the code as-is, and CDP never executes a bare arrow.
await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
arrayName: '__opencli_xhr',
patchGuard: '__opencli_interceptor_patched',
}));
}
async getInterceptedRequests(): Promise<any[]> {
const { generateReadInterceptedJs } = await import('../interceptor.js');
const result = await sendCommand('exec', {
code: generateReadInterceptedJs('__opencli_xhr'),
...this._tabOpt(),
});
// Same as installInterceptor: must go through evaluate() for IIFE wrapping
const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
return (result as any[]) || [];
}
}
src/clis/twitter/followers.ts
+37 -30
View File
@@ -15,65 +15,73 @@ cli({
func: async (page, kwargs) => {
let targetUser = kwargs.user;
// If no user is specified, we must figure out the logged-in user's handle
// If no user is specified, figure out the logged-in user's handle
if (!targetUser) {
await page.goto('https://x.com/home');
// wait for home page navigation
await page.wait(5);
const href = await page.evaluate(`() => {
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
return link ? link.getAttribute('href') : null;
}`);
if (!href) {
throw new Error('Could not find logged-in user profile link. Are you logged in?');
}
targetUser = href.replace('/', '');
}
// 1. Navigate to user profile page
// 1. Navigate to profile page
await page.goto(`https://x.com/${targetUser}`);
await page.wait(3);
// 2. Inject interceptor for the followers GraphQL API
// 2. Install interceptor BEFORE SPA navigation.
// goto() resets JS context, but SPA click preserves it.
await page.installInterceptor('Followers');
// 3. Click the followers link inside the profile page
await page.evaluate(`() => {
const target = '${targetUser}';
const link = document.querySelector('a[href="/' + target + '/followers"]');
if (link) link.click();
}`);
await page.wait(3);
// 4. Trigger API by scrolling
// 3. Click the followers link via SPA navigation (preserves interceptor).
// Twitter uses /verified_followers instead of /followers now.
const safeUser = JSON.stringify(targetUser);
const clicked = await page.evaluate(`() => {
const target = ${safeUser};
const selectors = [
'a[href="/' + target + '/verified_followers"]',
'a[href="/' + target + '/followers"]',
];
for (const sel of selectors) {
const link = document.querySelector(sel);
if (link) { link.click(); return true; }
}
return false;
}`);
if (!clicked) {
throw new Error('Could not find followers link on profile page. Twitter may have changed the layout.');
}
await page.wait(5);
// 4. Scroll to trigger pagination API calls
await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });
// 4. Retrieve data from opencli's registered interceptors
const allRequests = await page.getInterceptedRequests();
const requestList = Array.isArray(allRequests) ? allRequests : [];
// 5. Retrieve intercepted data
const requests = await page.getInterceptedRequests();
const requestList = Array.isArray(requests) ? requests : [];
if (requestList.length === 0) {
return [];
}
const requests = requestList.filter((r: any) => r?.url?.includes('Followers'));
if (!requests || requests.length === 0) {
return [];
}
let results: any[] = [];
for (const req of requests) {
for (const req of requestList) {
try {
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
// GraphQL response: { data: { user: { result: { timeline: ... } } } }
let instructions = req.data?.user?.result?.timeline?.timeline?.instructions;
if (!instructions) continue;
let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
if (!addEntries) {
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
}
if (!addEntries) continue;
for (const entry of addEntries.entries) {
@@ -82,10 +90,9 @@ cli({
const item = entry.content?.itemContent?.user_results?.result;
if (!item || item.__typename !== 'User') continue;
// Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state
const core = item.core || {};
const legacy = item.legacy || {};
results.push({
screen_name: core.screen_name || legacy.screen_name || 'unknown',
name: core.name || legacy.name || 'unknown',
@@ -98,7 +105,7 @@ cli({
}
}
// Deduplicate by screen_name in case multiple scrolls caught the same
// Deduplicate by screen_name
const unique = new Map();
results.forEach(r => unique.set(r.screen_name, r));
const deduplicatedResults = Array.from(unique.values());
src/clis/twitter/following.ts
+27 -22
View File
@@ -15,45 +15,50 @@ cli({
func: async (page, kwargs) => {
let targetUser = kwargs.user;
// If no user is specified, we must figure out the logged-in user's handle
// If no user is specified, figure out the logged-in user's handle
if (!targetUser) {
await page.goto('https://x.com/home');
// wait for home page navigation
await page.wait(5);
const href = await page.evaluate(`() => {
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
return link ? link.getAttribute('href') : null;
}`);
if (!href) {
throw new Error('Could not find logged-in user profile link. Are you logged in?');
}
targetUser = href.replace('/', '');
}
// 1. Navigate to user profile page
// 1. Navigate to profile page
await page.goto(`https://x.com/${targetUser}`);
await page.wait(3);
// 2. Inject interceptor for Following GraphQL API
// 2. Install interceptor BEFORE SPA navigation.
// goto() resets JS context, but SPA click preserves it.
await page.installInterceptor('Following');
// 3. Click the following link inside the profile page
await page.evaluate(`() => {
const target = '${targetUser}';
const link = document.querySelector('a[href="/' + target + '/following"]');
if (link) link.click();
}`);
await page.wait(3);
// 4. Trigger API by scrolling
// 3. Click the following link via SPA navigation (preserves interceptor)
const safeUser = JSON.stringify(targetUser);
const clicked = await page.evaluate(`() => {
const target = ${safeUser};
const link = document.querySelector('a[href="/' + target + '/following"]');
if (link) { link.click(); return true; }
return false;
}`);
if (!clicked) {
throw new Error('Could not find following link on profile page. Twitter may have changed the layout.');
}
await page.wait(5);
// 4. Scroll to trigger pagination API calls
await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });
// 4. Retrieve data from opencli's registered interceptors
// 5. Retrieve intercepted data
const requests = await page.getInterceptedRequests();
const requestList = Array.isArray(requests) ? requests : [];
if (requestList.length === 0) {
return [];
}
@@ -61,14 +66,15 @@ cli({
let results: any[] = [];
for (const req of requestList) {
try {
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
// GraphQL response: { data: { user: { result: { timeline: ... } } } }
let instructions = req.data?.user?.result?.timeline?.timeline?.instructions;
if (!instructions) continue;
let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
if (!addEntries) {
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
}
if (!addEntries) continue;
for (const entry of addEntries.entries) {
@@ -77,10 +83,9 @@ cli({
const item = entry.content?.itemContent?.user_results?.result;
if (!item || item.__typename !== 'User') continue;
// Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state
const core = item.core || {};
const legacy = item.legacy || {};
results.push({
screen_name: core.screen_name || legacy.screen_name || 'unknown',
name: core.name || legacy.name || 'unknown',
@@ -93,7 +98,7 @@ cli({
}
}
// Deduplicate by screen_name in case multiple scrolls caught the same
// Deduplicate by screen_name
const unique = new Map();
results.forEach(r => unique.set(r.screen_name, r));
const deduplicatedResults = Array.from(unique.values());
src/clis/twitter/notifications.ts
+34 -27
View File
@@ -12,20 +12,30 @@ cli({
],
columns: ['id', 'action', 'author', 'text', 'url'],
func: async (page, kwargs) => {
// 1. Navigate directly to notifications
await page.goto('https://x.com/notifications');
// 1. Navigate to home first (we need a loaded Twitter page for SPA navigation)
await page.goto('https://x.com/home');
await page.wait(3);
// 2. Install interceptor after page load (must be after goto, not before,
// because goto triggers a full navigation that resets the JS context).
// Note: this misses the initial request fired during hydration;
// we rely on scroll-triggered pagination to capture data.
// 2. Install interceptor BEFORE SPA navigation
await page.installInterceptor('NotificationsTimeline');
// 3. Scroll to trigger API calls (load more notifications)
// 3. SPA navigate to notifications via history API
await page.evaluate(`() => {
window.history.pushState({}, '', '/notifications');
window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
}`);
await page.wait(5);
// Verify SPA navigation succeeded
const currentUrl = await page.evaluate('() => window.location.pathname');
if (currentUrl !== '/notifications') {
throw new Error('SPA navigation to notifications failed. Twitter may have changed its routing.');
}
// 4. Scroll to trigger pagination
await page.autoScroll({ times: 2, delayMs: 2000 });
// 4. Retrieve data
// 5. Retrieve data
const requests = await page.getInterceptedRequests();
if (!requests || requests.length === 0) return [];
@@ -33,22 +43,20 @@ cli({
const seen = new Set<string>();
for (const req of requests) {
try {
// GraphQL response: { data: { viewer: ... } } (one level of .data)
let instructions: any[] = [];
if (req.data?.data?.viewer?.timeline_response?.timeline?.instructions) {
instructions = req.data.data.viewer.timeline_response.timeline.instructions;
} else if (req.data?.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) {
instructions = req.data.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions;
} else if (req.data?.data?.timeline?.instructions) {
instructions = req.data.data.timeline.instructions;
if (req.data?.viewer?.timeline_response?.timeline?.instructions) {
instructions = req.data.viewer.timeline_response.timeline.instructions;
} else if (req.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) {
instructions = req.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions;
} else if (req.data?.timeline?.instructions) {
instructions = req.data.timeline.instructions;
}
let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
// Sometimes it's the first object without a 'type' field but has 'entries'
if (!addEntries) {
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
}
if (!addEntries) continue;
for (const entry of addEntries.entries) {
@@ -66,24 +74,22 @@ cli({
function processNotificationItem(itemContent: any, entryId: string) {
if (!itemContent) return;
// Twitter wraps standard notifications
let item = itemContent?.notification_results?.result || itemContent?.tweet_results?.result || itemContent;
let actionText = 'Notification';
let author = 'unknown';
let text = '';
let urlStr = '';
if (item.__typename === 'TimelineNotification') {
// Greet likes, retweet, mentions
text = item.rich_message?.text || item.message?.text || '';
const fromUser = item.template?.from_users?.[0]?.user_results?.result;
author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown';
// Twitter moved screen_name from legacy to core
author = fromUser?.core?.screen_name || fromUser?.legacy?.screen_name || 'unknown';
urlStr = item.notification_url?.url || '';
actionText = item.notification_icon || 'Activity';
// If there's an attached tweet
const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
if (targetTweet) {
const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || '';
@@ -93,14 +99,15 @@ cli({
}
}
} else if (item.__typename === 'TweetNotification') {
// Direct mention/reply
const tweet = item.tweet_result?.result;
author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
const tweetUser = tweet?.core?.user_results?.result;
author = tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown';
text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || '';
actionText = 'Mention/Reply';
urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
} else if (item.__typename === 'Tweet') {
author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
const tweetUser = item.core?.user_results?.result;
author = tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown';
text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || '';
actionText = 'Mention';
urlStr = `https://x.com/i/status/${item.rest_id}`;
src/clis/twitter/search.ts
+3 -1
View File
@@ -91,9 +91,11 @@ cli({
if (!tweet.rest_id || seen.has(tweet.rest_id)) continue;
seen.add(tweet.rest_id);
// Twitter moved screen_name from legacy to core
const tweetUser = tweet.core?.user_results?.result;
results.push({
id: tweet.rest_id,
author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown',
text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
likes: tweet.legacy?.favorite_count || 0,
views: tweet.views?.count || '0',