fix: install XHR interceptor after navigation to prevent context reset (#91)
goto() triggers a full page navigation that resets the JS execution context, wiping any previously injected fetch/XHR monkey-patches. The old code installed the interceptor on x.com then navigated away, so the interceptor was always destroyed before it could capture data. Fix: navigate directly to the target page, install interceptor after page load, then scroll to trigger API calls via pagination. Also fixes the same bug in notifications.ts. Closes #86
This commit is contained in:
1 parent
4e260ecdeb
commit
aeb1cb6a3a
2 files changed
+18
-18
No files matched your search
@@ -12,17 +12,17 @@ cli({
|
||||
],
|
||||
columns: ['id', 'action', 'author', 'text', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// Install the interceptor before loading the notifications page so we
|
||||
// capture the initial timeline request triggered during page load.
|
||||
await page.goto('https://x.com');
|
||||
await page.wait(2);
|
||||
// 1. Navigate directly to notifications
|
||||
await page.goto('https://x.com/notifications');
|
||||
await page.wait(3);
|
||||
|
||||
// 2. Install interceptor after page load (must be after goto, not before,
|
||||
// because goto triggers a full navigation that resets the JS context).
|
||||
// Note: this misses the initial request fired during hydration;
|
||||
// we rely on scroll-triggered pagination to capture data.
|
||||
await page.installInterceptor('NotificationsTimeline');
|
||||
|
||||
// 1. Navigate to notifications
|
||||
await page.goto('https://x.com/notifications');
|
||||
await page.wait(5);
|
||||
|
||||
// 3. Trigger API by scrolling (if we need to load more)
|
||||
// 3. Scroll to trigger API calls (load more notifications)
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data
|
||||
|
||||
@@ -13,18 +13,18 @@ cli({
|
||||
],
|
||||
columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// Install the interceptor before opening the target page so we don't miss
|
||||
// the initial SearchTimeline request fired during hydration.
|
||||
await page.goto('https://x.com');
|
||||
await page.wait(2);
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// 1. Navigate to the search page
|
||||
// 1. Navigate directly to the search page
|
||||
const q = encodeURIComponent(kwargs.query);
|
||||
await page.goto(`https://x.com/search?q=${q}&f=top`);
|
||||
await page.wait(5);
|
||||
await page.wait(3);
|
||||
|
||||
// 3. Trigger API by scrolling
|
||||
// 2. Install interceptor after page load (must be after goto, not before,
|
||||
// because goto triggers a full navigation that resets the JS context).
|
||||
// Note: this misses the initial SearchTimeline request fired during
|
||||
// hydration; we rely on scroll-triggered pagination to capture data.
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// 3. Scroll to trigger SearchTimeline API calls (pagination)
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data
|
||||
|
||||
Reference in new issue
Block a user