fix(twitter): rewrite timeline adapter to use direct GraphQL API
The previous implementation injected a fetch interceptor after page navigation, but by that time the HomeTimeline API call had already completed, resulting in 'no data captured' every time. Rewrote to directly call Twitter's HomeTimeline GraphQL endpoint (same pattern as profile.ts and thread.ts): - Dynamic queryId resolution with hardcoded fallback - Pagination support with cursor - Filters out promoted content - Returns structured tweet data (id, author, text, likes, retweets, replies, views, created_at, url) Fixes #36
This commit is contained in:
jackwener committed
1 parent
612c0ab1af
commit
9ae9eb3fc6
1 file changed
+204
-36
+204
-36
@@ -1,50 +1,218 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
// ── Twitter GraphQL constants ──────────────────────────────────────────
|
||||
|
||||
const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
||||
const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ';
|
||||
|
||||
const FEATURES = {
|
||||
rweb_video_screen_enabled: false,
|
||||
profile_label_improvements_pcf_label_in_post_enabled: true,
|
||||
rweb_tipjar_consumption_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
premium_content_api_read_enabled: false,
|
||||
communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
responsive_web_grok_analyze_button_fetch_trends_enabled: false,
|
||||
responsive_web_grok_analyze_post_followups_enabled: true,
|
||||
responsive_web_jetfuel_frame: false,
|
||||
responsive_web_grok_share_attachment_enabled: true,
|
||||
articles_preview_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
responsive_web_grok_show_grok_translated_post: false,
|
||||
responsive_web_grok_analysis_button_from_backend: false,
|
||||
creator_subscriptions_quote_tweet_preview_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_grok_image_annotation_enabled: true,
|
||||
responsive_web_enhance_cards_enabled: false,
|
||||
};
|
||||
|
||||
// ── Pure functions (type-safe, testable) ───────────────────────────────
|
||||
|
||||
interface TimelineTweet {
|
||||
id: string;
|
||||
author: string;
|
||||
text: string;
|
||||
likes: number;
|
||||
retweets: number;
|
||||
replies: number;
|
||||
views: number;
|
||||
created_at: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function buildHomeTimelineUrl(count: number, cursor?: string | null): string {
|
||||
const vars: Record<string, any> = {
|
||||
count,
|
||||
includePromotedContent: false,
|
||||
latestControlAvailable: true,
|
||||
requestContext: 'launch',
|
||||
withCommunity: true,
|
||||
};
|
||||
if (cursor) vars.cursor = cursor;
|
||||
|
||||
return `/i/api/graphql/${HOME_TIMELINE_QUERY_ID}/HomeTimeline`
|
||||
+ `?variables=${encodeURIComponent(JSON.stringify(vars))}`
|
||||
+ `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
|
||||
}
|
||||
|
||||
function extractTweet(result: any, seen: Set<string>): TimelineTweet | null {
|
||||
if (!result) return null;
|
||||
const tw = result.tweet || result;
|
||||
const l = tw.legacy || {};
|
||||
if (!tw.rest_id || seen.has(tw.rest_id)) return null;
|
||||
seen.add(tw.rest_id);
|
||||
|
||||
const u = tw.core?.user_results?.result;
|
||||
const screenName = u?.legacy?.screen_name || u?.core?.screen_name || 'unknown';
|
||||
const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
|
||||
const views = tw.views?.count ? parseInt(tw.views.count, 10) : 0;
|
||||
|
||||
return {
|
||||
id: tw.rest_id,
|
||||
author: screenName,
|
||||
text: noteText || l.full_text || '',
|
||||
likes: l.favorite_count || 0,
|
||||
retweets: l.retweet_count || 0,
|
||||
replies: l.reply_count || 0,
|
||||
views,
|
||||
created_at: l.created_at || '',
|
||||
url: `https://x.com/${screenName}/status/${tw.rest_id}`,
|
||||
};
|
||||
}
|
||||
|
||||
function parseHomeTimeline(data: any, seen: Set<string>): { tweets: TimelineTweet[]; nextCursor: string | null } {
|
||||
const tweets: TimelineTweet[] = [];
|
||||
let nextCursor: string | null = null;
|
||||
|
||||
const instructions =
|
||||
data?.data?.home?.home_timeline_urt?.instructions || [];
|
||||
|
||||
for (const inst of instructions) {
|
||||
for (const entry of inst.entries || []) {
|
||||
const c = entry.content;
|
||||
|
||||
// Cursor entries
|
||||
if (c?.entryType === 'TimelineTimelineCursor' || c?.__typename === 'TimelineTimelineCursor') {
|
||||
if (c.cursorType === 'Bottom') nextCursor = c.value;
|
||||
continue;
|
||||
}
|
||||
if (entry.entryId?.startsWith('cursor-bottom-')) {
|
||||
nextCursor = c?.value || nextCursor;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Single tweet entry
|
||||
const tweetResult = c?.itemContent?.tweet_results?.result;
|
||||
if (tweetResult) {
|
||||
// Skip promoted content
|
||||
if (c?.itemContent?.promotedMetadata) continue;
|
||||
const tw = extractTweet(tweetResult, seen);
|
||||
if (tw) tweets.push(tw);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Conversation module (grouped tweets)
|
||||
for (const item of c?.items || []) {
|
||||
const nested = item.item?.itemContent?.tweet_results?.result;
|
||||
if (nested) {
|
||||
if (item.item?.itemContent?.promotedMetadata) continue;
|
||||
const tw = extractTweet(nested, seen);
|
||||
if (tw) tweets.push(tw);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { tweets, nextCursor };
|
||||
}
|
||||
|
||||
// ── CLI definition ────────────────────────────────────────────────────
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'timeline',
|
||||
description: 'Twitter Home Timeline',
|
||||
description: 'Fetch Twitter Home Timeline',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20 },
|
||||
],
|
||||
columns: ['responseType', 'first'],
|
||||
columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://x.com/home');
|
||||
await page.wait(5);
|
||||
// Inject the fetch interceptor manually to see exactly what happens
|
||||
await page.evaluate(`
|
||||
() => {
|
||||
window.__intercept_data = [];
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
||||
const res = await origFetch.apply(this, args);
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (u.includes('HomeTimeline')) {
|
||||
const clone = res.clone();
|
||||
const j = await clone.json();
|
||||
window.__intercept_data.push(j);
|
||||
}
|
||||
} catch(e) {}
|
||||
}, 0);
|
||||
return res;
|
||||
};
|
||||
const limit = kwargs.limit || 20;
|
||||
|
||||
// Navigate to x.com for cookie context
|
||||
await page.goto('https://x.com');
|
||||
await page.wait(3);
|
||||
|
||||
// Extract CSRF token
|
||||
const ct0 = await page.evaluate(`() => {
|
||||
return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
|
||||
}`);
|
||||
if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
|
||||
|
||||
// Dynamically resolve queryId
|
||||
const queryId = await page.evaluate(`async () => {
|
||||
try {
|
||||
const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
|
||||
if (ghResp.ok) {
|
||||
const data = await ghResp.json();
|
||||
const entry = data['HomeTimeline'];
|
||||
if (entry && entry.queryId) return entry.queryId;
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}`) || HOME_TIMELINE_QUERY_ID;
|
||||
|
||||
// Build auth headers
|
||||
const headers = JSON.stringify({
|
||||
'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
|
||||
'X-Csrf-Token': ct0,
|
||||
'X-Twitter-Auth-Type': 'OAuth2Session',
|
||||
'X-Twitter-Active-User': 'yes',
|
||||
});
|
||||
|
||||
// Paginate — fetch in browser, parse in TypeScript
|
||||
const allTweets: TimelineTweet[] = [];
|
||||
const seen = new Set<string>();
|
||||
let cursor: string | null = null;
|
||||
|
||||
for (let i = 0; i < 5 && allTweets.length < limit; i++) {
|
||||
const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering
|
||||
const apiUrl = buildHomeTimelineUrl(fetchCount, cursor)
|
||||
.replace(HOME_TIMELINE_QUERY_ID, queryId);
|
||||
|
||||
const data = await page.evaluate(`async () => {
|
||||
const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' });
|
||||
return r.ok ? await r.json() : { error: r.status };
|
||||
}`);
|
||||
|
||||
if (data?.error) {
|
||||
if (allTweets.length === 0) throw new Error(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`);
|
||||
break;
|
||||
}
|
||||
`);
|
||||
|
||||
// trigger scroll
|
||||
for(let i=0; i<3; i++) {
|
||||
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
||||
await page.wait(2);
|
||||
|
||||
const { tweets, nextCursor } = parseHomeTimeline(data, seen);
|
||||
allTweets.push(...tweets);
|
||||
|
||||
if (!nextCursor || nextCursor === cursor) break;
|
||||
cursor = nextCursor;
|
||||
}
|
||||
|
||||
// extract
|
||||
const data = await page.evaluate('() => window.__intercept_data');
|
||||
if (!data || data.length === 0) return [{responseType: 'no data captured'}];
|
||||
|
||||
return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}];
|
||||
}
|
||||
|
||||
return allTweets.slice(0, limit);
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user