fix(twitter): harden remaining twitter commands
Co-authored-by: Sheng-Yan, Zhang <yancode@qq.com>
This commit is contained in:
1 parent
546c0b997a
commit
7b5bdfa7d5
7 files changed
+48
-41
No files matched your search
@@ -15,7 +15,6 @@ cli({
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
console.log(`Navigating to tweet: ${kwargs.url}`);
|
||||
await page.goto(kwargs.url);
|
||||
await page.wait(5); // Wait for tweet to load completely
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import * as fs from 'fs';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
@@ -37,8 +36,8 @@ cli({
|
||||
await page.goto(`https://x.com/${targetUser}`);
|
||||
await page.wait(3);
|
||||
|
||||
// 2. Inject interceptor for Followers GraphQL API (or user_flow.json)
|
||||
await page.installInterceptor('graphql');
|
||||
// 2. Inject interceptor for the followers GraphQL API
|
||||
await page.installInterceptor('Followers');
|
||||
|
||||
// 3. Click the followers link inside the profile page
|
||||
await page.evaluate(`() => {
|
||||
@@ -53,24 +52,14 @@ cli({
|
||||
|
||||
// 4. Retrieve data from opencli's registered interceptors
|
||||
const allRequests = await page.getInterceptedRequests();
|
||||
const requestList = Array.isArray(allRequests) ? allRequests : [];
|
||||
|
||||
// Debug: Force dump all intercepted XHRs that match followers
|
||||
if (!allRequests || allRequests.length === 0) {
|
||||
console.log('No GraphQL requests captured by the interceptor backend.');
|
||||
if (requestList.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
console.log('Intercepted keys:', allRequests.map((r: any) => {
|
||||
try {
|
||||
const u = new URL(r.url); return u.pathname;
|
||||
} catch (e) {
|
||||
return r.url;
|
||||
}
|
||||
}));
|
||||
|
||||
const requests = allRequests.filter((r: any) => r.url.includes('Followers'));
|
||||
const requests = requestList.filter((r: any) => r?.url?.includes('Followers'));
|
||||
if (!requests || requests.length === 0) {
|
||||
console.log('No specific Followers requests captured. Check keys printed above.');
|
||||
return [];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import * as fs from 'fs';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
@@ -53,15 +52,14 @@ cli({
|
||||
|
||||
// 4. Retrieve data from opencli's registered interceptors
|
||||
const requests = await page.getInterceptedRequests();
|
||||
const requestList = Array.isArray(requests) ? requests : [];
|
||||
|
||||
// Debug: Force dump all intercepted XHRs that match following
|
||||
if (!requests || requests.length === 0) {
|
||||
console.log('No Following requests captured by the interceptor backend.');
|
||||
if (requestList.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
for (const req of requestList) {
|
||||
try {
|
||||
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
|
||||
if (!instructions) continue;
|
||||
|
||||
@@ -15,7 +15,6 @@ cli({
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
console.log(`Navigating to tweet: ${kwargs.url}`);
|
||||
await page.goto(kwargs.url);
|
||||
await page.wait(5); // Wait for tweet to load completely
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import * as fs from 'fs';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
@@ -13,13 +12,16 @@ cli({
|
||||
],
|
||||
columns: ['id', 'action', 'author', 'text', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// Install the interceptor before loading the notifications page so we
|
||||
// capture the initial timeline request triggered during page load.
|
||||
await page.goto('https://x.com');
|
||||
await page.wait(2);
|
||||
await page.installInterceptor('NotificationsTimeline');
|
||||
|
||||
// 1. Navigate to notifications
|
||||
await page.goto('https://x.com/notifications');
|
||||
await page.wait(5);
|
||||
|
||||
// 2. Inject interceptor
|
||||
await page.installInterceptor('NotificationsTimeline');
|
||||
|
||||
// 3. Trigger API by scrolling (if we need to load more)
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
@@ -28,6 +30,7 @@ cli({
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const req of requests) {
|
||||
try {
|
||||
let instructions: any[] = [];
|
||||
@@ -75,14 +78,16 @@ cli({
|
||||
if (item.__typename === 'TimelineNotification') {
|
||||
// Greet likes, retweet, mentions
|
||||
text = item.rich_message?.text || item.message?.text || '';
|
||||
author = item.template?.from_users?.[0]?.user_results?.result?.core?.screen_name || 'unknown';
|
||||
const fromUser = item.template?.from_users?.[0]?.user_results?.result;
|
||||
author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown';
|
||||
urlStr = item.notification_url?.url || '';
|
||||
actionText = item.notification_icon || 'Activity';
|
||||
|
||||
// If there's an attached tweet
|
||||
const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
|
||||
if (targetTweet) {
|
||||
text += ' | ' + (targetTweet.legacy?.full_text || '');
|
||||
const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || '';
|
||||
text += text && targetText ? ' | ' + targetText : targetText;
|
||||
if (!urlStr) {
|
||||
urlStr = `https://x.com/i/status/${targetTweet.rest_id}`;
|
||||
}
|
||||
@@ -91,18 +96,22 @@ cli({
|
||||
// Direct mention/reply
|
||||
const tweet = item.tweet_result?.result;
|
||||
author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
|
||||
text = tweet?.legacy?.full_text || item.message?.text || '';
|
||||
text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || '';
|
||||
actionText = 'Mention/Reply';
|
||||
urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
|
||||
} else if (item.__typename === 'Tweet') {
|
||||
author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
|
||||
text = item.legacy?.full_text || '';
|
||||
text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || '';
|
||||
actionText = 'Mention';
|
||||
urlStr = `https://x.com/i/status/${item.rest_id}`;
|
||||
}
|
||||
|
||||
const id = item.id || item.rest_id || entryId;
|
||||
if (seen.has(id)) return;
|
||||
seen.add(id);
|
||||
|
||||
results.push({
|
||||
id: item.id || item.rest_id || entryId,
|
||||
id,
|
||||
action: actionText,
|
||||
author: author,
|
||||
text: text,
|
||||
|
||||
@@ -13,14 +13,17 @@ cli({
|
||||
],
|
||||
columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// Install the interceptor before opening the target page so we don't miss
|
||||
// the initial SearchTimeline request fired during hydration.
|
||||
await page.goto('https://x.com');
|
||||
await page.wait(2);
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// 1. Navigate to the search page
|
||||
const q = encodeURIComponent(kwargs.query);
|
||||
await page.goto(`https://x.com/search?q=${q}&f=top`);
|
||||
await page.wait(5);
|
||||
|
||||
// 2. Inject XHR interceptor
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// 3. Trigger API by scrolling
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
@@ -29,11 +32,13 @@ cli({
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const req of requests) {
|
||||
try {
|
||||
const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
|
||||
const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
|
||||
if (!addEntries) continue;
|
||||
const insts = req.data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
|
||||
const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries')
|
||||
|| insts.find((i: any) => i.entries && Array.isArray(i.entries));
|
||||
if (!addEntries?.entries) continue;
|
||||
|
||||
for (const entry of addEntries.entries) {
|
||||
if (!entry.entryId.startsWith('tweet-')) continue;
|
||||
@@ -45,11 +50,13 @@ cli({
|
||||
if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
|
||||
tweet = tweet.tweet;
|
||||
}
|
||||
if (!tweet.rest_id || seen.has(tweet.rest_id)) continue;
|
||||
seen.add(tweet.rest_id);
|
||||
|
||||
results.push({
|
||||
id: tweet.rest_id,
|
||||
author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
|
||||
text: tweet.legacy?.full_text || '',
|
||||
text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
|
||||
likes: tweet.legacy?.favorite_count || 0,
|
||||
views: tweet.views?.count || '0',
|
||||
url: `https://x.com/i/status/${tweet.rest_id}`
|
||||
|
||||
@@ -25,9 +25,15 @@ pipeline:
|
||||
credentials: 'include',
|
||||
headers: { 'x-twitter-active-user': 'yes', 'x-csrf-token': csrfToken, 'authorization': 'Bearer ' + bearerToken }
|
||||
});
|
||||
if (!res.ok) throw new Error('HTTP ' + res.status + '. Hint: trending endpoint may require login or API shape changed.');
|
||||
const data = await res.json();
|
||||
const trends = data?.timeline?.instructions?.[1]?.addEntries?.entries || [];
|
||||
return trends.filter(e => e.content?.timelineModule).flatMap(e => e.content.timelineModule.items || []).map(t => t?.item?.content?.trend).filter(Boolean);
|
||||
const instructions = data?.timeline?.instructions || [];
|
||||
const entries = instructions.flatMap(inst => inst?.addEntries?.entries || inst?.entries || []);
|
||||
return entries
|
||||
.filter(e => e.content?.timelineModule)
|
||||
.flatMap(e => e.content.timelineModule.items || [])
|
||||
.map(t => t?.item?.content?.trend)
|
||||
.filter(Boolean);
|
||||
})()
|
||||
|
||||
- map:
|
||||
|
||||
Reference in new issue
Block a user