refactor: remove raw CDP code, use IPage throughout
- Remove fetchCreatorNotesByCdp() and captureNoteDetailApiPayload() raw WebSocket code (~240 lines) — adapters should use IPage, not raw CDP - Replace direct CDP WebSocket with IPage.evaluate() in-page fetch - Fix page: any → IPage in all function signatures - Simplify to two-tier fallback: API+interceptor → DOM parse - Rebase onto latest main (resolves cdp.ts/daemon.ts conflicts)
This commit is contained in:
jackwener committed
1 parent
8f1725982e
commit
4f74b45963
2 files changed
+34
-243
No files matched your search
@@ -10,6 +10,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { cli, Strategy } from '../../registry.js';
|
import { cli, Strategy } from '../../registry.js';
|
||||||
|
import type { IPage } from '../../types.js';
|
||||||
|
|
||||||
type CreatorNoteDetailRow = {
|
type CreatorNoteDetailRow = {
|
||||||
section: string;
|
section: string;
|
||||||
@@ -285,124 +286,50 @@ export function appendTrendRows(rows: CreatorNoteDetailRow[], payload?: NoteDeta
|
|||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
function deriveCdpHttpBase(endpoint?: string): string | null {
|
const DETAIL_API_ENDPOINTS: Array<{ suffix: string; key: keyof NoteDetailApiPayload }> = [
|
||||||
if (!endpoint) return null;
|
{ suffix: '/api/galaxy/creator/datacenter/note/base', key: 'noteBase' },
|
||||||
try {
|
{ suffix: '/api/galaxy/creator/datacenter/note/analyze/audience/trend', key: 'audienceTrend' },
|
||||||
const url = new URL(endpoint);
|
{ suffix: '/api/galaxy/creator/datacenter/note/audience/source/detail', key: 'audienceSourceDetail' },
|
||||||
if (url.protocol === 'ws:' || url.protocol === 'wss:') {
|
{ suffix: '/api/galaxy/creator/datacenter/note/audience', key: 'audienceSource' },
|
||||||
url.protocol = url.protocol === 'wss:' ? 'https:' : 'http:';
|
];
|
||||||
url.pathname = '';
|
|
||||||
url.search = '';
|
|
||||||
url.hash = '';
|
|
||||||
return url.toString().replace(/\/$/, '');
|
|
||||||
}
|
|
||||||
return `${url.protocol}//${url.host}`;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveRequestedCdpEndpoint(): { endpoint: string | null; requestedCdp: boolean } {
|
async function captureNoteDetailPayload(page: IPage, noteId: string): Promise<NoteDetailApiPayload | null> {
|
||||||
const endpoint = process.env.OPENCLI_CDP_ENDPOINT?.trim();
|
|
||||||
return {
|
|
||||||
endpoint: endpoint || null,
|
|
||||||
requestedCdp: Boolean(endpoint),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function captureNoteDetailApiPayload(noteId: string): Promise<NoteDetailApiPayload | null> {
|
|
||||||
const { endpoint, requestedCdp } = resolveRequestedCdpEndpoint();
|
|
||||||
if (!requestedCdp) return null;
|
|
||||||
|
|
||||||
const httpBase = deriveCdpHttpBase(endpoint ?? 'http://127.0.0.1:9222') ?? 'http://127.0.0.1:9222';
|
|
||||||
const targets = await fetch(`${httpBase}/json/list`).then((resp) => resp.json()) as Array<{
|
|
||||||
url?: string;
|
|
||||||
webSocketDebuggerUrl?: string;
|
|
||||||
}>;
|
|
||||||
const target = targets.find((entry) => entry.url?.includes(`/statistics/note-detail?noteId=${noteId}`) && entry.webSocketDebuggerUrl);
|
|
||||||
if (!target?.webSocketDebuggerUrl) return null;
|
|
||||||
|
|
||||||
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
|
||||||
const payload: NoteDetailApiPayload = {};
|
const payload: NoteDetailApiPayload = {};
|
||||||
const pending = new Map<number, { resolve: (value: any) => void; reject: (err: Error) => void }>();
|
let captured = 0;
|
||||||
const desiredUrls = new Map<string, keyof NoteDetailApiPayload>([
|
|
||||||
[`/api/galaxy/creator/datacenter/note/base?note_id=${noteId}`, 'noteBase'],
|
|
||||||
[`/api/galaxy/creator/datacenter/note/analyze/audience/trend?note_id=${noteId}`, 'audienceTrend'],
|
|
||||||
[`/api/galaxy/creator/datacenter/note/audience/source?note_id=${noteId}`, 'audienceSource'],
|
|
||||||
[`/api/galaxy/creator/datacenter/note/audience/source/detail?note_id=${noteId}`, 'audienceSourceDetail'],
|
|
||||||
]);
|
|
||||||
let nextId = 1;
|
|
||||||
|
|
||||||
const send = (method: string, params: Record<string, unknown> = {}) => {
|
// Try to fetch each API endpoint through the page context (uses the browser's cookies)
|
||||||
const id = nextId++;
|
for (const { suffix, key } of DETAIL_API_ENDPOINTS) {
|
||||||
ws.send(JSON.stringify({ id, method, params }));
|
const apiUrl = `${suffix}?note_id=${noteId}`;
|
||||||
return new Promise<any>((resolve, reject) => {
|
try {
|
||||||
pending.set(id, { resolve, reject });
|
const data = await page.evaluate(`
|
||||||
});
|
async () => {
|
||||||
};
|
|
||||||
|
|
||||||
const completed = new Set<keyof NoteDetailApiPayload>();
|
|
||||||
const waitForPayload = new Promise<void>((resolve) => {
|
|
||||||
const timeout = setTimeout(() => resolve(), 8000);
|
|
||||||
ws.addEventListener('message', async (event) => {
|
|
||||||
const msg = JSON.parse(String(event.data));
|
|
||||||
if (typeof msg.id === 'number' && pending.has(msg.id)) {
|
|
||||||
const waiter = pending.get(msg.id)!;
|
|
||||||
pending.delete(msg.id);
|
|
||||||
if (msg.error) waiter.reject(new Error(msg.error.message));
|
|
||||||
else waiter.resolve(msg.result);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (msg.method === 'Network.responseReceived') {
|
|
||||||
const requestId = msg.params?.requestId as string | undefined;
|
|
||||||
const responseUrl = msg.params?.response?.url as string | undefined;
|
|
||||||
if (!requestId || !responseUrl) return;
|
|
||||||
for (const [suffix, key] of desiredUrls) {
|
|
||||||
if (!responseUrl.includes(suffix)) continue;
|
|
||||||
try {
|
try {
|
||||||
const bodyResult = await send('Network.getResponseBody', { requestId });
|
const resp = await fetch(${JSON.stringify(apiUrl)}, { credentials: 'include' });
|
||||||
const parsed = JSON.parse(bodyResult.body);
|
if (!resp.ok) return null;
|
||||||
payload[key] = parsed.data ?? {};
|
const json = await resp.json();
|
||||||
completed.add(key);
|
return JSON.stringify(json.data ?? {});
|
||||||
if (completed.size === desiredUrls.size) {
|
} catch { return null; }
|
||||||
clearTimeout(timeout);
|
|
||||||
resolve();
|
|
||||||
}
|
|
||||||
} catch {}
|
|
||||||
}
|
}
|
||||||
|
`);
|
||||||
|
if (data && typeof data === 'string') {
|
||||||
|
try {
|
||||||
|
payload[key] = JSON.parse(data);
|
||||||
|
captured++;
|
||||||
|
} catch {}
|
||||||
}
|
}
|
||||||
});
|
} catch {}
|
||||||
});
|
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
ws.addEventListener('open', () => resolve(), { once: true });
|
|
||||||
ws.addEventListener('error', () => reject(new Error('Failed to connect to CDP note-detail target')), { once: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await send('Network.enable');
|
|
||||||
await send('Page.enable');
|
|
||||||
await send('Page.reload', { ignoreCache: true });
|
|
||||||
await waitForPayload;
|
|
||||||
} finally {
|
|
||||||
for (const waiter of pending.values()) {
|
|
||||||
waiter.reject(new Error('CDP note detail capture closed'));
|
|
||||||
}
|
|
||||||
pending.clear();
|
|
||||||
ws.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return completed.size > 0 ? payload : null;
|
return captured > 0 ? payload : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchCreatorNoteDetailRows(page: any, noteId: string): Promise<CreatorNoteDetailRow[]> {
|
export async function fetchCreatorNoteDetailRows(page: IPage, noteId: string): Promise<CreatorNoteDetailRow[]> {
|
||||||
await page.goto(`https://creator.xiaohongshu.com/statistics/note-detail?noteId=${encodeURIComponent(noteId)}`);
|
await page.goto(`https://creator.xiaohongshu.com/statistics/note-detail?noteId=${encodeURIComponent(noteId)}`);
|
||||||
await page.wait(4);
|
await page.wait(4);
|
||||||
|
|
||||||
const bodyText = await page.evaluate('() => document.body.innerText');
|
const bodyText = await page.evaluate('() => document.body.innerText');
|
||||||
const rows = parseCreatorNoteDetailText(typeof bodyText === 'string' ? bodyText : '', noteId);
|
const rows = parseCreatorNoteDetailText(typeof bodyText === 'string' ? bodyText : '', noteId);
|
||||||
const apiPayload = await captureNoteDetailApiPayload(noteId).catch(() => null);
|
const apiPayload = await captureNoteDetailPayload(page, noteId).catch(() => null);
|
||||||
appendTrendRows(rows, apiPayload ?? undefined);
|
appendTrendRows(rows, apiPayload ?? undefined);
|
||||||
appendAudienceRows(rows, apiPayload ?? undefined);
|
appendAudienceRows(rows, apiPayload ?? undefined);
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { cli, Strategy } from '../../registry.js';
|
import { cli, Strategy } from '../../registry.js';
|
||||||
|
import type { IPage } from '../../types.js';
|
||||||
|
|
||||||
const DATE_LINE_RE = /^发布于 (\d{4}年\d{2}月\d{2}日 \d{2}:\d{2})$/;
|
const DATE_LINE_RE = /^发布于 (\d{4}年\d{2}月\d{2}日 \d{2}:\d{2})$/;
|
||||||
const METRIC_LINE_RE = /^\d+$/;
|
const METRIC_LINE_RE = /^\d+$/;
|
||||||
@@ -134,7 +135,7 @@ function mapAnalyzeItems(items: NonNullable<CreatorAnalyzeApiResponse['data']>['
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchCreatorNotesByApi(page: any, limit: number): Promise<CreatorNoteRow[]> {
|
async function fetchCreatorNotesByApi(page: IPage, limit: number): Promise<CreatorNoteRow[]> {
|
||||||
const pageSize = Math.min(Math.max(limit, 10), 20);
|
const pageSize = Math.min(Math.max(limit, 10), 20);
|
||||||
const maxPages = Math.max(1, Math.ceil(limit / pageSize));
|
const maxPages = Math.max(1, Math.ceil(limit / pageSize));
|
||||||
const notes: CreatorNoteRow[] = [];
|
const notes: CreatorNoteRow[] = [];
|
||||||
@@ -183,145 +184,8 @@ async function fetchCreatorNotesByApi(page: any, limit: number): Promise<Creator
|
|||||||
return notes.slice(0, limit);
|
return notes.slice(0, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
function deriveCdpHttpBase(endpoint?: string): string | null {
|
export async function fetchCreatorNotes(page: IPage, limit: number): Promise<CreatorNoteRow[]> {
|
||||||
if (!endpoint) return null;
|
let notes = await fetchCreatorNotesByApi(page, limit);
|
||||||
try {
|
|
||||||
const url = new URL(endpoint);
|
|
||||||
if (url.protocol === 'ws:' || url.protocol === 'wss:') {
|
|
||||||
url.protocol = url.protocol === 'wss:' ? 'https:' : 'http:';
|
|
||||||
url.pathname = '';
|
|
||||||
url.search = '';
|
|
||||||
url.hash = '';
|
|
||||||
return url.toString().replace(/\/$/, '');
|
|
||||||
}
|
|
||||||
return `${url.protocol}//${url.host}`;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveRequestedCdpEndpoint(): { endpoint: string | null; requestedCdp: boolean } {
|
|
||||||
const endpoint = process.env.OPENCLI_CDP_ENDPOINT?.trim();
|
|
||||||
return {
|
|
||||||
endpoint: endpoint || null,
|
|
||||||
requestedCdp: Boolean(endpoint),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchCreatorNotesByCdp(limit: number): Promise<CreatorNoteRow[]> {
|
|
||||||
if (process.env.VITEST) return [];
|
|
||||||
|
|
||||||
const { endpoint, requestedCdp } = resolveRequestedCdpEndpoint();
|
|
||||||
if (!requestedCdp) return [];
|
|
||||||
|
|
||||||
const httpBase = deriveCdpHttpBase(endpoint ?? 'http://127.0.0.1:9222') ?? 'http://127.0.0.1:9222';
|
|
||||||
const targets = await fetch(`${httpBase}/json/list`).then((resp) => resp.json()) as Array<{
|
|
||||||
url?: string;
|
|
||||||
webSocketDebuggerUrl?: string;
|
|
||||||
}>;
|
|
||||||
const target = targets.find((entry) => entry.url?.includes('/statistics/data-analysis') && entry.webSocketDebuggerUrl);
|
|
||||||
if (!target?.webSocketDebuggerUrl) return [];
|
|
||||||
|
|
||||||
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
|
||||||
let nextId = 1;
|
|
||||||
const pending = new Map<number, { resolve: (value: any) => void; reject: (err: Error) => void }>();
|
|
||||||
|
|
||||||
const send = (method: string, params: Record<string, unknown> = {}) => {
|
|
||||||
const id = nextId++;
|
|
||||||
ws.send(JSON.stringify({ id, method, params }));
|
|
||||||
return new Promise<any>((resolve, reject) => {
|
|
||||||
pending.set(id, { resolve, reject });
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
const handlePendingMessage = (event: MessageEvent) => {
|
|
||||||
const msg = JSON.parse(String(event.data));
|
|
||||||
if (typeof msg.id !== 'number' || !pending.has(msg.id)) return;
|
|
||||||
const waiter = pending.get(msg.id)!;
|
|
||||||
pending.delete(msg.id);
|
|
||||||
if (msg.error) waiter.reject(new Error(msg.error.message));
|
|
||||||
else waiter.resolve(msg.result);
|
|
||||||
};
|
|
||||||
|
|
||||||
ws.addEventListener('message', handlePendingMessage);
|
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
|
||||||
ws.addEventListener('open', () => resolve(), { once: true });
|
|
||||||
ws.addEventListener('error', () => reject(new Error('Failed to connect to CDP data-analysis target')), { once: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
const pageSize = Math.min(Math.max(limit, 10), 20);
|
|
||||||
const maxPages = Math.max(1, Math.ceil(limit / pageSize));
|
|
||||||
const notes: CreatorNoteRow[] = [];
|
|
||||||
|
|
||||||
try {
|
|
||||||
await send('Network.enable');
|
|
||||||
await send('Page.enable');
|
|
||||||
|
|
||||||
for (let pageNum = 1; pageNum <= maxPages && notes.length < limit; pageNum++) {
|
|
||||||
const apiSuffix = `${NOTE_ANALYZE_API_PATH}?type=0&page_size=${pageSize}&page_num=${pageNum}`;
|
|
||||||
const pageUrl = `https://creator.xiaohongshu.com/statistics/data-analysis?type=0&page_size=${pageSize}&page_num=${pageNum}`;
|
|
||||||
|
|
||||||
const payload = await new Promise<CreatorAnalyzeApiResponse | null>((resolve) => {
|
|
||||||
const listener = async (event: MessageEvent) => {
|
|
||||||
const msg = JSON.parse(String(event.data));
|
|
||||||
if (typeof msg.id === 'number') return;
|
|
||||||
|
|
||||||
if (msg.method !== 'Network.responseReceived') return;
|
|
||||||
const requestId = msg.params?.requestId as string | undefined;
|
|
||||||
const responseUrl = msg.params?.response?.url as string | undefined;
|
|
||||||
if (!requestId || !responseUrl || !responseUrl.includes(apiSuffix)) return;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const bodyResult = await send('Network.getResponseBody', { requestId });
|
|
||||||
const parsed = JSON.parse(bodyResult.body) as CreatorAnalyzeApiResponse;
|
|
||||||
clearTimeout(timeout);
|
|
||||||
ws.removeEventListener('message', listener);
|
|
||||||
resolve(parsed);
|
|
||||||
} catch {
|
|
||||||
clearTimeout(timeout);
|
|
||||||
ws.removeEventListener('message', listener);
|
|
||||||
resolve(null);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const timeout = setTimeout(() => {
|
|
||||||
ws.removeEventListener('message', listener);
|
|
||||||
resolve(null);
|
|
||||||
}, 8000);
|
|
||||||
|
|
||||||
ws.addEventListener('message', listener);
|
|
||||||
void send('Page.navigate', { url: pageUrl }).catch(() => {
|
|
||||||
clearTimeout(timeout);
|
|
||||||
ws.removeEventListener('message', listener);
|
|
||||||
resolve(null);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
const items = payload?.data?.note_infos ?? [];
|
|
||||||
if (!items.length) break;
|
|
||||||
|
|
||||||
notes.push(...mapAnalyzeItems(items));
|
|
||||||
if (items.length < pageSize) break;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
ws.removeEventListener('message', handlePendingMessage);
|
|
||||||
for (const waiter of pending.values()) {
|
|
||||||
waiter.reject(new Error('CDP creator-notes capture closed'));
|
|
||||||
}
|
|
||||||
pending.clear();
|
|
||||||
ws.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
return notes.slice(0, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fetchCreatorNotes(page: any, limit: number): Promise<CreatorNoteRow[]> {
|
|
||||||
let notes = await fetchCreatorNotesByCdp(limit).catch(() => []);
|
|
||||||
|
|
||||||
if (notes.length === 0) {
|
|
||||||
notes = await fetchCreatorNotesByApi(page, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (notes.length === 0) {
|
if (notes.length === 0) {
|
||||||
await page.goto('https://creator.xiaohongshu.com/new/note-manager');
|
await page.goto('https://creator.xiaohongshu.com/new/note-manager');
|
||||||
|
|||||||
Reference in new issue
Block a user