refactor: remove raw CDP code, use IPage throughout
- Remove fetchCreatorNotesByCdp() and captureNoteDetailApiPayload() raw WebSocket code (~240 lines) — adapters should use IPage, not raw CDP - Replace direct CDP WebSocket with IPage.evaluate() in-page fetch - Fix page: any → IPage in all function signatures - Simplify to two-tier fallback: API+interceptor → DOM parse - Rebase onto latest main (resolves cdp.ts/daemon.ts conflicts)
This commit is contained in:
1 parent
8f1725982e
commit
4f74b45963
2 files changed
+34
-243
No files matched your search
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
type CreatorNoteDetailRow = {
|
||||
section: string;
|
||||
@@ -285,124 +286,50 @@ export function appendTrendRows(rows: CreatorNoteDetailRow[], payload?: NoteDeta
|
||||
return rows;
|
||||
}
|
||||
|
||||
function deriveCdpHttpBase(endpoint?: string): string | null {
|
||||
if (!endpoint) return null;
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
if (url.protocol === 'ws:' || url.protocol === 'wss:') {
|
||||
url.protocol = url.protocol === 'wss:' ? 'https:' : 'http:';
|
||||
url.pathname = '';
|
||||
url.search = '';
|
||||
url.hash = '';
|
||||
return url.toString().replace(/\/$/, '');
|
||||
}
|
||||
return `${url.protocol}//${url.host}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
const DETAIL_API_ENDPOINTS: Array<{ suffix: string; key: keyof NoteDetailApiPayload }> = [
|
||||
{ suffix: '/api/galaxy/creator/datacenter/note/base', key: 'noteBase' },
|
||||
{ suffix: '/api/galaxy/creator/datacenter/note/analyze/audience/trend', key: 'audienceTrend' },
|
||||
{ suffix: '/api/galaxy/creator/datacenter/note/audience/source/detail', key: 'audienceSourceDetail' },
|
||||
{ suffix: '/api/galaxy/creator/datacenter/note/audience', key: 'audienceSource' },
|
||||
];
|
||||
|
||||
function resolveRequestedCdpEndpoint(): { endpoint: string | null; requestedCdp: boolean } {
|
||||
const endpoint = process.env.OPENCLI_CDP_ENDPOINT?.trim();
|
||||
return {
|
||||
endpoint: endpoint || null,
|
||||
requestedCdp: Boolean(endpoint),
|
||||
};
|
||||
}
|
||||
|
||||
async function captureNoteDetailApiPayload(noteId: string): Promise<NoteDetailApiPayload | null> {
|
||||
const { endpoint, requestedCdp } = resolveRequestedCdpEndpoint();
|
||||
if (!requestedCdp) return null;
|
||||
|
||||
const httpBase = deriveCdpHttpBase(endpoint ?? 'http://127.0.0.1:9222') ?? 'http://127.0.0.1:9222';
|
||||
const targets = await fetch(`${httpBase}/json/list`).then((resp) => resp.json()) as Array<{
|
||||
url?: string;
|
||||
webSocketDebuggerUrl?: string;
|
||||
}>;
|
||||
const target = targets.find((entry) => entry.url?.includes(`/statistics/note-detail?noteId=${noteId}`) && entry.webSocketDebuggerUrl);
|
||||
if (!target?.webSocketDebuggerUrl) return null;
|
||||
|
||||
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
||||
async function captureNoteDetailPayload(page: IPage, noteId: string): Promise<NoteDetailApiPayload | null> {
|
||||
const payload: NoteDetailApiPayload = {};
|
||||
const pending = new Map<number, { resolve: (value: any) => void; reject: (err: Error) => void }>();
|
||||
const desiredUrls = new Map<string, keyof NoteDetailApiPayload>([
|
||||
[`/api/galaxy/creator/datacenter/note/base?note_id=${noteId}`, 'noteBase'],
|
||||
[`/api/galaxy/creator/datacenter/note/analyze/audience/trend?note_id=${noteId}`, 'audienceTrend'],
|
||||
[`/api/galaxy/creator/datacenter/note/audience/source?note_id=${noteId}`, 'audienceSource'],
|
||||
[`/api/galaxy/creator/datacenter/note/audience/source/detail?note_id=${noteId}`, 'audienceSourceDetail'],
|
||||
]);
|
||||
let nextId = 1;
|
||||
let captured = 0;
|
||||
|
||||
const send = (method: string, params: Record<string, unknown> = {}) => {
|
||||
const id = nextId++;
|
||||
ws.send(JSON.stringify({ id, method, params }));
|
||||
return new Promise<any>((resolve, reject) => {
|
||||
pending.set(id, { resolve, reject });
|
||||
});
|
||||
};
|
||||
|
||||
const completed = new Set<keyof NoteDetailApiPayload>();
|
||||
const waitForPayload = new Promise<void>((resolve) => {
|
||||
const timeout = setTimeout(() => resolve(), 8000);
|
||||
ws.addEventListener('message', async (event) => {
|
||||
const msg = JSON.parse(String(event.data));
|
||||
if (typeof msg.id === 'number' && pending.has(msg.id)) {
|
||||
const waiter = pending.get(msg.id)!;
|
||||
pending.delete(msg.id);
|
||||
if (msg.error) waiter.reject(new Error(msg.error.message));
|
||||
else waiter.resolve(msg.result);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.method === 'Network.responseReceived') {
|
||||
const requestId = msg.params?.requestId as string | undefined;
|
||||
const responseUrl = msg.params?.response?.url as string | undefined;
|
||||
if (!requestId || !responseUrl) return;
|
||||
for (const [suffix, key] of desiredUrls) {
|
||||
if (!responseUrl.includes(suffix)) continue;
|
||||
// Try to fetch each API endpoint through the page context (uses the browser's cookies)
|
||||
for (const { suffix, key } of DETAIL_API_ENDPOINTS) {
|
||||
const apiUrl = `${suffix}?note_id=${noteId}`;
|
||||
try {
|
||||
const data = await page.evaluate(`
|
||||
async () => {
|
||||
try {
|
||||
const bodyResult = await send('Network.getResponseBody', { requestId });
|
||||
const parsed = JSON.parse(bodyResult.body);
|
||||
payload[key] = parsed.data ?? {};
|
||||
completed.add(key);
|
||||
if (completed.size === desiredUrls.size) {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
}
|
||||
} catch {}
|
||||
const resp = await fetch(${JSON.stringify(apiUrl)}, { credentials: 'include' });
|
||||
if (!resp.ok) return null;
|
||||
const json = await resp.json();
|
||||
return JSON.stringify(json.data ?? {});
|
||||
} catch { return null; }
|
||||
}
|
||||
`);
|
||||
if (data && typeof data === 'string') {
|
||||
try {
|
||||
payload[key] = JSON.parse(data);
|
||||
captured++;
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
ws.addEventListener('open', () => resolve(), { once: true });
|
||||
ws.addEventListener('error', () => reject(new Error('Failed to connect to CDP note-detail target')), { once: true });
|
||||
});
|
||||
|
||||
try {
|
||||
await send('Network.enable');
|
||||
await send('Page.enable');
|
||||
await send('Page.reload', { ignoreCache: true });
|
||||
await waitForPayload;
|
||||
} finally {
|
||||
for (const waiter of pending.values()) {
|
||||
waiter.reject(new Error('CDP note detail capture closed'));
|
||||
}
|
||||
pending.clear();
|
||||
ws.close();
|
||||
} catch {}
|
||||
}
|
||||
|
||||
return completed.size > 0 ? payload : null;
|
||||
return captured > 0 ? payload : null;
|
||||
}
|
||||
|
||||
export async function fetchCreatorNoteDetailRows(page: any, noteId: string): Promise<CreatorNoteDetailRow[]> {
|
||||
export async function fetchCreatorNoteDetailRows(page: IPage, noteId: string): Promise<CreatorNoteDetailRow[]> {
|
||||
await page.goto(`https://creator.xiaohongshu.com/statistics/note-detail?noteId=${encodeURIComponent(noteId)}`);
|
||||
await page.wait(4);
|
||||
|
||||
const bodyText = await page.evaluate('() => document.body.innerText');
|
||||
const rows = parseCreatorNoteDetailText(typeof bodyText === 'string' ? bodyText : '', noteId);
|
||||
const apiPayload = await captureNoteDetailApiPayload(noteId).catch(() => null);
|
||||
const apiPayload = await captureNoteDetailPayload(page, noteId).catch(() => null);
|
||||
appendTrendRows(rows, apiPayload ?? undefined);
|
||||
appendAudienceRows(rows, apiPayload ?? undefined);
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
const DATE_LINE_RE = /^发布于 (\d{4}年\d{2}月\d{2}日 \d{2}:\d{2})$/;
|
||||
const METRIC_LINE_RE = /^\d+$/;
|
||||
@@ -134,7 +135,7 @@ function mapAnalyzeItems(items: NonNullable<CreatorAnalyzeApiResponse['data']>['
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchCreatorNotesByApi(page: any, limit: number): Promise<CreatorNoteRow[]> {
|
||||
async function fetchCreatorNotesByApi(page: IPage, limit: number): Promise<CreatorNoteRow[]> {
|
||||
const pageSize = Math.min(Math.max(limit, 10), 20);
|
||||
const maxPages = Math.max(1, Math.ceil(limit / pageSize));
|
||||
const notes: CreatorNoteRow[] = [];
|
||||
@@ -183,145 +184,8 @@ async function fetchCreatorNotesByApi(page: any, limit: number): Promise<Creator
|
||||
return notes.slice(0, limit);
|
||||
}
|
||||
|
||||
function deriveCdpHttpBase(endpoint?: string): string | null {
|
||||
if (!endpoint) return null;
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
if (url.protocol === 'ws:' || url.protocol === 'wss:') {
|
||||
url.protocol = url.protocol === 'wss:' ? 'https:' : 'http:';
|
||||
url.pathname = '';
|
||||
url.search = '';
|
||||
url.hash = '';
|
||||
return url.toString().replace(/\/$/, '');
|
||||
}
|
||||
return `${url.protocol}//${url.host}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveRequestedCdpEndpoint(): { endpoint: string | null; requestedCdp: boolean } {
|
||||
const endpoint = process.env.OPENCLI_CDP_ENDPOINT?.trim();
|
||||
return {
|
||||
endpoint: endpoint || null,
|
||||
requestedCdp: Boolean(endpoint),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchCreatorNotesByCdp(limit: number): Promise<CreatorNoteRow[]> {
|
||||
if (process.env.VITEST) return [];
|
||||
|
||||
const { endpoint, requestedCdp } = resolveRequestedCdpEndpoint();
|
||||
if (!requestedCdp) return [];
|
||||
|
||||
const httpBase = deriveCdpHttpBase(endpoint ?? 'http://127.0.0.1:9222') ?? 'http://127.0.0.1:9222';
|
||||
const targets = await fetch(`${httpBase}/json/list`).then((resp) => resp.json()) as Array<{
|
||||
url?: string;
|
||||
webSocketDebuggerUrl?: string;
|
||||
}>;
|
||||
const target = targets.find((entry) => entry.url?.includes('/statistics/data-analysis') && entry.webSocketDebuggerUrl);
|
||||
if (!target?.webSocketDebuggerUrl) return [];
|
||||
|
||||
const ws = new WebSocket(target.webSocketDebuggerUrl);
|
||||
let nextId = 1;
|
||||
const pending = new Map<number, { resolve: (value: any) => void; reject: (err: Error) => void }>();
|
||||
|
||||
const send = (method: string, params: Record<string, unknown> = {}) => {
|
||||
const id = nextId++;
|
||||
ws.send(JSON.stringify({ id, method, params }));
|
||||
return new Promise<any>((resolve, reject) => {
|
||||
pending.set(id, { resolve, reject });
|
||||
});
|
||||
};
|
||||
|
||||
const handlePendingMessage = (event: MessageEvent) => {
|
||||
const msg = JSON.parse(String(event.data));
|
||||
if (typeof msg.id !== 'number' || !pending.has(msg.id)) return;
|
||||
const waiter = pending.get(msg.id)!;
|
||||
pending.delete(msg.id);
|
||||
if (msg.error) waiter.reject(new Error(msg.error.message));
|
||||
else waiter.resolve(msg.result);
|
||||
};
|
||||
|
||||
ws.addEventListener('message', handlePendingMessage);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
ws.addEventListener('open', () => resolve(), { once: true });
|
||||
ws.addEventListener('error', () => reject(new Error('Failed to connect to CDP data-analysis target')), { once: true });
|
||||
});
|
||||
|
||||
const pageSize = Math.min(Math.max(limit, 10), 20);
|
||||
const maxPages = Math.max(1, Math.ceil(limit / pageSize));
|
||||
const notes: CreatorNoteRow[] = [];
|
||||
|
||||
try {
|
||||
await send('Network.enable');
|
||||
await send('Page.enable');
|
||||
|
||||
for (let pageNum = 1; pageNum <= maxPages && notes.length < limit; pageNum++) {
|
||||
const apiSuffix = `${NOTE_ANALYZE_API_PATH}?type=0&page_size=${pageSize}&page_num=${pageNum}`;
|
||||
const pageUrl = `https://creator.xiaohongshu.com/statistics/data-analysis?type=0&page_size=${pageSize}&page_num=${pageNum}`;
|
||||
|
||||
const payload = await new Promise<CreatorAnalyzeApiResponse | null>((resolve) => {
|
||||
const listener = async (event: MessageEvent) => {
|
||||
const msg = JSON.parse(String(event.data));
|
||||
if (typeof msg.id === 'number') return;
|
||||
|
||||
if (msg.method !== 'Network.responseReceived') return;
|
||||
const requestId = msg.params?.requestId as string | undefined;
|
||||
const responseUrl = msg.params?.response?.url as string | undefined;
|
||||
if (!requestId || !responseUrl || !responseUrl.includes(apiSuffix)) return;
|
||||
|
||||
try {
|
||||
const bodyResult = await send('Network.getResponseBody', { requestId });
|
||||
const parsed = JSON.parse(bodyResult.body) as CreatorAnalyzeApiResponse;
|
||||
clearTimeout(timeout);
|
||||
ws.removeEventListener('message', listener);
|
||||
resolve(parsed);
|
||||
} catch {
|
||||
clearTimeout(timeout);
|
||||
ws.removeEventListener('message', listener);
|
||||
resolve(null);
|
||||
}
|
||||
};
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
ws.removeEventListener('message', listener);
|
||||
resolve(null);
|
||||
}, 8000);
|
||||
|
||||
ws.addEventListener('message', listener);
|
||||
void send('Page.navigate', { url: pageUrl }).catch(() => {
|
||||
clearTimeout(timeout);
|
||||
ws.removeEventListener('message', listener);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
|
||||
const items = payload?.data?.note_infos ?? [];
|
||||
if (!items.length) break;
|
||||
|
||||
notes.push(...mapAnalyzeItems(items));
|
||||
if (items.length < pageSize) break;
|
||||
}
|
||||
} finally {
|
||||
ws.removeEventListener('message', handlePendingMessage);
|
||||
for (const waiter of pending.values()) {
|
||||
waiter.reject(new Error('CDP creator-notes capture closed'));
|
||||
}
|
||||
pending.clear();
|
||||
ws.close();
|
||||
}
|
||||
|
||||
return notes.slice(0, limit);
|
||||
}
|
||||
|
||||
export async function fetchCreatorNotes(page: any, limit: number): Promise<CreatorNoteRow[]> {
|
||||
let notes = await fetchCreatorNotesByCdp(limit).catch(() => []);
|
||||
|
||||
if (notes.length === 0) {
|
||||
notes = await fetchCreatorNotesByApi(page, limit);
|
||||
}
|
||||
export async function fetchCreatorNotes(page: IPage, limit: number): Promise<CreatorNoteRow[]> {
|
||||
let notes = await fetchCreatorNotesByApi(page, limit);
|
||||
|
||||
if (notes.length === 0) {
|
||||
await page.goto('https://creator.xiaohongshu.com/new/note-manager');
|
||||
|
||||
Reference in new issue
Block a user