From 4f74b45963bae4ea559df6cec660e43841967494 Mon Sep 17 00:00:00 2001 From: jackwener Date: Fri, 20 Mar 2026 16:26:56 +0800 Subject: [PATCH] refactor: remove raw CDP code, use IPage throughout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove fetchCreatorNotesByCdp() and captureNoteDetailApiPayload() raw WebSocket code (~240 lines) — adapters should use IPage, not raw CDP - Replace direct CDP WebSocket with IPage.evaluate() in-page fetch - Fix page: any → IPage in all function signatures - Simplify to two-tier fallback: API+interceptor → DOM parse - Rebase onto latest main (resolves cdp.ts/daemon.ts conflicts) --- src/clis/xiaohongshu/creator-note-detail.ts | 133 ++++-------------- src/clis/xiaohongshu/creator-notes.ts | 144 +------------------- 2 files changed, 34 insertions(+), 243 deletions(-) diff --git a/src/clis/xiaohongshu/creator-note-detail.ts b/src/clis/xiaohongshu/creator-note-detail.ts index 047d962..ec9e511 100644 --- a/src/clis/xiaohongshu/creator-note-detail.ts +++ b/src/clis/xiaohongshu/creator-note-detail.ts @@ -10,6 +10,7 @@ */ import { cli, Strategy } from '../../registry.js'; +import type { IPage } from '../../types.js'; type CreatorNoteDetailRow = { section: string; @@ -285,124 +286,50 @@ export function appendTrendRows(rows: CreatorNoteDetailRow[], payload?: NoteDeta return rows; } -function deriveCdpHttpBase(endpoint?: string): string | null { - if (!endpoint) return null; - try { - const url = new URL(endpoint); - if (url.protocol === 'ws:' || url.protocol === 'wss:') { - url.protocol = url.protocol === 'wss:' ? 'https:' : 'http:'; - url.pathname = ''; - url.search = ''; - url.hash = ''; - return url.toString().replace(/\/$/, ''); - } - return `${url.protocol}//${url.host}`; - } catch { - return null; - } -} +const DETAIL_API_ENDPOINTS: Array<{ suffix: string; key: keyof NoteDetailApiPayload }> = [ + { suffix: '/api/galaxy/creator/datacenter/note/base', key: 'noteBase' }, + { suffix: '/api/galaxy/creator/datacenter/note/analyze/audience/trend', key: 'audienceTrend' }, + { suffix: '/api/galaxy/creator/datacenter/note/audience/source/detail', key: 'audienceSourceDetail' }, + { suffix: '/api/galaxy/creator/datacenter/note/audience', key: 'audienceSource' }, +]; -function resolveRequestedCdpEndpoint(): { endpoint: string | null; requestedCdp: boolean } { - const endpoint = process.env.OPENCLI_CDP_ENDPOINT?.trim(); - return { - endpoint: endpoint || null, - requestedCdp: Boolean(endpoint), - }; -} - -async function captureNoteDetailApiPayload(noteId: string): Promise { - const { endpoint, requestedCdp } = resolveRequestedCdpEndpoint(); - if (!requestedCdp) return null; - - const httpBase = deriveCdpHttpBase(endpoint ?? 'http://127.0.0.1:9222') ?? 'http://127.0.0.1:9222'; - const targets = await fetch(`${httpBase}/json/list`).then((resp) => resp.json()) as Array<{ - url?: string; - webSocketDebuggerUrl?: string; - }>; - const target = targets.find((entry) => entry.url?.includes(`/statistics/note-detail?noteId=${noteId}`) && entry.webSocketDebuggerUrl); - if (!target?.webSocketDebuggerUrl) return null; - - const ws = new WebSocket(target.webSocketDebuggerUrl); +async function captureNoteDetailPayload(page: IPage, noteId: string): Promise { const payload: NoteDetailApiPayload = {}; - const pending = new Map void; reject: (err: Error) => void }>(); - const desiredUrls = new Map([ - [`/api/galaxy/creator/datacenter/note/base?note_id=${noteId}`, 'noteBase'], - [`/api/galaxy/creator/datacenter/note/analyze/audience/trend?note_id=${noteId}`, 'audienceTrend'], - [`/api/galaxy/creator/datacenter/note/audience/source?note_id=${noteId}`, 'audienceSource'], - [`/api/galaxy/creator/datacenter/note/audience/source/detail?note_id=${noteId}`, 'audienceSourceDetail'], - ]); - let nextId = 1; + let captured = 0; - const send = (method: string, params: Record = {}) => { - const id = nextId++; - ws.send(JSON.stringify({ id, method, params })); - return new Promise((resolve, reject) => { - pending.set(id, { resolve, reject }); - }); - }; - - const completed = new Set(); - const waitForPayload = new Promise((resolve) => { - const timeout = setTimeout(() => resolve(), 8000); - ws.addEventListener('message', async (event) => { - const msg = JSON.parse(String(event.data)); - if (typeof msg.id === 'number' && pending.has(msg.id)) { - const waiter = pending.get(msg.id)!; - pending.delete(msg.id); - if (msg.error) waiter.reject(new Error(msg.error.message)); - else waiter.resolve(msg.result); - return; - } - - if (msg.method === 'Network.responseReceived') { - const requestId = msg.params?.requestId as string | undefined; - const responseUrl = msg.params?.response?.url as string | undefined; - if (!requestId || !responseUrl) return; - for (const [suffix, key] of desiredUrls) { - if (!responseUrl.includes(suffix)) continue; + // Try to fetch each API endpoint through the page context (uses the browser's cookies) + for (const { suffix, key } of DETAIL_API_ENDPOINTS) { + const apiUrl = `${suffix}?note_id=${noteId}`; + try { + const data = await page.evaluate(` + async () => { try { - const bodyResult = await send('Network.getResponseBody', { requestId }); - const parsed = JSON.parse(bodyResult.body); - payload[key] = parsed.data ?? {}; - completed.add(key); - if (completed.size === desiredUrls.size) { - clearTimeout(timeout); - resolve(); - } - } catch {} + const resp = await fetch(${JSON.stringify(apiUrl)}, { credentials: 'include' }); + if (!resp.ok) return null; + const json = await resp.json(); + return JSON.stringify(json.data ?? {}); + } catch { return null; } } + `); + if (data && typeof data === 'string') { + try { + payload[key] = JSON.parse(data); + captured++; + } catch {} } - }); - }); - - await new Promise((resolve, reject) => { - ws.addEventListener('open', () => resolve(), { once: true }); - ws.addEventListener('error', () => reject(new Error('Failed to connect to CDP note-detail target')), { once: true }); - }); - - try { - await send('Network.enable'); - await send('Page.enable'); - await send('Page.reload', { ignoreCache: true }); - await waitForPayload; - } finally { - for (const waiter of pending.values()) { - waiter.reject(new Error('CDP note detail capture closed')); - } - pending.clear(); - ws.close(); + } catch {} } - return completed.size > 0 ? payload : null; + return captured > 0 ? payload : null; } -export async function fetchCreatorNoteDetailRows(page: any, noteId: string): Promise { +export async function fetchCreatorNoteDetailRows(page: IPage, noteId: string): Promise { await page.goto(`https://creator.xiaohongshu.com/statistics/note-detail?noteId=${encodeURIComponent(noteId)}`); await page.wait(4); const bodyText = await page.evaluate('() => document.body.innerText'); const rows = parseCreatorNoteDetailText(typeof bodyText === 'string' ? bodyText : '', noteId); - const apiPayload = await captureNoteDetailApiPayload(noteId).catch(() => null); + const apiPayload = await captureNoteDetailPayload(page, noteId).catch(() => null); appendTrendRows(rows, apiPayload ?? undefined); appendAudienceRows(rows, apiPayload ?? undefined); diff --git a/src/clis/xiaohongshu/creator-notes.ts b/src/clis/xiaohongshu/creator-notes.ts index 4e2d4a1..4757a99 100644 --- a/src/clis/xiaohongshu/creator-notes.ts +++ b/src/clis/xiaohongshu/creator-notes.ts @@ -9,6 +9,7 @@ */ import { cli, Strategy } from '../../registry.js'; +import type { IPage } from '../../types.js'; const DATE_LINE_RE = /^发布于 (\d{4}年\d{2}月\d{2}日 \d{2}:\d{2})$/; const METRIC_LINE_RE = /^\d+$/; @@ -134,7 +135,7 @@ function mapAnalyzeItems(items: NonNullable[' })); } -async function fetchCreatorNotesByApi(page: any, limit: number): Promise { +async function fetchCreatorNotesByApi(page: IPage, limit: number): Promise { const pageSize = Math.min(Math.max(limit, 10), 20); const maxPages = Math.max(1, Math.ceil(limit / pageSize)); const notes: CreatorNoteRow[] = []; @@ -183,145 +184,8 @@ async function fetchCreatorNotesByApi(page: any, limit: number): Promise { - if (process.env.VITEST) return []; - - const { endpoint, requestedCdp } = resolveRequestedCdpEndpoint(); - if (!requestedCdp) return []; - - const httpBase = deriveCdpHttpBase(endpoint ?? 'http://127.0.0.1:9222') ?? 'http://127.0.0.1:9222'; - const targets = await fetch(`${httpBase}/json/list`).then((resp) => resp.json()) as Array<{ - url?: string; - webSocketDebuggerUrl?: string; - }>; - const target = targets.find((entry) => entry.url?.includes('/statistics/data-analysis') && entry.webSocketDebuggerUrl); - if (!target?.webSocketDebuggerUrl) return []; - - const ws = new WebSocket(target.webSocketDebuggerUrl); - let nextId = 1; - const pending = new Map void; reject: (err: Error) => void }>(); - - const send = (method: string, params: Record = {}) => { - const id = nextId++; - ws.send(JSON.stringify({ id, method, params })); - return new Promise((resolve, reject) => { - pending.set(id, { resolve, reject }); - }); - }; - - const handlePendingMessage = (event: MessageEvent) => { - const msg = JSON.parse(String(event.data)); - if (typeof msg.id !== 'number' || !pending.has(msg.id)) return; - const waiter = pending.get(msg.id)!; - pending.delete(msg.id); - if (msg.error) waiter.reject(new Error(msg.error.message)); - else waiter.resolve(msg.result); - }; - - ws.addEventListener('message', handlePendingMessage); - - await new Promise((resolve, reject) => { - ws.addEventListener('open', () => resolve(), { once: true }); - ws.addEventListener('error', () => reject(new Error('Failed to connect to CDP data-analysis target')), { once: true }); - }); - - const pageSize = Math.min(Math.max(limit, 10), 20); - const maxPages = Math.max(1, Math.ceil(limit / pageSize)); - const notes: CreatorNoteRow[] = []; - - try { - await send('Network.enable'); - await send('Page.enable'); - - for (let pageNum = 1; pageNum <= maxPages && notes.length < limit; pageNum++) { - const apiSuffix = `${NOTE_ANALYZE_API_PATH}?type=0&page_size=${pageSize}&page_num=${pageNum}`; - const pageUrl = `https://creator.xiaohongshu.com/statistics/data-analysis?type=0&page_size=${pageSize}&page_num=${pageNum}`; - - const payload = await new Promise((resolve) => { - const listener = async (event: MessageEvent) => { - const msg = JSON.parse(String(event.data)); - if (typeof msg.id === 'number') return; - - if (msg.method !== 'Network.responseReceived') return; - const requestId = msg.params?.requestId as string | undefined; - const responseUrl = msg.params?.response?.url as string | undefined; - if (!requestId || !responseUrl || !responseUrl.includes(apiSuffix)) return; - - try { - const bodyResult = await send('Network.getResponseBody', { requestId }); - const parsed = JSON.parse(bodyResult.body) as CreatorAnalyzeApiResponse; - clearTimeout(timeout); - ws.removeEventListener('message', listener); - resolve(parsed); - } catch { - clearTimeout(timeout); - ws.removeEventListener('message', listener); - resolve(null); - } - }; - - const timeout = setTimeout(() => { - ws.removeEventListener('message', listener); - resolve(null); - }, 8000); - - ws.addEventListener('message', listener); - void send('Page.navigate', { url: pageUrl }).catch(() => { - clearTimeout(timeout); - ws.removeEventListener('message', listener); - resolve(null); - }); - }); - - const items = payload?.data?.note_infos ?? []; - if (!items.length) break; - - notes.push(...mapAnalyzeItems(items)); - if (items.length < pageSize) break; - } - } finally { - ws.removeEventListener('message', handlePendingMessage); - for (const waiter of pending.values()) { - waiter.reject(new Error('CDP creator-notes capture closed')); - } - pending.clear(); - ws.close(); - } - - return notes.slice(0, limit); -} - -export async function fetchCreatorNotes(page: any, limit: number): Promise { - let notes = await fetchCreatorNotesByCdp(limit).catch(() => []); - - if (notes.length === 0) { - notes = await fetchCreatorNotesByApi(page, limit); - } +export async function fetchCreatorNotes(page: IPage, limit: number): Promise { + let notes = await fetchCreatorNotesByApi(page, limit); if (notes.length === 0) { await page.goto('https://creator.xiaohongshu.com/new/note-manager');