diff --git a/src/clis/index.ts b/src/clis/index.ts index 3391b77..15086b3 100644 --- a/src/clis/index.ts +++ b/src/clis/index.ts @@ -16,4 +16,4 @@ import './bilibili/user-videos.js'; import './github/search.js'; // zhihu -import './zhihu/search.js'; +import './zhihu/question.js'; diff --git a/src/clis/reddit/hot.yaml b/src/clis/reddit/hot.yaml new file mode 100644 index 0000000..53e65f8 --- /dev/null +++ b/src/clis/reddit/hot.yaml @@ -0,0 +1,46 @@ +site: reddit +name: hot +description: Reddit 热门帖子 +domain: www.reddit.com + +args: + subreddit: + type: str + default: "" + description: "Subreddit name (e.g. programming). Empty for frontpage" + limit: + type: int + default: 20 + description: Number of posts + +pipeline: + - navigate: https://www.reddit.com + + - evaluate: | + (async () => { + const sub = '${{ args.subreddit }}'; + const path = sub ? '/r/' + sub + '/hot.json' : '/hot.json'; + const res = await fetch(path + '?limit=${{ args.limit }}&raw_json=1', { + credentials: 'include' + }); + const d = await res.json(); + return (d?.data?.children || []).map(c => ({ + title: c.data.title, + subreddit: c.data.subreddit_name_prefixed, + score: c.data.score, + comments: c.data.num_comments, + author: c.data.author, + url: 'https://www.reddit.com' + c.data.permalink, + })); + })() + + - map: + rank: ${{ index + 1 }} + title: ${{ item.title }} + subreddit: ${{ item.subreddit }} + score: ${{ item.score }} + comments: ${{ item.comments }} + + - limit: ${{ args.limit }} + +columns: [rank, title, subreddit, score, comments] diff --git a/src/clis/zhihu/hot.yaml b/src/clis/zhihu/hot.yaml index 01a212a..a754d1c 100644 --- a/src/clis/zhihu/hot.yaml +++ b/src/clis/zhihu/hot.yaml @@ -12,17 +12,31 @@ args: pipeline: - navigate: https://www.zhihu.com - - fetch: - url: https://www.zhihu.com/api/v4/search/top_search - params: - limit: ${{ args.limit }} - - - select: top_search.words + - evaluate: | + (async () => { + const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', { + credentials: 'include' + }); + const d = await res.json(); + return (d?.data || []).map((item) => { + const t = item.target || {}; + return { + title: t.title, + url: 'https://www.zhihu.com/question/' + t.id, + answer_count: t.answer_count, + follower_count: t.follower_count, + heat: item.detail_text || '', + }; + }); + })() - map: rank: ${{ index + 1 }} - title: ${{ item.query }} + title: ${{ item.title }} + heat: ${{ item.heat }} + answers: ${{ item.answer_count }} + url: ${{ item.url }} - limit: ${{ args.limit }} -columns: [rank, title] +columns: [rank, title, heat, answers] diff --git a/src/clis/zhihu/question.ts b/src/clis/zhihu/question.ts new file mode 100644 index 0000000..678a227 --- /dev/null +++ b/src/clis/zhihu/question.ts @@ -0,0 +1,45 @@ +import { cli, Strategy } from '../../registry.js'; + +cli({ + site: 'zhihu', + name: 'question', + description: '知乎问题详情和回答', + domain: 'www.zhihu.com', + strategy: Strategy.COOKIE, + args: [ + { name: 'id', required: true, help: 'Question ID (numeric)' }, + { name: 'limit', type: 'int', default: 5, help: 'Number of answers' }, + ], + columns: ['rank', 'author', 'votes', 'content'], + func: async (page, kwargs) => { + const { id, limit = 5 } = kwargs; + + const stripHtml = (html: string) => + (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').trim(); + + // Fetch question detail and answers in parallel via evaluate + const result = await page.evaluate(` + async () => { + const [qResp, aResp] = await Promise.all([ + fetch('https://www.zhihu.com/api/v4/questions/${id}?include=data[*].detail,excerpt,answer_count,follower_count,visit_count', {credentials: 'include'}), + fetch('https://www.zhihu.com/api/v4/questions/${id}/answers?limit=${limit}&offset=0&sort_by=default&include=data[*].content,voteup_count,comment_count,author', {credentials: 'include'}) + ]); + if (!qResp.ok || !aResp.ok) return { error: true }; + const q = await qResp.json(); + const a = await aResp.json(); + return { question: q, answers: a.data || [] }; + } + `); + + if (!result || result.error) throw new Error('Failed to fetch question. Are you logged in?'); + + const answers = (result.answers ?? []).slice(0, Number(limit)).map((a: any, i: number) => ({ + rank: i + 1, + author: a.author?.name ?? 'anonymous', + votes: a.voteup_count ?? 0, + content: stripHtml(a.content ?? '').slice(0, 200), + })); + + return answers; + }, +}); diff --git a/src/clis/zhihu/search.yaml b/src/clis/zhihu/search.yaml new file mode 100644 index 0000000..ab43d94 --- /dev/null +++ b/src/clis/zhihu/search.yaml @@ -0,0 +1,55 @@ +site: zhihu +name: search +description: 知乎搜索 +domain: www.zhihu.com + +args: + keyword: + type: str + required: true + description: Search keyword + limit: + type: int + default: 10 + description: Number of results + +pipeline: + - navigate: https://www.zhihu.com + + - evaluate: | + (async () => { + const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').replace(//g, '').replace(/<\/em>/g, '').trim(); + const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent('${{ args.keyword }}') + '&t=general&offset=0&limit=${{ args.limit }}', { + credentials: 'include' + }); + const d = await res.json(); + return (d?.data || []) + .filter(item => item.type === 'search_result') + .map(item => { + const obj = item.object || {}; + const q = obj.question || {}; + return { + type: obj.type, + title: strip(obj.title || q.name || ''), + excerpt: strip(obj.excerpt || '').substring(0, 100), + author: obj.author?.name || '', + votes: obj.voteup_count || 0, + url: obj.type === 'answer' + ? 'https://www.zhihu.com/question/' + q.id + '/answer/' + obj.id + : obj.type === 'article' + ? 'https://zhuanlan.zhihu.com/p/' + obj.id + : 'https://www.zhihu.com/question/' + obj.id, + }; + }); + })() + + - map: + rank: ${{ index + 1 }} + title: ${{ item.title }} + type: ${{ item.type }} + author: ${{ item.author }} + votes: ${{ item.votes }} + + - limit: ${{ args.limit }} + +columns: [rank, title, type, author, votes] diff --git a/src/explore.ts b/src/explore.ts index 22fedcf..b19e674 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -1,208 +1,128 @@ /** * Deep Explore: intelligent API discovery with response analysis. * - * Unlike simple page snapshots, Deep Explore intercepts network traffic, - * analyzes response schemas, and automatically infers capabilities that - * can be turned into CLI commands. - * - * Flow: - * 1. Navigate to target URL - * 2. Auto-scroll to trigger lazy loading - * 3. Capture network requests (with body analysis) - * 4. For each JSON response: detect list fields, infer columns, analyze auth - * 5. Detect frontend framework (Vue/React/Pinia/Next.js) - * 6. Generate structured capabilities.json + * Navigates to the target URL, auto-scrolls to trigger lazy loading, + * captures network traffic, analyzes JSON responses, and automatically + * infers CLI capabilities from discovered API endpoints. */ import * as fs from 'node:fs'; import * as path from 'node:path'; -import { browserSession, DEFAULT_BROWSER_EXPLORE_TIMEOUT, runWithTimeout } from './runtime.js'; +import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js'; // ── Site name detection ──────────────────────────────────────────────────── -const KNOWN_ALIASES: Record = { +const KNOWN_SITE_ALIASES: Record = { 'x.com': 'twitter', 'twitter.com': 'twitter', 'news.ycombinator.com': 'hackernews', 'www.zhihu.com': 'zhihu', 'www.bilibili.com': 'bilibili', + 'search.bilibili.com': 'bilibili', 'www.v2ex.com': 'v2ex', 'www.reddit.com': 'reddit', 'www.xiaohongshu.com': 'xiaohongshu', 'www.douban.com': 'douban', - 'www.weibo.com': 'weibo', 'search.bilibili.com': 'bilibili', + 'www.weibo.com': 'weibo', 'www.bbc.com': 'bbc', }; -function detectSiteName(url: string): string { +export function detectSiteName(url: string): string { try { const host = new URL(url).hostname.toLowerCase(); - if (host in KNOWN_ALIASES) return KNOWN_ALIASES[host]; + if (host in KNOWN_SITE_ALIASES) return KNOWN_SITE_ALIASES[host]; const parts = host.split('.').filter(p => p && p !== 'www'); if (parts.length >= 2) { if (['uk', 'jp', 'cn', 'com'].includes(parts[parts.length - 1]) && parts.length >= 3) { - return parts[parts.length - 3].replace(/[^a-z0-9]/g, ''); + return slugify(parts[parts.length - 3]); } - return parts[parts.length - 2].replace(/[^a-z0-9]/g, ''); + return slugify(parts[parts.length - 2]); } - return parts[0]?.replace(/[^a-z0-9]/g, '') ?? 'site'; + return parts[0] ? slugify(parts[0]) : 'site'; } catch { return 'site'; } } +export function slugify(value: string): string { + return value.trim().toLowerCase().replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '') || 'site'; +} + // ── Field & capability inference ─────────────────────────────────────────── -/** - * Common field names grouped by semantic role. - * Used to auto-detect which response fields map to which columns. - */ const FIELD_ROLES: Record = { - title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'], - url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'short_link', 'share_url'], - author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'], - score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'stat', 'play', 'favorite_count', 'reply_count'], - time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'], - id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'], - cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'], - category: ['category', 'tag', 'type', 'tname', 'channel', 'section'], + title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'], + url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'], + author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'], + score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'], + time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'], + id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'], + cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'], + category: ['category', 'tag', 'type', 'tname', 'channel', 'section'], }; -/** Param names that indicate searchable APIs */ const SEARCH_PARAMS = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w']); -/** Param names that indicate pagination */ const PAGINATION_PARAMS = new Set(['page', 'pn', 'offset', 'cursor', 'next', 'page_num']); -/** Param names that indicate limit control */ const LIMIT_PARAMS = new Set(['limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num']); -/** Content types to ignore */ -const IGNORED_CONTENT_TYPES = new Set(['image/', 'font/', 'text/css', 'text/javascript', 'application/javascript', 'application/wasm']); -/** Volatile query params to strip from patterns */ const VOLATILE_PARAMS = new Set(['w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign']); // ── Network analysis ─────────────────────────────────────────────────────── interface NetworkEntry { - method: string; - url: string; - status: number | null; - contentType: string; - responseBody?: any; - requestHeaders?: Record; - queryParams?: Record; + method: string; url: string; status: number | null; + contentType: string; responseBody?: any; requestHeaders?: Record; } interface AnalyzedEndpoint { - pattern: string; - method: string; - url: string; - status: number | null; - contentType: string; - queryParams: string[]; - hasSearchParam: boolean; - hasPaginationParam: boolean; - hasLimitParam: boolean; + pattern: string; method: string; url: string; status: number | null; + contentType: string; queryParams: string[]; score: number; + hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean; authIndicators: string[]; - responseAnalysis: ResponseAnalysis | null; -} - -interface ResponseAnalysis { - itemPath: string | null; - itemCount: number; - detectedFields: Record; // role → actual field name - sampleFieldNames: string[]; + responseAnalysis: { itemPath: string | null; itemCount: number; detectedFields: Record; sampleFields: string[] } | null; } interface InferredCapability { - name: string; - description: string; - strategy: string; - confidence: number; - endpoint: string; - itemPath: string | null; + name: string; description: string; strategy: string; confidence: number; + endpoint: string; itemPath: string | null; recommendedColumns: string[]; recommendedArgs: Array<{ name: string; type: string; required: boolean; default?: any }>; } /** - * Parse raw network output from Playwright MCP into structured entries. - * Handles both text format ([GET] url => [200]) and structured JSON. + * Parse raw network output from Playwright MCP. + * Handles text format: [GET] url => [200] */ -function parseNetworkOutput(raw: any): NetworkEntry[] { +function parseNetworkRequests(raw: any): NetworkEntry[] { if (typeof raw === 'string') { - // Playwright MCP returns network as text lines like: - // "[GET] https://api.example.com/xxx => [200] " - // May also have markdown headers like "### Result" const entries: NetworkEntry[] = []; - const lines = raw.split('\n').filter((l: string) => l.trim()); - for (const line of lines) { - // Format: [METHOD] URL => [STATUS] optional_extra - const bracketMatch = line.match(/^\[?(GET|POST|PUT|DELETE|PATCH|OPTIONS)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i); - if (bracketMatch) { - const [, method, url, status] = bracketMatch; + for (const line of raw.split('\n')) { + // Format: [GET] URL => [200] + const m = line.match(/\[?(GET|POST|PUT|DELETE|PATCH|OPTIONS)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i); + if (m) { + const [, method, url, status] = m; entries.push({ - method: method.toUpperCase(), - url, - status: status ? parseInt(status) : null, - contentType: url.endsWith('.json') ? 'application/json' : - (url.includes('/api/') || url.includes('/x/')) ? 'application/json' : '', - }); - continue; - } - // Legacy format: GET url → 200 (application/json) - const legacyMatch = line.match(/^(GET|POST|PUT|DELETE|PATCH|OPTIONS)\s+(\S+)\s*→?\s*(\d+)?\s*(?:\(([^)]*)\))?/i); - if (legacyMatch) { - const [, method, url, status, ct] = legacyMatch; - entries.push({ - method: method.toUpperCase(), - url, - status: status ? parseInt(status) : null, - contentType: ct ?? '', + method: method.toUpperCase(), url, status: status ? parseInt(status) : null, + contentType: (url.includes('/api/') || url.includes('/x/') || url.endsWith('.json')) ? 'application/json' : '', }); } } return entries; } if (Array.isArray(raw)) { - return raw.map((e: any) => ({ + return raw.filter(e => e && typeof e === 'object').map(e => ({ method: (e.method ?? 'GET').toUpperCase(), - url: e.url ?? e.request?.url ?? '', + url: String(e.url ?? e.request?.url ?? e.requestUrl ?? ''), status: e.status ?? e.statusCode ?? null, - contentType: e.contentType ?? e.mimeType ?? '', - responseBody: e.responseBody ?? e.body, - requestHeaders: e.requestHeaders ?? e.headers, + contentType: e.contentType ?? e.response?.contentType ?? '', + responseBody: e.responseBody, requestHeaders: e.requestHeaders, })); } return []; } - - -/** - * Normalize a URL into a pattern by replacing IDs with placeholders. - */ function urlToPattern(url: string): string { try { - const parsed = new URL(url); - const pathNorm = parsed.pathname - .replace(/\/\d+/g, '/{id}') - .replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}') - .replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}'); + const p = new URL(url); + const pathNorm = p.pathname.replace(/\/\d+/g, '/{id}').replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}').replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}'); const params: string[] = []; - parsed.searchParams.forEach((_v, k) => { - if (!VOLATILE_PARAMS.has(k)) params.push(k); - }); - const qs = params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''; - return `${parsed.host}${pathNorm}${qs}`; + p.searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) params.push(k); }); + return `${p.host}${pathNorm}${params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''}`; } catch { return url; } } -/** - * Extract query params from a URL. - */ -function extractQueryParams(url: string): Record { - try { - const params: Record = {}; - new URL(url).searchParams.forEach((v, k) => { params[k] = v; }); - return params; - } catch { return {}; } -} - -/** - * Detect auth indicators from request headers. - */ function detectAuthIndicators(headers?: Record): string[] { if (!headers) return []; const indicators: string[] = []; @@ -210,435 +130,257 @@ function detectAuthIndicators(headers?: Record): string[] { if (keys.some(k => k === 'authorization')) indicators.push('bearer'); if (keys.some(k => k.startsWith('x-csrf') || k.startsWith('x-xsrf'))) indicators.push('csrf'); if (keys.some(k => k.startsWith('x-s') || k === 'x-t' || k === 'x-s-common')) indicators.push('signature'); - if (keys.some(k => k === 'x-client-transaction-id')) indicators.push('transaction'); return indicators; } -/** - * Analyze a JSON response to find list data and field mappings. - */ -function analyzeResponseBody(body: any): ResponseAnalysis | null { +function analyzeResponseBody(body: any): AnalyzedEndpoint['responseAnalysis'] { if (!body || typeof body !== 'object') return null; - - // Try to find the main list in the response const candidates: Array<{ path: string; items: any[] }> = []; - function findArrays(obj: any, currentPath: string, depth: number) { + function findArrays(obj: any, path: string, depth: number) { if (depth > 4) return; - if (Array.isArray(obj) && obj.length >= 2) { - // Check if items are objects (not primitive arrays) - if (obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) { - candidates.push({ path: currentPath, items: obj }); - } + if (Array.isArray(obj) && obj.length >= 2 && obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) { + candidates.push({ path, items: obj }); } if (obj && typeof obj === 'object' && !Array.isArray(obj)) { - for (const [key, val] of Object.entries(obj)) { - const nextPath = currentPath ? `${currentPath}.${key}` : key; - findArrays(val, nextPath, depth + 1); - } + for (const [key, val] of Object.entries(obj)) findArrays(val, path ? `${path}.${key}` : key, depth + 1); } } - findArrays(body, '', 0); if (!candidates.length) return null; - // Pick the largest array as the main list candidates.sort((a, b) => b.items.length - a.items.length); const best = candidates[0]; + const sample = best.items[0]; + const sampleFields = sample && typeof sample === 'object' ? flattenFields(sample, '', 2) : []; - // Analyze field names in the first item - const sampleItem = best.items[0]; - const sampleFieldNames = sampleItem && typeof sampleItem === 'object' - ? flattenFieldNames(sampleItem, '', 2) - : []; - - // Match fields to semantic roles const detectedFields: Record = {}; for (const [role, aliases] of Object.entries(FIELD_ROLES)) { - for (const fieldName of sampleFieldNames) { - const basename = fieldName.split('.').pop()?.toLowerCase() ?? ''; - if (aliases.includes(basename)) { - detectedFields[role] = fieldName; - break; - } + for (const f of sampleFields) { + if (aliases.includes(f.split('.').pop()?.toLowerCase() ?? '')) { detectedFields[role] = f; break; } } } - return { - itemPath: best.path || null, - itemCount: best.items.length, - detectedFields, - sampleFieldNames, - }; + return { itemPath: best.path || null, itemCount: best.items.length, detectedFields, sampleFields }; } -/** - * Flatten nested object field names for analysis. - */ -function flattenFieldNames(obj: any, prefix: string, maxDepth: number): string[] { +function flattenFields(obj: any, prefix: string, maxDepth: number): string[] { if (maxDepth <= 0 || !obj || typeof obj !== 'object') return []; const names: string[] = []; for (const key of Object.keys(obj)) { - const fullKey = prefix ? `${prefix}.${key}` : key; - names.push(fullKey); - if (obj[key] && typeof obj[key] === 'object' && !Array.isArray(obj[key])) { - names.push(...flattenFieldNames(obj[key], fullKey, maxDepth - 1)); - } + const full = prefix ? `${prefix}.${key}` : key; + names.push(full); + if (obj[key] && typeof obj[key] === 'object' && !Array.isArray(obj[key])) names.push(...flattenFields(obj[key], full, maxDepth - 1)); } return names; } -/** - * Analyze a list of network entries into structured endpoints. - */ -function analyzeEndpoints(entries: NetworkEntry[], siteHost: string): AnalyzedEndpoint[] { - const seen = new Map(); - - for (const entry of entries) { - if (!entry.url) continue; - - // Skip static resources - const ct = entry.contentType.toLowerCase(); - if (IGNORED_CONTENT_TYPES.has(ct.split(';')[0]?.trim() ?? '') || - ct.includes('image/') || ct.includes('font/') || ct.includes('css') || - ct.includes('javascript') || ct.includes('wasm')) continue; - - // Skip non-JSON and failed responses - if (entry.status && entry.status >= 400) continue; - - const pattern = urlToPattern(entry.url); - const queryParams = extractQueryParams(entry.url); - const paramNames = Object.keys(queryParams).filter(k => !VOLATILE_PARAMS.has(k)); - - const key = `${entry.method}:${pattern}`; - if (seen.has(key)) continue; - - const endpoint: AnalyzedEndpoint = { - pattern, - method: entry.method, - url: entry.url, - status: entry.status, - contentType: ct, - queryParams: paramNames, - hasSearchParam: paramNames.some(p => SEARCH_PARAMS.has(p)), - hasPaginationParam: paramNames.some(p => PAGINATION_PARAMS.has(p)), - hasLimitParam: paramNames.some(p => LIMIT_PARAMS.has(p)), - authIndicators: detectAuthIndicators(entry.requestHeaders), - responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null, - }; - - seen.set(key, endpoint); - } - - return [...seen.values()]; +function scoreEndpoint(ep: { contentType: string; responseAnalysis: any; pattern: string; status: number | null; hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean }): number { + let s = 0; + if (ep.contentType.includes('json')) s += 10; + if (ep.responseAnalysis) { s += 5; s += Math.min(ep.responseAnalysis.itemCount, 10); s += Object.keys(ep.responseAnalysis.detectedFields).length * 2; } + if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/')) s += 3; + if (ep.hasSearchParam) s += 3; + if (ep.hasPaginationParam) s += 2; + if (ep.hasLimitParam) s += 2; + if (ep.status === 200) s += 2; + return s; } -/** - * Infer what strategy to use based on endpoint analysis. - */ -function inferStrategy(endpoint: AnalyzedEndpoint): string { - if (endpoint.authIndicators.includes('signature')) return 'intercept'; - if (endpoint.authIndicators.includes('transaction')) return 'header'; - if (endpoint.authIndicators.includes('bearer') || endpoint.authIndicators.includes('csrf')) return 'header'; - // Check if the URL is a public API (no auth indicators) - if (endpoint.authIndicators.length === 0) { - // If it's the same domain, likely cookie auth - return 'cookie'; - } - return 'cookie'; -} - -/** - * Infer the capability name from an endpoint pattern. - */ -function inferCapabilityName(endpoint: AnalyzedEndpoint, goal?: string): string { +function inferCapabilityName(url: string, goal?: string): string { if (goal) return goal; - - const u = endpoint.url.toLowerCase(); - const p = endpoint.pattern.toLowerCase(); - - // Match common patterns - if (endpoint.hasSearchParam) return 'search'; + const u = url.toLowerCase(); if (u.includes('hot') || u.includes('popular') || u.includes('ranking') || u.includes('trending')) return 'hot'; + if (u.includes('search')) return 'search'; if (u.includes('feed') || u.includes('timeline') || u.includes('dynamic')) return 'feed'; if (u.includes('comment') || u.includes('reply')) return 'comments'; if (u.includes('history')) return 'history'; - if (u.includes('profile') || u.includes('userinfo') || u.includes('/me') || u.includes('myinfo')) return 'me'; - if (u.includes('video') || u.includes('article') || u.includes('detail') || u.includes('view')) return 'detail'; + if (u.includes('profile') || u.includes('userinfo') || u.includes('/me')) return 'me'; if (u.includes('favorite') || u.includes('collect') || u.includes('bookmark')) return 'favorite'; - if (u.includes('notification') || u.includes('notice')) return 'notifications'; - - // Fallback: try to extract from path try { - const pathname = new URL(endpoint.url).pathname; - const segments = pathname.split('/').filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i)); - if (segments.length) return segments[segments.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase(); + const segs = new URL(url).pathname.split('/').filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i)); + if (segs.length) return segs[segs.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase(); } catch {} - return 'data'; } -/** - * Build recommended columns from response analysis. - */ -function buildRecommendedColumns(analysis: ResponseAnalysis | null): string[] { - if (!analysis) return ['title', 'url']; - const cols: string[] = []; - // Prioritize: title → url → author → score → time - const priority = ['title', 'url', 'author', 'score', 'time']; - for (const role of priority) { - if (analysis.detectedFields[role]) cols.push(role); - } - return cols.length ? cols : ['title', 'url']; -} - -/** - * Build recommended args from endpoint query params. - */ -function buildRecommendedArgs(endpoint: AnalyzedEndpoint): InferredCapability['recommendedArgs'] { - const args: InferredCapability['recommendedArgs'] = []; - - if (endpoint.hasSearchParam) { - const paramName = endpoint.queryParams.find(p => SEARCH_PARAMS.has(p)) ?? 'keyword'; - args.push({ name: 'keyword', type: 'str', required: true }); - } - - // Always add limit - args.push({ name: 'limit', type: 'int', required: false, default: 20 }); - - if (endpoint.hasPaginationParam) { - args.push({ name: 'page', type: 'int', required: false, default: 1 }); - } - - return args; -} - -/** - * Score an endpoint's interest level for capability generation. - * Higher score = more likely to be a useful API endpoint. - */ -function scoreEndpoint(ep: AnalyzedEndpoint): number { - let score = 0; - // JSON content type is strongly preferred - if (ep.contentType.includes('json')) score += 10; - // Has response analysis with items - if (ep.responseAnalysis) { - score += 5; - score += Math.min(ep.responseAnalysis.itemCount, 10); - score += Object.keys(ep.responseAnalysis.detectedFields).length * 2; - } - // API-like path patterns - if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/')) score += 3; - // Has search/pagination params - if (ep.hasSearchParam) score += 3; - if (ep.hasPaginationParam) score += 2; - if (ep.hasLimitParam) score += 2; - // 200 OK - if (ep.status === 200) score += 2; - return score; +function inferStrategy(authIndicators: string[]): string { + if (authIndicators.includes('signature')) return 'intercept'; + if (authIndicators.includes('bearer') || authIndicators.includes('csrf')) return 'header'; + return 'cookie'; } // ── Framework detection ──────────────────────────────────────────────────── const FRAMEWORK_DETECT_JS = ` -(() => { - const result = {}; - try { - const app = document.querySelector('#app'); - result.vue3 = !!(app && app.__vue_app__); - result.vue2 = !!(app && app.__vue__); - result.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]'); - result.nextjs = !!window.__NEXT_DATA__; - result.nuxt = !!window.__NUXT__; - if (result.vue3 && app.__vue_app__) { - const gp = app.__vue_app__.config?.globalProperties; - result.pinia = !!(gp && gp.$pinia); - result.vuex = !!(gp && gp.$store); - } - } catch {} - return JSON.stringify(result); -})() + () => { + const r = {}; + try { + const app = document.querySelector('#app'); + r.vue3 = !!(app && app.__vue_app__); + r.vue2 = !!(app && app.__vue__); + r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]'); + r.nextjs = !!window.__NEXT_DATA__; + r.nuxt = !!window.__NUXT__; + if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); } + } catch {} + return r; + } `; // ── Main explore function ────────────────────────────────────────────────── -export async function exploreUrl(url: string, opts: any = {}): Promise { - const site = opts.site ?? detectSiteName(url); - const outDir = opts.outDir ?? path.join('.opencli', 'explore', site); - fs.mkdirSync(outDir, { recursive: true }); +export async function exploreUrl( + url: string, + opts: { + BrowserFactory: new () => any; + site?: string; goal?: string; authenticated?: boolean; + outDir?: string; waitSeconds?: number; query?: string; + clickLabels?: string[]; auto?: boolean; + }, +): Promise> { + const waitSeconds = opts.waitSeconds ?? 3.0; + const exploreTimeout = Math.max(DEFAULT_BROWSER_EXPLORE_TIMEOUT, 45.0 + waitSeconds * 8.0); - const result: any = await browserSession(opts.BrowserFactory, async (page: any) => { + return browserSession(opts.BrowserFactory, async (page) => { return runWithTimeout((async () => { // Step 1: Navigate await page.goto(url); - await page.wait(opts.waitSeconds ?? 3); + await page.wait(waitSeconds); - // Step 2: Auto-scroll to trigger lazy loading - for (let i = 0; i < 3; i++) { - await page.scroll('down'); - await page.wait(1); - } + // Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist) + for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); } - // Step 3: Capture network traffic + // Step 3: Read page metadata + const metadata = await readPageMetadata(page); + + // Step 4: Capture network traffic const rawNetwork = await page.networkRequests(false); - const networkEntries = parseNetworkOutput(rawNetwork); - - // Step 4: For JSON endpoints, try to fetch response body in-browser - const jsonEndpoints = networkEntries.filter(e => - e.contentType.includes('json') && e.method === 'GET' && e.status === 200 - ); + const networkEntries = parseNetworkRequests(rawNetwork); + // Step 5: For JSON endpoints, re-fetch response body in-browser + const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200); for (const ep of jsonEndpoints.slice(0, 10)) { - // Only fetch body for promising-looking API endpoints - if (ep.url.includes('/api/') || ep.url.includes('/x/') || ep.url.includes('/web/') || - ep.contentType.includes('json')) { - try { - const bodyResult = await page.evaluate(` - async () => { - try { - const resp = await fetch(${JSON.stringify(ep.url)}, { credentials: 'include' }); - if (!resp.ok) return null; - const data = await resp.json(); - return JSON.stringify(data).slice(0, 10000); - } catch { return null; } - } - `); - if (bodyResult && typeof bodyResult === 'string') { - try { ep.responseBody = JSON.parse(bodyResult); } catch {} - } else if (bodyResult && typeof bodyResult === 'object') { - ep.responseBody = bodyResult; - } - } catch {} - } + try { + const body = await page.evaluate(`async () => { try { const r = await fetch(${JSON.stringify(ep.url)}, {credentials:'include'}); if (!r.ok) return null; const d = await r.json(); return JSON.stringify(d).slice(0,10000); } catch { return null; } }`); + if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} } + else if (body && typeof body === 'object') ep.responseBody = body; + } catch {} } - // Step 5: Detect frontend framework + // Step 6: Detect framework let framework: Record = {}; - try { - const fwResult = await page.evaluate(FRAMEWORK_DETECT_JS); - if (typeof fwResult === 'string') framework = JSON.parse(fwResult); - else if (typeof fwResult === 'object') framework = fwResult; - } catch {} - - // Step 6: Get page metadata - let title = '', finalUrl = ''; - try { - const meta = await page.evaluate(` - () => JSON.stringify({ url: window.location.href, title: document.title || '' }) - `); - if (typeof meta === 'string') { - const parsed = JSON.parse(meta); - title = parsed.title; finalUrl = parsed.url; - } else if (typeof meta === 'object') { - title = meta.title; finalUrl = meta.url; - } - } catch {} + try { const fw = await page.evaluate(FRAMEWORK_DETECT_JS); if (fw && typeof fw === 'object') framework = fw; } catch {} // Step 7: Analyze endpoints - let siteHost = ''; - try { siteHost = new URL(url).hostname; } catch {} - const analyzedEndpoints = analyzeEndpoints(networkEntries, siteHost); + const seen = new Map(); + for (const entry of networkEntries) { + if (!entry.url) continue; + const ct = entry.contentType.toLowerCase(); + if (ct.includes('image/') || ct.includes('font/') || ct.includes('css') || ct.includes('javascript') || ct.includes('wasm')) continue; + if (entry.status && entry.status >= 400) continue; - // Step 8: Score and rank endpoints - const scoredEndpoints = analyzedEndpoints - .map(ep => ({ ...ep, score: scoreEndpoint(ep) })) - .filter(ep => ep.score >= 5) - .sort((a, b) => b.score - a.score); + const pattern = urlToPattern(entry.url); + const key = `${entry.method}:${pattern}`; + if (seen.has(key)) continue; - // Step 9: Infer capabilities from top endpoints + const qp: string[] = []; + try { new URL(entry.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {} + + const ep: AnalyzedEndpoint = { + pattern, method: entry.method, url: entry.url, status: entry.status, contentType: ct, + queryParams: qp, hasSearchParam: qp.some(p => SEARCH_PARAMS.has(p)), + hasPaginationParam: qp.some(p => PAGINATION_PARAMS.has(p)), + hasLimitParam: qp.some(p => LIMIT_PARAMS.has(p)), + authIndicators: detectAuthIndicators(entry.requestHeaders), + responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null, + score: 0, + }; + ep.score = scoreEndpoint(ep); + seen.set(key, ep); + } + + const analyzedEndpoints = [...seen.values()].filter(ep => ep.score >= 5).sort((a, b) => b.score - a.score); + + // Step 8: Infer capabilities const capabilities: InferredCapability[] = []; const usedNames = new Set(); - - for (const ep of scoredEndpoints.slice(0, 8)) { - let capName = inferCapabilityName(ep, opts.goal); - // Deduplicate names + for (const ep of analyzedEndpoints.slice(0, 8)) { + let capName = inferCapabilityName(ep.url, opts.goal); if (usedNames.has(capName)) { const suffix = ep.pattern.split('/').filter(s => s && !s.startsWith('{') && !s.includes('.')).pop(); capName = suffix ? `${capName}_${suffix}` : `${capName}_${usedNames.size}`; } usedNames.add(capName); + const cols: string[] = []; + if (ep.responseAnalysis) { + for (const role of ['title', 'url', 'author', 'score', 'time']) { + if (ep.responseAnalysis.detectedFields[role]) cols.push(role); + } + } + + const args: InferredCapability['recommendedArgs'] = []; + if (ep.hasSearchParam) args.push({ name: 'keyword', type: 'str', required: true }); + args.push({ name: 'limit', type: 'int', required: false, default: 20 }); + if (ep.hasPaginationParam) args.push({ name: 'page', type: 'int', required: false, default: 1 }); + capabilities.push({ - name: capName, - description: `${site} ${capName}`, - strategy: inferStrategy(ep), - confidence: Math.min(ep.score / 20, 1.0), - endpoint: ep.pattern, + name: capName, description: `${opts.site ?? detectSiteName(url)} ${capName}`, + strategy: inferStrategy(ep.authIndicators), + confidence: Math.min(ep.score / 20, 1.0), endpoint: ep.pattern, itemPath: ep.responseAnalysis?.itemPath ?? null, - recommendedColumns: buildRecommendedColumns(ep.responseAnalysis), - recommendedArgs: buildRecommendedArgs(ep), + recommendedColumns: cols.length ? cols : ['title', 'url'], + recommendedArgs: args, }); } - // Step 10: Determine auth strategy - const allAuthIndicators = new Set(analyzedEndpoints.flatMap(ep => ep.authIndicators)); - let topStrategy = 'cookie'; - if (allAuthIndicators.has('signature')) topStrategy = 'intercept'; - else if (allAuthIndicators.has('transaction') || allAuthIndicators.has('bearer')) topStrategy = 'header'; - else if (allAuthIndicators.size === 0 && scoredEndpoints.some(ep => ep.contentType.includes('json'))) topStrategy = 'public'; + // Step 9: Determine overall auth strategy + const allAuth = new Set(analyzedEndpoints.flatMap(ep => ep.authIndicators)); + const topStrategy = allAuth.has('signature') ? 'intercept' : allAuth.has('bearer') || allAuth.has('csrf') ? 'header' : allAuth.size === 0 ? 'public' : 'cookie'; - return { - site, - target_url: url, - final_url: finalUrl, - title, - framework, - top_strategy: topStrategy, - endpoint_count: analyzedEndpoints.length, - api_endpoint_count: scoredEndpoints.length, - capabilities, - endpoints: scoredEndpoints.map(ep => ({ - pattern: ep.pattern, - method: ep.method, - url: ep.url, - status: ep.status, - contentType: ep.contentType, - score: ep.score, - queryParams: ep.queryParams, - itemPath: ep.responseAnalysis?.itemPath ?? null, - itemCount: ep.responseAnalysis?.itemCount ?? 0, - detectedFields: ep.responseAnalysis?.detectedFields ?? {}, - authIndicators: ep.authIndicators, - })), - auth_indicators: [...allAuthIndicators], + const siteName = opts.site ?? detectSiteName(metadata.url || url); + const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName); + fs.mkdirSync(targetDir, { recursive: true }); + + const result = { + site: siteName, target_url: url, final_url: metadata.url, title: metadata.title, + framework, top_strategy: topStrategy, + endpoint_count: analyzedEndpoints.length + [...seen.values()].filter(ep => ep.score < 5).length, + api_endpoint_count: analyzedEndpoints.length, + capabilities, auth_indicators: [...allAuth], }; - })(), { timeout: DEFAULT_BROWSER_EXPLORE_TIMEOUT, label: 'explore' }); + + // Write artifacts + fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify({ + site: siteName, target_url: url, final_url: metadata.url, title: metadata.title, + framework, top_strategy: topStrategy, explored_at: new Date().toISOString(), + }, null, 2)); + fs.writeFileSync(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({ + pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status, + contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams, + itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0, + detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators, + })), null, 2)); + fs.writeFileSync(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)); + fs.writeFileSync(path.join(targetDir, 'auth.json'), JSON.stringify({ + top_strategy: topStrategy, indicators: [...allAuth], framework, + }, null, 2)); + + return { ...result, out_dir: targetDir }; + })(), { timeout: exploreTimeout, label: `Explore ${url}` }); }); - - // Write artifacts - const manifest = { - site: result.site, - target_url: result.target_url, - final_url: result.final_url, - title: result.title, - framework: result.framework, - top_strategy: result.top_strategy, - explored_at: new Date().toISOString(), - }; - fs.writeFileSync(path.join(outDir, 'manifest.json'), JSON.stringify(manifest, null, 2)); - fs.writeFileSync(path.join(outDir, 'endpoints.json'), JSON.stringify(result.endpoints ?? [], null, 2)); - fs.writeFileSync(path.join(outDir, 'capabilities.json'), JSON.stringify(result.capabilities ?? [], null, 2)); - fs.writeFileSync(path.join(outDir, 'auth.json'), JSON.stringify({ - top_strategy: result.top_strategy, - indicators: result.auth_indicators ?? [], - framework: result.framework ?? {}, - }, null, 2)); - - return { ...result, out_dir: outDir }; } -export function renderExploreSummary(result: any): string { +export function renderExploreSummary(result: Record): string { const lines = [ - 'opencli explore: OK', - `Site: ${result.site}`, - `URL: ${result.target_url}`, - `Title: ${result.title || '(none)'}`, - `Strategy: ${result.top_strategy}`, + 'opencli explore: OK', `Site: ${result.site}`, `URL: ${result.target_url}`, + `Title: ${result.title || '(none)'}`, `Strategy: ${result.top_strategy}`, `Endpoints: ${result.endpoint_count} total, ${result.api_endpoint_count} API`, `Capabilities: ${result.capabilities?.length ?? 0}`, ]; for (const cap of (result.capabilities ?? []).slice(0, 5)) { - lines.push(` • ${cap.name} (${cap.strategy}, confidence: ${(cap.confidence * 100).toFixed(0)}%)`); + lines.push(` • ${cap.name} (${cap.strategy}, ${(cap.confidence * 100).toFixed(0)}%)`); } const fw = result.framework ?? {}; const fwNames = Object.entries(fw).filter(([, v]) => v).map(([k]) => k); @@ -646,3 +388,11 @@ export function renderExploreSummary(result: any): string { lines.push(`Output: ${result.out_dir}`); return lines.join('\n'); } + +async function readPageMetadata(page: any): Promise<{ url: string; title: string }> { + try { + const result = await page.evaluate(`() => ({ url: window.location.href, title: document.title || '' })`); + if (result && typeof result === 'object') return { url: String(result.url ?? ''), title: String(result.title ?? '') }; + } catch {} + return { url: '', title: '' }; +} diff --git a/src/synthesize.ts b/src/synthesize.ts index d61b477..cae054c 100644 --- a/src/synthesize.ts +++ b/src/synthesize.ts @@ -1,167 +1,164 @@ /** - * Synthesize: turn explore capabilities into ready-to-use CLI definitions. - * - * Takes the structured capabilities from Deep Explore and generates - * YAML pipeline files that can be directly registered as CLI commands. - * - * This is the bridge between discovery (explore) and usability (CLI). + * Synthesize candidate CLIs from explore artifacts. + * Generates evaluate-based YAML pipelines (matching hand-written adapter patterns). */ import * as fs from 'node:fs'; import * as path from 'node:path'; import yaml from 'js-yaml'; -export function synthesizeFromExplore(target: string, opts: any = {}): any { - const exploreDir = fs.existsSync(target) ? target : path.join('.opencli', 'explore', target); - if (!fs.existsSync(exploreDir)) throw new Error(`Explore dir not found: ${target}`); - - const manifest = JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8')); - const capabilities = JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8')); - const endpoints = JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8')); - const auth = JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8')); - - const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates'); - fs.mkdirSync(targetDir, { recursive: true }); - - const site = manifest.site; - const topN = opts.top ?? 5; - const candidates: any[] = []; - - // Sort capabilities by confidence - const sortedCaps = [...capabilities] - .sort((a: any, b: any) => (b.confidence ?? 0) - (a.confidence ?? 0)) - .slice(0, topN); - - for (const cap of sortedCaps) { - // Find the matching endpoint for more detail - const endpoint = endpoints.find((ep: any) => ep.pattern === cap.endpoint) ?? - endpoints[0]; - - const candidate = buildCandidateYaml(site, manifest, cap, endpoint); - const fileName = `${cap.name}.yaml`; - const filePath = path.join(targetDir, fileName); - fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 })); - - candidates.push({ - name: cap.name, - path: filePath, - strategy: cap.strategy, - endpoint: cap.endpoint, - confidence: cap.confidence, - columns: candidate.yaml.columns, - }); - } - - const index = { - site, - target_url: manifest.target_url, - generated_from: exploreDir, - candidate_count: candidates.length, - candidates, - }; - fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2)); - - return { - site, - explore_dir: exploreDir, - out_dir: targetDir, - candidate_count: candidates.length, - candidates, - }; -} - /** Volatile params to strip from generated URLs */ const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']); const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']); const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']); const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']); -/** - * Build a clean templated URL from a raw API URL. - * - Strips volatile params (w_rid, wts, etc.) - * - Templates search, limit, and pagination params - * - Builds URL string manually to avoid URL encoding of ${{ }} expressions - */ -function buildTemplatedUrl(rawUrl: string, cap: any, endpoint: any): string { +export function synthesizeFromExplore( + target: string, + opts: { outDir?: string; top?: number } = {}, +): Record { + const exploreDir = resolveExploreDir(target); + const bundle = loadExploreBundle(exploreDir); + + const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates'); + fs.mkdirSync(targetDir, { recursive: true }); + + const site = bundle.manifest.site; + const capabilities = (bundle.capabilities ?? []) + .sort((a: any, b: any) => (b.confidence ?? 0) - (a.confidence ?? 0)) + .slice(0, opts.top ?? 3); + const candidates: any[] = []; + + for (const cap of capabilities) { + const endpoint = chooseEndpoint(cap, bundle.endpoints); + if (!endpoint) continue; + const candidate = buildCandidateYaml(site, bundle.manifest, cap, endpoint); + const filePath = path.join(targetDir, `${candidate.name}.yaml`); + fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 })); + candidates.push({ name: candidate.name, path: filePath, strategy: cap.strategy, confidence: cap.confidence }); + } + + const index = { site, target_url: bundle.manifest.target_url, generated_from: exploreDir, candidate_count: candidates.length, candidates }; + fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2)); + + return { site, explore_dir: exploreDir, out_dir: targetDir, candidate_count: candidates.length, candidates }; +} + +export function renderSynthesizeSummary(result: Record): string { + const lines = ['opencli synthesize: OK', `Site: ${result.site}`, `Source: ${result.explore_dir}`, `Candidates: ${result.candidate_count}`]; + for (const c of result.candidates ?? []) lines.push(` • ${c.name} (${c.strategy}, ${((c.confidence ?? 0) * 100).toFixed(0)}% confidence) → ${c.path}`); + return lines.join('\n'); +} + +export function resolveExploreDir(target: string): string { + if (fs.existsSync(target)) return target; + const candidate = path.join('.opencli', 'explore', target); + if (fs.existsSync(candidate)) return candidate; + throw new Error(`Explore directory not found: ${target}`); +} + +export function loadExploreBundle(exploreDir: string): Record { + return { + manifest: JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8')), + endpoints: JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8')), + capabilities: JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8')), + auth: JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8')), + }; +} + +function chooseEndpoint(cap: any, endpoints: any[]): any | null { + if (!endpoints.length) return null; + // Match by endpoint pattern from capability + if (cap.endpoint) { + const match = endpoints.find((e: any) => e.pattern === cap.endpoint || e.url?.includes(cap.endpoint)); + if (match) return match; + } + return endpoints.sort((a: any, b: any) => (b.score ?? 0) - (a.score ?? 0))[0]; +} + +// ── URL templating ───────────────────────────────────────────────────────── + +function buildTemplatedUrl(rawUrl: string, cap: any, _endpoint: any): string { try { const u = new URL(rawUrl); const base = `${u.protocol}//${u.host}${u.pathname}`; const params: Array<[string, string]> = []; - const hasKeyword = cap.recommendedArgs?.some((a: any) => a.name === 'keyword'); u.searchParams.forEach((v, k) => { - // Skip volatile params if (VOLATILE_PARAMS.has(k)) return; - - // Template known param types - if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) { - params.push([k, '${{ args.keyword }}']); - } else if (LIMIT_PARAM_NAMES.has(k)) { - params.push([k, '${{ args.limit | default(20) }}']); - } else if (PAGE_PARAM_NAMES.has(k)) { - params.push([k, '${{ args.page | default(1) }}']); - } else { - params.push([k, v]); - } + if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) params.push([k, '${{ args.keyword }}']); + else if (LIMIT_PARAM_NAMES.has(k)) params.push([k, '${{ args.limit | default(20) }}']); + else if (PAGE_PARAM_NAMES.has(k)) params.push([k, '${{ args.page | default(1) }}']); + else params.push([k, v]); }); - if (params.length === 0) return base; - return base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&'); - } catch { - return rawUrl; - } + return params.length ? base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&') : base; + } catch { return rawUrl; } } /** - * Build a YAML pipeline definition from a capability + endpoint. + * Build inline evaluate script for browser-based fetch+parse. + * Follows patterns from bilibili/hot.yaml and twitter/trending.yaml. */ +function buildEvaluateScript(url: string, itemPath: string, endpoint: any): string { + const pathChain = itemPath.split('.').map((p: string) => `?.${p}`).join(''); + const detectedFields = endpoint?.detectedFields ?? {}; + const hasFields = Object.keys(detectedFields).length > 0; + + let mapCode = ''; + if (hasFields) { + const mappings = Object.entries(detectedFields) + .map(([role, field]) => ` ${role}: item${String(field).split('.').map(p => `?.${p}`).join('')}`) + .join(',\n'); + mapCode = `.map((item) => ({\n${mappings}\n }))`; + } + + return [ + '(async () => {', + ` const res = await fetch('${url}', {`, + ` credentials: 'include'`, + ' });', + ' const data = await res.json();', + ` return (data${pathChain} || [])${mapCode};`, + '})()\n', + ].join('\n'); +} + +// ── YAML pipeline generation ─────────────────────────────────────────────── + function buildCandidateYaml(site: string, manifest: any, cap: any, endpoint: any): { name: string; yaml: any } { const needsBrowser = cap.strategy !== 'public'; const pipeline: any[] = []; + const templatedUrl = buildTemplatedUrl(endpoint?.url ?? manifest.target_url, cap, endpoint); + + let domain = ''; + try { domain = new URL(manifest.target_url).hostname; } catch {} - // Step 1: Navigate (if browser-based) if (needsBrowser) { + // Browser-based: navigate + evaluate (like bilibili/hot.yaml, twitter/trending.yaml) pipeline.push({ navigate: manifest.target_url }); + const itemPath = cap.itemPath ?? 'data.data.list'; + pipeline.push({ evaluate: buildEvaluateScript(templatedUrl, itemPath, endpoint) }); + } else { + // Public API: direct fetch (like hackernews/top.yaml) + pipeline.push({ fetch: { url: templatedUrl } }); + if (cap.itemPath) pipeline.push({ select: cap.itemPath }); } - // Step 2: Fetch the API — build a clean URL with templates - const rawUrl = endpoint?.url ?? manifest.target_url; - const fetchStep: any = { url: buildTemplatedUrl(rawUrl, cap, endpoint) }; - - pipeline.push({ fetch: fetchStep }); - - // Step 3: Select the item path - if (cap.itemPath) { - pipeline.push({ select: cap.itemPath }); - } - - // Step 4: Map fields to columns + // Map fields const mapStep: Record = {}; const columns = cap.recommendedColumns ?? ['title', 'url']; - - // Add a rank column if not doing search - if (!cap.recommendedArgs?.some((a: any) => a.name === 'keyword')) { - mapStep['rank'] = '${{ index + 1 }}'; - } - - // Build field mappings from the endpoint's detected fields + if (!cap.recommendedArgs?.some((a: any) => a.name === 'keyword')) mapStep['rank'] = '${{ index + 1 }}'; const detectedFields = endpoint?.detectedFields ?? {}; for (const col of columns) { const fieldPath = detectedFields[col]; - if (fieldPath) { - mapStep[col] = `\${{ item.${fieldPath} }}`; - } else { - mapStep[col] = `\${{ item.${col} }}`; - } + mapStep[col] = fieldPath ? `\${{ item.${fieldPath} }}` : `\${{ item.${col} }}`; } - pipeline.push({ map: mapStep }); - - // Step 5: Limit pipeline.push({ limit: '${{ args.limit | default(20) }}' }); - // Build args definition + // Args const argsDef: Record = {}; for (const arg of cap.recommendedArgs ?? []) { const def: any = { type: arg.type ?? 'str' }; @@ -172,39 +169,26 @@ function buildCandidateYaml(site: string, manifest: any, cap: any, endpoint: any else if (arg.name === 'page') def.description = 'Page number'; argsDef[arg.name] = def; } - - // Ensure limit arg always exists - if (!argsDef['limit']) { - argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' }; - } - - const allColumns = Object.keys(mapStep); + if (!argsDef['limit']) argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' }; return { name: cap.name, yaml: { - site, - name: cap.name, - description: `${site} ${cap.name} (auto-generated)`, - domain: manifest.final_url ? new URL(manifest.final_url).hostname : undefined, - strategy: cap.strategy, - browser: needsBrowser, - args: argsDef, - pipeline, - columns: allColumns, + site, name: cap.name, description: `${cap.description || site + ' ' + cap.name} (auto-generated)`, + domain, strategy: cap.strategy, browser: needsBrowser, + args: argsDef, pipeline, columns: Object.keys(mapStep), }, }; } -export function renderSynthesizeSummary(r: any): string { - const lines = [ - 'opencli synthesize: OK', - `Site: ${r.site}`, - `Source: ${r.explore_dir}`, - `Candidates: ${r.candidate_count}`, - ]; - for (const c of r.candidates ?? []) { - lines.push(` • ${c.name} (${c.strategy}, ${(c.confidence * 100).toFixed(0)}% confidence) → ${c.path}`); - } - return lines.join('\n'); +/** Backward-compatible export for scaffold.ts */ +export function buildCandidate(site: string, targetUrl: string, cap: any, endpoint: any): any { + // Map old-style field names to new ones + const normalizedCap = { + ...cap, + recommendedArgs: cap.recommendedArgs ?? cap.recommended_args, + recommendedColumns: cap.recommendedColumns ?? cap.recommended_columns, + }; + const manifest = { target_url: targetUrl, final_url: targetUrl }; + return buildCandidateYaml(site, manifest, normalizedCap, endpoint); }