feat: Deep Explore + Strategy Cascade + adapter migration
- Rewrite explore.ts: network interception, response body analysis, endpoint scoring, capability inference, framework detection - Rewrite synthesize.ts: evaluate-based YAML generation, buildTemplatedUrl, volatile param stripping - Add cascade.ts: automatic strategy probe (PUBLIC→COOKIE→HEADER) - Upgrade zhihu/hot.yaml: better API with heat/answers - Add zhihu/search.yaml, zhihu/question.ts, reddit/hot.yaml - Migrate bilibili adapters: favorite, feed, history, user-videos - 22 commands across 12 sites
This commit is contained in:
jackwener committed
1 parent
c019f8c6f2
commit
36c4a3a480
7 files changed
+517
-623
No files matched your search
+1
-1
@@ -16,4 +16,4 @@ import './bilibili/user-videos.js';
|
||||
import './github/search.js';
|
||||
|
||||
// zhihu
|
||||
import './zhihu/search.js';
|
||||
import './zhihu/question.js';
|
||||
@@ -0,0 +1,46 @@
|
||||
site: reddit
|
||||
name: hot
|
||||
description: Reddit 热门帖子
|
||||
domain: www.reddit.com
|
||||
|
||||
args:
|
||||
subreddit:
|
||||
type: str
|
||||
default: ""
|
||||
description: "Subreddit name (e.g. programming). Empty for frontpage"
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: Number of posts
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.reddit.com
|
||||
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const sub = '${{ args.subreddit }}';
|
||||
const path = sub ? '/r/' + sub + '/hot.json' : '/hot.json';
|
||||
const res = await fetch(path + '?limit=${{ args.limit }}&raw_json=1', {
|
||||
credentials: 'include'
|
||||
});
|
||||
const d = await res.json();
|
||||
return (d?.data?.children || []).map(c => ({
|
||||
title: c.data.title,
|
||||
subreddit: c.data.subreddit_name_prefixed,
|
||||
score: c.data.score,
|
||||
comments: c.data.num_comments,
|
||||
author: c.data.author,
|
||||
url: 'https://www.reddit.com' + c.data.permalink,
|
||||
}));
|
||||
})()
|
||||
|
||||
- map:
|
||||
rank: ${{ index + 1 }}
|
||||
title: ${{ item.title }}
|
||||
subreddit: ${{ item.subreddit }}
|
||||
score: ${{ item.score }}
|
||||
comments: ${{ item.comments }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [rank, title, subreddit, score, comments]
|
||||
+22
-8
@@ -12,17 +12,31 @@ args:
|
||||
pipeline:
|
||||
- navigate: https://www.zhihu.com
|
||||
|
||||
- fetch:
|
||||
url: https://www.zhihu.com/api/v4/search/top_search
|
||||
params:
|
||||
limit: ${{ args.limit }}
|
||||
|
||||
- select: top_search.words
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', {
|
||||
credentials: 'include'
|
||||
});
|
||||
const d = await res.json();
|
||||
return (d?.data || []).map((item) => {
|
||||
const t = item.target || {};
|
||||
return {
|
||||
title: t.title,
|
||||
url: 'https://www.zhihu.com/question/' + t.id,
|
||||
answer_count: t.answer_count,
|
||||
follower_count: t.follower_count,
|
||||
heat: item.detail_text || '',
|
||||
};
|
||||
});
|
||||
})()
|
||||
|
||||
- map:
|
||||
rank: ${{ index + 1 }}
|
||||
title: ${{ item.query }}
|
||||
title: ${{ item.title }}
|
||||
heat: ${{ item.heat }}
|
||||
answers: ${{ item.answer_count }}
|
||||
url: ${{ item.url }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [rank, title]
|
||||
columns: [rank, title, heat, answers]
|
||||
@@ -0,0 +1,45 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'zhihu',
|
||||
name: 'question',
|
||||
description: '知乎问题详情和回答',
|
||||
domain: 'www.zhihu.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'id', required: true, help: 'Question ID (numeric)' },
|
||||
{ name: 'limit', type: 'int', default: 5, help: 'Number of answers' },
|
||||
],
|
||||
columns: ['rank', 'author', 'votes', 'content'],
|
||||
func: async (page, kwargs) => {
|
||||
const { id, limit = 5 } = kwargs;
|
||||
|
||||
const stripHtml = (html: string) =>
|
||||
(html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').trim();
|
||||
|
||||
// Fetch question detail and answers in parallel via evaluate
|
||||
const result = await page.evaluate(`
|
||||
async () => {
|
||||
const [qResp, aResp] = await Promise.all([
|
||||
fetch('https://www.zhihu.com/api/v4/questions/${id}?include=data[*].detail,excerpt,answer_count,follower_count,visit_count', {credentials: 'include'}),
|
||||
fetch('https://www.zhihu.com/api/v4/questions/${id}/answers?limit=${limit}&offset=0&sort_by=default&include=data[*].content,voteup_count,comment_count,author', {credentials: 'include'})
|
||||
]);
|
||||
if (!qResp.ok || !aResp.ok) return { error: true };
|
||||
const q = await qResp.json();
|
||||
const a = await aResp.json();
|
||||
return { question: q, answers: a.data || [] };
|
||||
}
|
||||
`);
|
||||
|
||||
if (!result || result.error) throw new Error('Failed to fetch question. Are you logged in?');
|
||||
|
||||
const answers = (result.answers ?? []).slice(0, Number(limit)).map((a: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
author: a.author?.name ?? 'anonymous',
|
||||
votes: a.voteup_count ?? 0,
|
||||
content: stripHtml(a.content ?? '').slice(0, 200),
|
||||
}));
|
||||
|
||||
return answers;
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
site: zhihu
|
||||
name: search
|
||||
description: 知乎搜索
|
||||
domain: www.zhihu.com
|
||||
|
||||
args:
|
||||
keyword:
|
||||
type: str
|
||||
required: true
|
||||
description: Search keyword
|
||||
limit:
|
||||
type: int
|
||||
default: 10
|
||||
description: Number of results
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.zhihu.com
|
||||
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').replace(/<em>/g, '').replace(/<\/em>/g, '').trim();
|
||||
const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent('${{ args.keyword }}') + '&t=general&offset=0&limit=${{ args.limit }}', {
|
||||
credentials: 'include'
|
||||
});
|
||||
const d = await res.json();
|
||||
return (d?.data || [])
|
||||
.filter(item => item.type === 'search_result')
|
||||
.map(item => {
|
||||
const obj = item.object || {};
|
||||
const q = obj.question || {};
|
||||
return {
|
||||
type: obj.type,
|
||||
title: strip(obj.title || q.name || ''),
|
||||
excerpt: strip(obj.excerpt || '').substring(0, 100),
|
||||
author: obj.author?.name || '',
|
||||
votes: obj.voteup_count || 0,
|
||||
url: obj.type === 'answer'
|
||||
? 'https://www.zhihu.com/question/' + q.id + '/answer/' + obj.id
|
||||
: obj.type === 'article'
|
||||
? 'https://zhuanlan.zhihu.com/p/' + obj.id
|
||||
: 'https://www.zhihu.com/question/' + obj.id,
|
||||
};
|
||||
});
|
||||
})()
|
||||
|
||||
- map:
|
||||
rank: ${{ index + 1 }}
|
||||
title: ${{ item.title }}
|
||||
type: ${{ item.type }}
|
||||
author: ${{ item.author }}
|
||||
votes: ${{ item.votes }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [rank, title, type, author, votes]
|
||||
+217
-467
@@ -1,208 +1,128 @@
|
||||
/**
|
||||
* Deep Explore: intelligent API discovery with response analysis.
|
||||
*
|
||||
* Unlike simple page snapshots, Deep Explore intercepts network traffic,
|
||||
* analyzes response schemas, and automatically infers capabilities that
|
||||
* can be turned into CLI commands.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Navigate to target URL
|
||||
* 2. Auto-scroll to trigger lazy loading
|
||||
* 3. Capture network requests (with body analysis)
|
||||
* 4. For each JSON response: detect list fields, infer columns, analyze auth
|
||||
* 5. Detect frontend framework (Vue/React/Pinia/Next.js)
|
||||
* 6. Generate structured capabilities.json
|
||||
* Navigates to the target URL, auto-scrolls to trigger lazy loading,
|
||||
* captures network traffic, analyzes JSON responses, and automatically
|
||||
* infers CLI capabilities from discovered API endpoints.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { browserSession, DEFAULT_BROWSER_EXPLORE_TIMEOUT, runWithTimeout } from './runtime.js';
|
||||
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
|
||||
|
||||
// ── Site name detection ────────────────────────────────────────────────────
|
||||
|
||||
const KNOWN_ALIASES: Record<string, string> = {
|
||||
const KNOWN_SITE_ALIASES: Record<string, string> = {
|
||||
'x.com': 'twitter', 'twitter.com': 'twitter',
|
||||
'news.ycombinator.com': 'hackernews',
|
||||
'www.zhihu.com': 'zhihu', 'www.bilibili.com': 'bilibili',
|
||||
'search.bilibili.com': 'bilibili',
|
||||
'www.v2ex.com': 'v2ex', 'www.reddit.com': 'reddit',
|
||||
'www.xiaohongshu.com': 'xiaohongshu', 'www.douban.com': 'douban',
|
||||
'www.weibo.com': 'weibo', 'search.bilibili.com': 'bilibili',
|
||||
'www.weibo.com': 'weibo', 'www.bbc.com': 'bbc',
|
||||
};
|
||||
|
||||
function detectSiteName(url: string): string {
|
||||
export function detectSiteName(url: string): string {
|
||||
try {
|
||||
const host = new URL(url).hostname.toLowerCase();
|
||||
if (host in KNOWN_ALIASES) return KNOWN_ALIASES[host];
|
||||
if (host in KNOWN_SITE_ALIASES) return KNOWN_SITE_ALIASES[host];
|
||||
const parts = host.split('.').filter(p => p && p !== 'www');
|
||||
if (parts.length >= 2) {
|
||||
if (['uk', 'jp', 'cn', 'com'].includes(parts[parts.length - 1]) && parts.length >= 3) {
|
||||
return parts[parts.length - 3].replace(/[^a-z0-9]/g, '');
|
||||
return slugify(parts[parts.length - 3]);
|
||||
}
|
||||
return parts[parts.length - 2].replace(/[^a-z0-9]/g, '');
|
||||
return slugify(parts[parts.length - 2]);
|
||||
}
|
||||
return parts[0]?.replace(/[^a-z0-9]/g, '') ?? 'site';
|
||||
return parts[0] ? slugify(parts[0]) : 'site';
|
||||
} catch { return 'site'; }
|
||||
}
|
||||
|
||||
export function slugify(value: string): string {
|
||||
return value.trim().toLowerCase().replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '') || 'site';
|
||||
}
|
||||
|
||||
// ── Field & capability inference ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Common field names grouped by semantic role.
|
||||
* Used to auto-detect which response fields map to which columns.
|
||||
*/
|
||||
const FIELD_ROLES: Record<string, string[]> = {
|
||||
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
|
||||
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'short_link', 'share_url'],
|
||||
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
|
||||
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'stat', 'play', 'favorite_count', 'reply_count'],
|
||||
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
|
||||
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
|
||||
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
|
||||
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
|
||||
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
|
||||
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
|
||||
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
|
||||
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
|
||||
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
|
||||
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
|
||||
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
|
||||
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
|
||||
};
|
||||
|
||||
/** Param names that indicate searchable APIs */
|
||||
const SEARCH_PARAMS = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w']);
|
||||
/** Param names that indicate pagination */
|
||||
const PAGINATION_PARAMS = new Set(['page', 'pn', 'offset', 'cursor', 'next', 'page_num']);
|
||||
/** Param names that indicate limit control */
|
||||
const LIMIT_PARAMS = new Set(['limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num']);
|
||||
/** Content types to ignore */
|
||||
const IGNORED_CONTENT_TYPES = new Set(['image/', 'font/', 'text/css', 'text/javascript', 'application/javascript', 'application/wasm']);
|
||||
/** Volatile query params to strip from patterns */
|
||||
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign']);
|
||||
|
||||
// ── Network analysis ───────────────────────────────────────────────────────
|
||||
|
||||
interface NetworkEntry {
|
||||
method: string;
|
||||
url: string;
|
||||
status: number | null;
|
||||
contentType: string;
|
||||
responseBody?: any;
|
||||
requestHeaders?: Record<string, string>;
|
||||
queryParams?: Record<string, string>;
|
||||
method: string; url: string; status: number | null;
|
||||
contentType: string; responseBody?: any; requestHeaders?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface AnalyzedEndpoint {
|
||||
pattern: string;
|
||||
method: string;
|
||||
url: string;
|
||||
status: number | null;
|
||||
contentType: string;
|
||||
queryParams: string[];
|
||||
hasSearchParam: boolean;
|
||||
hasPaginationParam: boolean;
|
||||
hasLimitParam: boolean;
|
||||
pattern: string; method: string; url: string; status: number | null;
|
||||
contentType: string; queryParams: string[]; score: number;
|
||||
hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean;
|
||||
authIndicators: string[];
|
||||
responseAnalysis: ResponseAnalysis | null;
|
||||
}
|
||||
|
||||
interface ResponseAnalysis {
|
||||
itemPath: string | null;
|
||||
itemCount: number;
|
||||
detectedFields: Record<string, string>; // role → actual field name
|
||||
sampleFieldNames: string[];
|
||||
responseAnalysis: { itemPath: string | null; itemCount: number; detectedFields: Record<string, string>; sampleFields: string[] } | null;
|
||||
}
|
||||
|
||||
interface InferredCapability {
|
||||
name: string;
|
||||
description: string;
|
||||
strategy: string;
|
||||
confidence: number;
|
||||
endpoint: string;
|
||||
itemPath: string | null;
|
||||
name: string; description: string; strategy: string; confidence: number;
|
||||
endpoint: string; itemPath: string | null;
|
||||
recommendedColumns: string[];
|
||||
recommendedArgs: Array<{ name: string; type: string; required: boolean; default?: any }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse raw network output from Playwright MCP into structured entries.
|
||||
* Handles both text format ([GET] url => [200]) and structured JSON.
|
||||
* Parse raw network output from Playwright MCP.
|
||||
* Handles text format: [GET] url => [200]
|
||||
*/
|
||||
function parseNetworkOutput(raw: any): NetworkEntry[] {
|
||||
function parseNetworkRequests(raw: any): NetworkEntry[] {
|
||||
if (typeof raw === 'string') {
|
||||
// Playwright MCP returns network as text lines like:
|
||||
// "[GET] https://api.example.com/xxx => [200] "
|
||||
// May also have markdown headers like "### Result"
|
||||
const entries: NetworkEntry[] = [];
|
||||
const lines = raw.split('\n').filter((l: string) => l.trim());
|
||||
for (const line of lines) {
|
||||
// Format: [METHOD] URL => [STATUS] optional_extra
|
||||
const bracketMatch = line.match(/^\[?(GET|POST|PUT|DELETE|PATCH|OPTIONS)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
|
||||
if (bracketMatch) {
|
||||
const [, method, url, status] = bracketMatch;
|
||||
for (const line of raw.split('\n')) {
|
||||
// Format: [GET] URL => [200]
|
||||
const m = line.match(/\[?(GET|POST|PUT|DELETE|PATCH|OPTIONS)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
|
||||
if (m) {
|
||||
const [, method, url, status] = m;
|
||||
entries.push({
|
||||
method: method.toUpperCase(),
|
||||
url,
|
||||
status: status ? parseInt(status) : null,
|
||||
contentType: url.endsWith('.json') ? 'application/json' :
|
||||
(url.includes('/api/') || url.includes('/x/')) ? 'application/json' : '',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
// Legacy format: GET url → 200 (application/json)
|
||||
const legacyMatch = line.match(/^(GET|POST|PUT|DELETE|PATCH|OPTIONS)\s+(\S+)\s*→?\s*(\d+)?\s*(?:\(([^)]*)\))?/i);
|
||||
if (legacyMatch) {
|
||||
const [, method, url, status, ct] = legacyMatch;
|
||||
entries.push({
|
||||
method: method.toUpperCase(),
|
||||
url,
|
||||
status: status ? parseInt(status) : null,
|
||||
contentType: ct ?? '',
|
||||
method: method.toUpperCase(), url, status: status ? parseInt(status) : null,
|
||||
contentType: (url.includes('/api/') || url.includes('/x/') || url.endsWith('.json')) ? 'application/json' : '',
|
||||
});
|
||||
}
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
if (Array.isArray(raw)) {
|
||||
return raw.map((e: any) => ({
|
||||
return raw.filter(e => e && typeof e === 'object').map(e => ({
|
||||
method: (e.method ?? 'GET').toUpperCase(),
|
||||
url: e.url ?? e.request?.url ?? '',
|
||||
url: String(e.url ?? e.request?.url ?? e.requestUrl ?? ''),
|
||||
status: e.status ?? e.statusCode ?? null,
|
||||
contentType: e.contentType ?? e.mimeType ?? '',
|
||||
responseBody: e.responseBody ?? e.body,
|
||||
requestHeaders: e.requestHeaders ?? e.headers,
|
||||
contentType: e.contentType ?? e.response?.contentType ?? '',
|
||||
responseBody: e.responseBody, requestHeaders: e.requestHeaders,
|
||||
}));
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Normalize a URL into a pattern by replacing IDs with placeholders.
|
||||
*/
|
||||
function urlToPattern(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const pathNorm = parsed.pathname
|
||||
.replace(/\/\d+/g, '/{id}')
|
||||
.replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}')
|
||||
.replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}');
|
||||
const p = new URL(url);
|
||||
const pathNorm = p.pathname.replace(/\/\d+/g, '/{id}').replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}').replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}');
|
||||
const params: string[] = [];
|
||||
parsed.searchParams.forEach((_v, k) => {
|
||||
if (!VOLATILE_PARAMS.has(k)) params.push(k);
|
||||
});
|
||||
const qs = params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : '';
|
||||
return `${parsed.host}${pathNorm}${qs}`;
|
||||
p.searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) params.push(k); });
|
||||
return `${p.host}${pathNorm}${params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''}`;
|
||||
} catch { return url; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract query params from a URL.
|
||||
*/
|
||||
function extractQueryParams(url: string): Record<string, string> {
|
||||
try {
|
||||
const params: Record<string, string> = {};
|
||||
new URL(url).searchParams.forEach((v, k) => { params[k] = v; });
|
||||
return params;
|
||||
} catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect auth indicators from request headers.
|
||||
*/
|
||||
function detectAuthIndicators(headers?: Record<string, string>): string[] {
|
||||
if (!headers) return [];
|
||||
const indicators: string[] = [];
|
||||
@@ -210,435 +130,257 @@ function detectAuthIndicators(headers?: Record<string, string>): string[] {
|
||||
if (keys.some(k => k === 'authorization')) indicators.push('bearer');
|
||||
if (keys.some(k => k.startsWith('x-csrf') || k.startsWith('x-xsrf'))) indicators.push('csrf');
|
||||
if (keys.some(k => k.startsWith('x-s') || k === 'x-t' || k === 'x-s-common')) indicators.push('signature');
|
||||
if (keys.some(k => k === 'x-client-transaction-id')) indicators.push('transaction');
|
||||
return indicators;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a JSON response to find list data and field mappings.
|
||||
*/
|
||||
function analyzeResponseBody(body: any): ResponseAnalysis | null {
|
||||
function analyzeResponseBody(body: any): AnalyzedEndpoint['responseAnalysis'] {
|
||||
if (!body || typeof body !== 'object') return null;
|
||||
|
||||
// Try to find the main list in the response
|
||||
const candidates: Array<{ path: string; items: any[] }> = [];
|
||||
|
||||
function findArrays(obj: any, currentPath: string, depth: number) {
|
||||
function findArrays(obj: any, path: string, depth: number) {
|
||||
if (depth > 4) return;
|
||||
if (Array.isArray(obj) && obj.length >= 2) {
|
||||
// Check if items are objects (not primitive arrays)
|
||||
if (obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) {
|
||||
candidates.push({ path: currentPath, items: obj });
|
||||
}
|
||||
if (Array.isArray(obj) && obj.length >= 2 && obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) {
|
||||
candidates.push({ path, items: obj });
|
||||
}
|
||||
if (obj && typeof obj === 'object' && !Array.isArray(obj)) {
|
||||
for (const [key, val] of Object.entries(obj)) {
|
||||
const nextPath = currentPath ? `${currentPath}.${key}` : key;
|
||||
findArrays(val, nextPath, depth + 1);
|
||||
}
|
||||
for (const [key, val] of Object.entries(obj)) findArrays(val, path ? `${path}.${key}` : key, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
findArrays(body, '', 0);
|
||||
if (!candidates.length) return null;
|
||||
|
||||
// Pick the largest array as the main list
|
||||
candidates.sort((a, b) => b.items.length - a.items.length);
|
||||
const best = candidates[0];
|
||||
const sample = best.items[0];
|
||||
const sampleFields = sample && typeof sample === 'object' ? flattenFields(sample, '', 2) : [];
|
||||
|
||||
// Analyze field names in the first item
|
||||
const sampleItem = best.items[0];
|
||||
const sampleFieldNames = sampleItem && typeof sampleItem === 'object'
|
||||
? flattenFieldNames(sampleItem, '', 2)
|
||||
: [];
|
||||
|
||||
// Match fields to semantic roles
|
||||
const detectedFields: Record<string, string> = {};
|
||||
for (const [role, aliases] of Object.entries(FIELD_ROLES)) {
|
||||
for (const fieldName of sampleFieldNames) {
|
||||
const basename = fieldName.split('.').pop()?.toLowerCase() ?? '';
|
||||
if (aliases.includes(basename)) {
|
||||
detectedFields[role] = fieldName;
|
||||
break;
|
||||
}
|
||||
for (const f of sampleFields) {
|
||||
if (aliases.includes(f.split('.').pop()?.toLowerCase() ?? '')) { detectedFields[role] = f; break; }
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
itemPath: best.path || null,
|
||||
itemCount: best.items.length,
|
||||
detectedFields,
|
||||
sampleFieldNames,
|
||||
};
|
||||
return { itemPath: best.path || null, itemCount: best.items.length, detectedFields, sampleFields };
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten nested object field names for analysis.
|
||||
*/
|
||||
function flattenFieldNames(obj: any, prefix: string, maxDepth: number): string[] {
|
||||
function flattenFields(obj: any, prefix: string, maxDepth: number): string[] {
|
||||
if (maxDepth <= 0 || !obj || typeof obj !== 'object') return [];
|
||||
const names: string[] = [];
|
||||
for (const key of Object.keys(obj)) {
|
||||
const fullKey = prefix ? `${prefix}.${key}` : key;
|
||||
names.push(fullKey);
|
||||
if (obj[key] && typeof obj[key] === 'object' && !Array.isArray(obj[key])) {
|
||||
names.push(...flattenFieldNames(obj[key], fullKey, maxDepth - 1));
|
||||
}
|
||||
const full = prefix ? `${prefix}.${key}` : key;
|
||||
names.push(full);
|
||||
if (obj[key] && typeof obj[key] === 'object' && !Array.isArray(obj[key])) names.push(...flattenFields(obj[key], full, maxDepth - 1));
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a list of network entries into structured endpoints.
|
||||
*/
|
||||
function analyzeEndpoints(entries: NetworkEntry[], siteHost: string): AnalyzedEndpoint[] {
|
||||
const seen = new Map<string, AnalyzedEndpoint>();
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.url) continue;
|
||||
|
||||
// Skip static resources
|
||||
const ct = entry.contentType.toLowerCase();
|
||||
if (IGNORED_CONTENT_TYPES.has(ct.split(';')[0]?.trim() ?? '') ||
|
||||
ct.includes('image/') || ct.includes('font/') || ct.includes('css') ||
|
||||
ct.includes('javascript') || ct.includes('wasm')) continue;
|
||||
|
||||
// Skip non-JSON and failed responses
|
||||
if (entry.status && entry.status >= 400) continue;
|
||||
|
||||
const pattern = urlToPattern(entry.url);
|
||||
const queryParams = extractQueryParams(entry.url);
|
||||
const paramNames = Object.keys(queryParams).filter(k => !VOLATILE_PARAMS.has(k));
|
||||
|
||||
const key = `${entry.method}:${pattern}`;
|
||||
if (seen.has(key)) continue;
|
||||
|
||||
const endpoint: AnalyzedEndpoint = {
|
||||
pattern,
|
||||
method: entry.method,
|
||||
url: entry.url,
|
||||
status: entry.status,
|
||||
contentType: ct,
|
||||
queryParams: paramNames,
|
||||
hasSearchParam: paramNames.some(p => SEARCH_PARAMS.has(p)),
|
||||
hasPaginationParam: paramNames.some(p => PAGINATION_PARAMS.has(p)),
|
||||
hasLimitParam: paramNames.some(p => LIMIT_PARAMS.has(p)),
|
||||
authIndicators: detectAuthIndicators(entry.requestHeaders),
|
||||
responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
|
||||
};
|
||||
|
||||
seen.set(key, endpoint);
|
||||
}
|
||||
|
||||
return [...seen.values()];
|
||||
function scoreEndpoint(ep: { contentType: string; responseAnalysis: any; pattern: string; status: number | null; hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean }): number {
|
||||
let s = 0;
|
||||
if (ep.contentType.includes('json')) s += 10;
|
||||
if (ep.responseAnalysis) { s += 5; s += Math.min(ep.responseAnalysis.itemCount, 10); s += Object.keys(ep.responseAnalysis.detectedFields).length * 2; }
|
||||
if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/')) s += 3;
|
||||
if (ep.hasSearchParam) s += 3;
|
||||
if (ep.hasPaginationParam) s += 2;
|
||||
if (ep.hasLimitParam) s += 2;
|
||||
if (ep.status === 200) s += 2;
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer what strategy to use based on endpoint analysis.
|
||||
*/
|
||||
function inferStrategy(endpoint: AnalyzedEndpoint): string {
|
||||
if (endpoint.authIndicators.includes('signature')) return 'intercept';
|
||||
if (endpoint.authIndicators.includes('transaction')) return 'header';
|
||||
if (endpoint.authIndicators.includes('bearer') || endpoint.authIndicators.includes('csrf')) return 'header';
|
||||
// Check if the URL is a public API (no auth indicators)
|
||||
if (endpoint.authIndicators.length === 0) {
|
||||
// If it's the same domain, likely cookie auth
|
||||
return 'cookie';
|
||||
}
|
||||
return 'cookie';
|
||||
}
|
||||
|
||||
/**
|
||||
* Infer the capability name from an endpoint pattern.
|
||||
*/
|
||||
function inferCapabilityName(endpoint: AnalyzedEndpoint, goal?: string): string {
|
||||
function inferCapabilityName(url: string, goal?: string): string {
|
||||
if (goal) return goal;
|
||||
|
||||
const u = endpoint.url.toLowerCase();
|
||||
const p = endpoint.pattern.toLowerCase();
|
||||
|
||||
// Match common patterns
|
||||
if (endpoint.hasSearchParam) return 'search';
|
||||
const u = url.toLowerCase();
|
||||
if (u.includes('hot') || u.includes('popular') || u.includes('ranking') || u.includes('trending')) return 'hot';
|
||||
if (u.includes('search')) return 'search';
|
||||
if (u.includes('feed') || u.includes('timeline') || u.includes('dynamic')) return 'feed';
|
||||
if (u.includes('comment') || u.includes('reply')) return 'comments';
|
||||
if (u.includes('history')) return 'history';
|
||||
if (u.includes('profile') || u.includes('userinfo') || u.includes('/me') || u.includes('myinfo')) return 'me';
|
||||
if (u.includes('video') || u.includes('article') || u.includes('detail') || u.includes('view')) return 'detail';
|
||||
if (u.includes('profile') || u.includes('userinfo') || u.includes('/me')) return 'me';
|
||||
if (u.includes('favorite') || u.includes('collect') || u.includes('bookmark')) return 'favorite';
|
||||
if (u.includes('notification') || u.includes('notice')) return 'notifications';
|
||||
|
||||
// Fallback: try to extract from path
|
||||
try {
|
||||
const pathname = new URL(endpoint.url).pathname;
|
||||
const segments = pathname.split('/').filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i));
|
||||
if (segments.length) return segments[segments.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
||||
const segs = new URL(url).pathname.split('/').filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i));
|
||||
if (segs.length) return segs[segs.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
||||
} catch {}
|
||||
|
||||
return 'data';
|
||||
}
|
||||
|
||||
/**
|
||||
* Build recommended columns from response analysis.
|
||||
*/
|
||||
function buildRecommendedColumns(analysis: ResponseAnalysis | null): string[] {
|
||||
if (!analysis) return ['title', 'url'];
|
||||
const cols: string[] = [];
|
||||
// Prioritize: title → url → author → score → time
|
||||
const priority = ['title', 'url', 'author', 'score', 'time'];
|
||||
for (const role of priority) {
|
||||
if (analysis.detectedFields[role]) cols.push(role);
|
||||
}
|
||||
return cols.length ? cols : ['title', 'url'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build recommended args from endpoint query params.
|
||||
*/
|
||||
function buildRecommendedArgs(endpoint: AnalyzedEndpoint): InferredCapability['recommendedArgs'] {
|
||||
const args: InferredCapability['recommendedArgs'] = [];
|
||||
|
||||
if (endpoint.hasSearchParam) {
|
||||
const paramName = endpoint.queryParams.find(p => SEARCH_PARAMS.has(p)) ?? 'keyword';
|
||||
args.push({ name: 'keyword', type: 'str', required: true });
|
||||
}
|
||||
|
||||
// Always add limit
|
||||
args.push({ name: 'limit', type: 'int', required: false, default: 20 });
|
||||
|
||||
if (endpoint.hasPaginationParam) {
|
||||
args.push({ name: 'page', type: 'int', required: false, default: 1 });
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score an endpoint's interest level for capability generation.
|
||||
* Higher score = more likely to be a useful API endpoint.
|
||||
*/
|
||||
function scoreEndpoint(ep: AnalyzedEndpoint): number {
|
||||
let score = 0;
|
||||
// JSON content type is strongly preferred
|
||||
if (ep.contentType.includes('json')) score += 10;
|
||||
// Has response analysis with items
|
||||
if (ep.responseAnalysis) {
|
||||
score += 5;
|
||||
score += Math.min(ep.responseAnalysis.itemCount, 10);
|
||||
score += Object.keys(ep.responseAnalysis.detectedFields).length * 2;
|
||||
}
|
||||
// API-like path patterns
|
||||
if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/')) score += 3;
|
||||
// Has search/pagination params
|
||||
if (ep.hasSearchParam) score += 3;
|
||||
if (ep.hasPaginationParam) score += 2;
|
||||
if (ep.hasLimitParam) score += 2;
|
||||
// 200 OK
|
||||
if (ep.status === 200) score += 2;
|
||||
return score;
|
||||
function inferStrategy(authIndicators: string[]): string {
|
||||
if (authIndicators.includes('signature')) return 'intercept';
|
||||
if (authIndicators.includes('bearer') || authIndicators.includes('csrf')) return 'header';
|
||||
return 'cookie';
|
||||
}
|
||||
|
||||
// ── Framework detection ────────────────────────────────────────────────────
|
||||
|
||||
const FRAMEWORK_DETECT_JS = `
|
||||
(() => {
|
||||
const result = {};
|
||||
try {
|
||||
const app = document.querySelector('#app');
|
||||
result.vue3 = !!(app && app.__vue_app__);
|
||||
result.vue2 = !!(app && app.__vue__);
|
||||
result.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]');
|
||||
result.nextjs = !!window.__NEXT_DATA__;
|
||||
result.nuxt = !!window.__NUXT__;
|
||||
if (result.vue3 && app.__vue_app__) {
|
||||
const gp = app.__vue_app__.config?.globalProperties;
|
||||
result.pinia = !!(gp && gp.$pinia);
|
||||
result.vuex = !!(gp && gp.$store);
|
||||
}
|
||||
} catch {}
|
||||
return JSON.stringify(result);
|
||||
})()
|
||||
() => {
|
||||
const r = {};
|
||||
try {
|
||||
const app = document.querySelector('#app');
|
||||
r.vue3 = !!(app && app.__vue_app__);
|
||||
r.vue2 = !!(app && app.__vue__);
|
||||
r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]');
|
||||
r.nextjs = !!window.__NEXT_DATA__;
|
||||
r.nuxt = !!window.__NUXT__;
|
||||
if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); }
|
||||
} catch {}
|
||||
return r;
|
||||
}
|
||||
`;
|
||||
|
||||
// ── Main explore function ──────────────────────────────────────────────────
|
||||
|
||||
export async function exploreUrl(url: string, opts: any = {}): Promise<any> {
|
||||
const site = opts.site ?? detectSiteName(url);
|
||||
const outDir = opts.outDir ?? path.join('.opencli', 'explore', site);
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
export async function exploreUrl(
|
||||
url: string,
|
||||
opts: {
|
||||
BrowserFactory: new () => any;
|
||||
site?: string; goal?: string; authenticated?: boolean;
|
||||
outDir?: string; waitSeconds?: number; query?: string;
|
||||
clickLabels?: string[]; auto?: boolean;
|
||||
},
|
||||
): Promise<Record<string, any>> {
|
||||
const waitSeconds = opts.waitSeconds ?? 3.0;
|
||||
const exploreTimeout = Math.max(DEFAULT_BROWSER_EXPLORE_TIMEOUT, 45.0 + waitSeconds * 8.0);
|
||||
|
||||
const result: any = await browserSession(opts.BrowserFactory, async (page: any) => {
|
||||
return browserSession(opts.BrowserFactory, async (page) => {
|
||||
return runWithTimeout((async () => {
|
||||
// Step 1: Navigate
|
||||
await page.goto(url);
|
||||
await page.wait(opts.waitSeconds ?? 3);
|
||||
await page.wait(waitSeconds);
|
||||
|
||||
// Step 2: Auto-scroll to trigger lazy loading
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await page.scroll('down');
|
||||
await page.wait(1);
|
||||
}
|
||||
// Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist)
|
||||
for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); }
|
||||
|
||||
// Step 3: Capture network traffic
|
||||
// Step 3: Read page metadata
|
||||
const metadata = await readPageMetadata(page);
|
||||
|
||||
// Step 4: Capture network traffic
|
||||
const rawNetwork = await page.networkRequests(false);
|
||||
const networkEntries = parseNetworkOutput(rawNetwork);
|
||||
|
||||
// Step 4: For JSON endpoints, try to fetch response body in-browser
|
||||
const jsonEndpoints = networkEntries.filter(e =>
|
||||
e.contentType.includes('json') && e.method === 'GET' && e.status === 200
|
||||
);
|
||||
const networkEntries = parseNetworkRequests(rawNetwork);
|
||||
|
||||
// Step 5: For JSON endpoints, re-fetch response body in-browser
|
||||
const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200);
|
||||
for (const ep of jsonEndpoints.slice(0, 10)) {
|
||||
// Only fetch body for promising-looking API endpoints
|
||||
if (ep.url.includes('/api/') || ep.url.includes('/x/') || ep.url.includes('/web/') ||
|
||||
ep.contentType.includes('json')) {
|
||||
try {
|
||||
const bodyResult = await page.evaluate(`
|
||||
async () => {
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(ep.url)}, { credentials: 'include' });
|
||||
if (!resp.ok) return null;
|
||||
const data = await resp.json();
|
||||
return JSON.stringify(data).slice(0, 10000);
|
||||
} catch { return null; }
|
||||
}
|
||||
`);
|
||||
if (bodyResult && typeof bodyResult === 'string') {
|
||||
try { ep.responseBody = JSON.parse(bodyResult); } catch {}
|
||||
} else if (bodyResult && typeof bodyResult === 'object') {
|
||||
ep.responseBody = bodyResult;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
try {
|
||||
const body = await page.evaluate(`async () => { try { const r = await fetch(${JSON.stringify(ep.url)}, {credentials:'include'}); if (!r.ok) return null; const d = await r.json(); return JSON.stringify(d).slice(0,10000); } catch { return null; } }`);
|
||||
if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} }
|
||||
else if (body && typeof body === 'object') ep.responseBody = body;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Step 5: Detect frontend framework
|
||||
// Step 6: Detect framework
|
||||
let framework: Record<string, boolean> = {};
|
||||
try {
|
||||
const fwResult = await page.evaluate(FRAMEWORK_DETECT_JS);
|
||||
if (typeof fwResult === 'string') framework = JSON.parse(fwResult);
|
||||
else if (typeof fwResult === 'object') framework = fwResult;
|
||||
} catch {}
|
||||
|
||||
// Step 6: Get page metadata
|
||||
let title = '', finalUrl = '';
|
||||
try {
|
||||
const meta = await page.evaluate(`
|
||||
() => JSON.stringify({ url: window.location.href, title: document.title || '' })
|
||||
`);
|
||||
if (typeof meta === 'string') {
|
||||
const parsed = JSON.parse(meta);
|
||||
title = parsed.title; finalUrl = parsed.url;
|
||||
} else if (typeof meta === 'object') {
|
||||
title = meta.title; finalUrl = meta.url;
|
||||
}
|
||||
} catch {}
|
||||
try { const fw = await page.evaluate(FRAMEWORK_DETECT_JS); if (fw && typeof fw === 'object') framework = fw; } catch {}
|
||||
|
||||
// Step 7: Analyze endpoints
|
||||
let siteHost = '';
|
||||
try { siteHost = new URL(url).hostname; } catch {}
|
||||
const analyzedEndpoints = analyzeEndpoints(networkEntries, siteHost);
|
||||
const seen = new Map<string, AnalyzedEndpoint>();
|
||||
for (const entry of networkEntries) {
|
||||
if (!entry.url) continue;
|
||||
const ct = entry.contentType.toLowerCase();
|
||||
if (ct.includes('image/') || ct.includes('font/') || ct.includes('css') || ct.includes('javascript') || ct.includes('wasm')) continue;
|
||||
if (entry.status && entry.status >= 400) continue;
|
||||
|
||||
// Step 8: Score and rank endpoints
|
||||
const scoredEndpoints = analyzedEndpoints
|
||||
.map(ep => ({ ...ep, score: scoreEndpoint(ep) }))
|
||||
.filter(ep => ep.score >= 5)
|
||||
.sort((a, b) => b.score - a.score);
|
||||
const pattern = urlToPattern(entry.url);
|
||||
const key = `${entry.method}:${pattern}`;
|
||||
if (seen.has(key)) continue;
|
||||
|
||||
// Step 9: Infer capabilities from top endpoints
|
||||
const qp: string[] = [];
|
||||
try { new URL(entry.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {}
|
||||
|
||||
const ep: AnalyzedEndpoint = {
|
||||
pattern, method: entry.method, url: entry.url, status: entry.status, contentType: ct,
|
||||
queryParams: qp, hasSearchParam: qp.some(p => SEARCH_PARAMS.has(p)),
|
||||
hasPaginationParam: qp.some(p => PAGINATION_PARAMS.has(p)),
|
||||
hasLimitParam: qp.some(p => LIMIT_PARAMS.has(p)),
|
||||
authIndicators: detectAuthIndicators(entry.requestHeaders),
|
||||
responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
|
||||
score: 0,
|
||||
};
|
||||
ep.score = scoreEndpoint(ep);
|
||||
seen.set(key, ep);
|
||||
}
|
||||
|
||||
const analyzedEndpoints = [...seen.values()].filter(ep => ep.score >= 5).sort((a, b) => b.score - a.score);
|
||||
|
||||
// Step 8: Infer capabilities
|
||||
const capabilities: InferredCapability[] = [];
|
||||
const usedNames = new Set<string>();
|
||||
|
||||
for (const ep of scoredEndpoints.slice(0, 8)) {
|
||||
let capName = inferCapabilityName(ep, opts.goal);
|
||||
// Deduplicate names
|
||||
for (const ep of analyzedEndpoints.slice(0, 8)) {
|
||||
let capName = inferCapabilityName(ep.url, opts.goal);
|
||||
if (usedNames.has(capName)) {
|
||||
const suffix = ep.pattern.split('/').filter(s => s && !s.startsWith('{') && !s.includes('.')).pop();
|
||||
capName = suffix ? `${capName}_${suffix}` : `${capName}_${usedNames.size}`;
|
||||
}
|
||||
usedNames.add(capName);
|
||||
|
||||
const cols: string[] = [];
|
||||
if (ep.responseAnalysis) {
|
||||
for (const role of ['title', 'url', 'author', 'score', 'time']) {
|
||||
if (ep.responseAnalysis.detectedFields[role]) cols.push(role);
|
||||
}
|
||||
}
|
||||
|
||||
const args: InferredCapability['recommendedArgs'] = [];
|
||||
if (ep.hasSearchParam) args.push({ name: 'keyword', type: 'str', required: true });
|
||||
args.push({ name: 'limit', type: 'int', required: false, default: 20 });
|
||||
if (ep.hasPaginationParam) args.push({ name: 'page', type: 'int', required: false, default: 1 });
|
||||
|
||||
capabilities.push({
|
||||
name: capName,
|
||||
description: `${site} ${capName}`,
|
||||
strategy: inferStrategy(ep),
|
||||
confidence: Math.min(ep.score / 20, 1.0),
|
||||
endpoint: ep.pattern,
|
||||
name: capName, description: `${opts.site ?? detectSiteName(url)} ${capName}`,
|
||||
strategy: inferStrategy(ep.authIndicators),
|
||||
confidence: Math.min(ep.score / 20, 1.0), endpoint: ep.pattern,
|
||||
itemPath: ep.responseAnalysis?.itemPath ?? null,
|
||||
recommendedColumns: buildRecommendedColumns(ep.responseAnalysis),
|
||||
recommendedArgs: buildRecommendedArgs(ep),
|
||||
recommendedColumns: cols.length ? cols : ['title', 'url'],
|
||||
recommendedArgs: args,
|
||||
});
|
||||
}
|
||||
|
||||
// Step 10: Determine auth strategy
|
||||
const allAuthIndicators = new Set(analyzedEndpoints.flatMap(ep => ep.authIndicators));
|
||||
let topStrategy = 'cookie';
|
||||
if (allAuthIndicators.has('signature')) topStrategy = 'intercept';
|
||||
else if (allAuthIndicators.has('transaction') || allAuthIndicators.has('bearer')) topStrategy = 'header';
|
||||
else if (allAuthIndicators.size === 0 && scoredEndpoints.some(ep => ep.contentType.includes('json'))) topStrategy = 'public';
|
||||
// Step 9: Determine overall auth strategy
|
||||
const allAuth = new Set(analyzedEndpoints.flatMap(ep => ep.authIndicators));
|
||||
const topStrategy = allAuth.has('signature') ? 'intercept' : allAuth.has('bearer') || allAuth.has('csrf') ? 'header' : allAuth.size === 0 ? 'public' : 'cookie';
|
||||
|
||||
return {
|
||||
site,
|
||||
target_url: url,
|
||||
final_url: finalUrl,
|
||||
title,
|
||||
framework,
|
||||
top_strategy: topStrategy,
|
||||
endpoint_count: analyzedEndpoints.length,
|
||||
api_endpoint_count: scoredEndpoints.length,
|
||||
capabilities,
|
||||
endpoints: scoredEndpoints.map(ep => ({
|
||||
pattern: ep.pattern,
|
||||
method: ep.method,
|
||||
url: ep.url,
|
||||
status: ep.status,
|
||||
contentType: ep.contentType,
|
||||
score: ep.score,
|
||||
queryParams: ep.queryParams,
|
||||
itemPath: ep.responseAnalysis?.itemPath ?? null,
|
||||
itemCount: ep.responseAnalysis?.itemCount ?? 0,
|
||||
detectedFields: ep.responseAnalysis?.detectedFields ?? {},
|
||||
authIndicators: ep.authIndicators,
|
||||
})),
|
||||
auth_indicators: [...allAuthIndicators],
|
||||
const siteName = opts.site ?? detectSiteName(metadata.url || url);
|
||||
const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName);
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
const result = {
|
||||
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
|
||||
framework, top_strategy: topStrategy,
|
||||
endpoint_count: analyzedEndpoints.length + [...seen.values()].filter(ep => ep.score < 5).length,
|
||||
api_endpoint_count: analyzedEndpoints.length,
|
||||
capabilities, auth_indicators: [...allAuth],
|
||||
};
|
||||
})(), { timeout: DEFAULT_BROWSER_EXPLORE_TIMEOUT, label: 'explore' });
|
||||
|
||||
// Write artifacts
|
||||
fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify({
|
||||
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
|
||||
framework, top_strategy: topStrategy, explored_at: new Date().toISOString(),
|
||||
}, null, 2));
|
||||
fs.writeFileSync(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
|
||||
pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
|
||||
contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
|
||||
itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
|
||||
detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
|
||||
})), null, 2));
|
||||
fs.writeFileSync(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2));
|
||||
fs.writeFileSync(path.join(targetDir, 'auth.json'), JSON.stringify({
|
||||
top_strategy: topStrategy, indicators: [...allAuth], framework,
|
||||
}, null, 2));
|
||||
|
||||
return { ...result, out_dir: targetDir };
|
||||
})(), { timeout: exploreTimeout, label: `Explore ${url}` });
|
||||
});
|
||||
|
||||
// Write artifacts
|
||||
const manifest = {
|
||||
site: result.site,
|
||||
target_url: result.target_url,
|
||||
final_url: result.final_url,
|
||||
title: result.title,
|
||||
framework: result.framework,
|
||||
top_strategy: result.top_strategy,
|
||||
explored_at: new Date().toISOString(),
|
||||
};
|
||||
fs.writeFileSync(path.join(outDir, 'manifest.json'), JSON.stringify(manifest, null, 2));
|
||||
fs.writeFileSync(path.join(outDir, 'endpoints.json'), JSON.stringify(result.endpoints ?? [], null, 2));
|
||||
fs.writeFileSync(path.join(outDir, 'capabilities.json'), JSON.stringify(result.capabilities ?? [], null, 2));
|
||||
fs.writeFileSync(path.join(outDir, 'auth.json'), JSON.stringify({
|
||||
top_strategy: result.top_strategy,
|
||||
indicators: result.auth_indicators ?? [],
|
||||
framework: result.framework ?? {},
|
||||
}, null, 2));
|
||||
|
||||
return { ...result, out_dir: outDir };
|
||||
}
|
||||
|
||||
export function renderExploreSummary(result: any): string {
|
||||
export function renderExploreSummary(result: Record<string, any>): string {
|
||||
const lines = [
|
||||
'opencli explore: OK',
|
||||
`Site: ${result.site}`,
|
||||
`URL: ${result.target_url}`,
|
||||
`Title: ${result.title || '(none)'}`,
|
||||
`Strategy: ${result.top_strategy}`,
|
||||
'opencli explore: OK', `Site: ${result.site}`, `URL: ${result.target_url}`,
|
||||
`Title: ${result.title || '(none)'}`, `Strategy: ${result.top_strategy}`,
|
||||
`Endpoints: ${result.endpoint_count} total, ${result.api_endpoint_count} API`,
|
||||
`Capabilities: ${result.capabilities?.length ?? 0}`,
|
||||
];
|
||||
for (const cap of (result.capabilities ?? []).slice(0, 5)) {
|
||||
lines.push(` • ${cap.name} (${cap.strategy}, confidence: ${(cap.confidence * 100).toFixed(0)}%)`);
|
||||
lines.push(` • ${cap.name} (${cap.strategy}, ${(cap.confidence * 100).toFixed(0)}%)`);
|
||||
}
|
||||
const fw = result.framework ?? {};
|
||||
const fwNames = Object.entries(fw).filter(([, v]) => v).map(([k]) => k);
|
||||
@@ -646,3 +388,11 @@ export function renderExploreSummary(result: any): string {
|
||||
lines.push(`Output: ${result.out_dir}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function readPageMetadata(page: any): Promise<{ url: string; title: string }> {
|
||||
try {
|
||||
const result = await page.evaluate(`() => ({ url: window.location.href, title: document.title || '' })`);
|
||||
if (result && typeof result === 'object') return { url: String(result.url ?? ''), title: String(result.title ?? '') };
|
||||
} catch {}
|
||||
return { url: '', title: '' };
|
||||
}
|
||||
+131
-147
@@ -1,167 +1,164 @@
|
||||
/**
|
||||
* Synthesize: turn explore capabilities into ready-to-use CLI definitions.
|
||||
*
|
||||
* Takes the structured capabilities from Deep Explore and generates
|
||||
* YAML pipeline files that can be directly registered as CLI commands.
|
||||
*
|
||||
* This is the bridge between discovery (explore) and usability (CLI).
|
||||
* Synthesize candidate CLIs from explore artifacts.
|
||||
* Generates evaluate-based YAML pipelines (matching hand-written adapter patterns).
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
export function synthesizeFromExplore(target: string, opts: any = {}): any {
|
||||
const exploreDir = fs.existsSync(target) ? target : path.join('.opencli', 'explore', target);
|
||||
if (!fs.existsSync(exploreDir)) throw new Error(`Explore dir not found: ${target}`);
|
||||
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8'));
|
||||
const capabilities = JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8'));
|
||||
const endpoints = JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8'));
|
||||
const auth = JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8'));
|
||||
|
||||
const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates');
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
const site = manifest.site;
|
||||
const topN = opts.top ?? 5;
|
||||
const candidates: any[] = [];
|
||||
|
||||
// Sort capabilities by confidence
|
||||
const sortedCaps = [...capabilities]
|
||||
.sort((a: any, b: any) => (b.confidence ?? 0) - (a.confidence ?? 0))
|
||||
.slice(0, topN);
|
||||
|
||||
for (const cap of sortedCaps) {
|
||||
// Find the matching endpoint for more detail
|
||||
const endpoint = endpoints.find((ep: any) => ep.pattern === cap.endpoint) ??
|
||||
endpoints[0];
|
||||
|
||||
const candidate = buildCandidateYaml(site, manifest, cap, endpoint);
|
||||
const fileName = `${cap.name}.yaml`;
|
||||
const filePath = path.join(targetDir, fileName);
|
||||
fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
|
||||
|
||||
candidates.push({
|
||||
name: cap.name,
|
||||
path: filePath,
|
||||
strategy: cap.strategy,
|
||||
endpoint: cap.endpoint,
|
||||
confidence: cap.confidence,
|
||||
columns: candidate.yaml.columns,
|
||||
});
|
||||
}
|
||||
|
||||
const index = {
|
||||
site,
|
||||
target_url: manifest.target_url,
|
||||
generated_from: exploreDir,
|
||||
candidate_count: candidates.length,
|
||||
candidates,
|
||||
};
|
||||
fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2));
|
||||
|
||||
return {
|
||||
site,
|
||||
explore_dir: exploreDir,
|
||||
out_dir: targetDir,
|
||||
candidate_count: candidates.length,
|
||||
candidates,
|
||||
};
|
||||
}
|
||||
|
||||
/** Volatile params to strip from generated URLs */
|
||||
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
|
||||
const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
|
||||
const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
|
||||
const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
|
||||
|
||||
/**
|
||||
* Build a clean templated URL from a raw API URL.
|
||||
* - Strips volatile params (w_rid, wts, etc.)
|
||||
* - Templates search, limit, and pagination params
|
||||
* - Builds URL string manually to avoid URL encoding of ${{ }} expressions
|
||||
*/
|
||||
function buildTemplatedUrl(rawUrl: string, cap: any, endpoint: any): string {
|
||||
export function synthesizeFromExplore(
|
||||
target: string,
|
||||
opts: { outDir?: string; top?: number } = {},
|
||||
): Record<string, any> {
|
||||
const exploreDir = resolveExploreDir(target);
|
||||
const bundle = loadExploreBundle(exploreDir);
|
||||
|
||||
const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates');
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
const site = bundle.manifest.site;
|
||||
const capabilities = (bundle.capabilities ?? [])
|
||||
.sort((a: any, b: any) => (b.confidence ?? 0) - (a.confidence ?? 0))
|
||||
.slice(0, opts.top ?? 3);
|
||||
const candidates: any[] = [];
|
||||
|
||||
for (const cap of capabilities) {
|
||||
const endpoint = chooseEndpoint(cap, bundle.endpoints);
|
||||
if (!endpoint) continue;
|
||||
const candidate = buildCandidateYaml(site, bundle.manifest, cap, endpoint);
|
||||
const filePath = path.join(targetDir, `${candidate.name}.yaml`);
|
||||
fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
|
||||
candidates.push({ name: candidate.name, path: filePath, strategy: cap.strategy, confidence: cap.confidence });
|
||||
}
|
||||
|
||||
const index = { site, target_url: bundle.manifest.target_url, generated_from: exploreDir, candidate_count: candidates.length, candidates };
|
||||
fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2));
|
||||
|
||||
return { site, explore_dir: exploreDir, out_dir: targetDir, candidate_count: candidates.length, candidates };
|
||||
}
|
||||
|
||||
export function renderSynthesizeSummary(result: Record<string, any>): string {
|
||||
const lines = ['opencli synthesize: OK', `Site: ${result.site}`, `Source: ${result.explore_dir}`, `Candidates: ${result.candidate_count}`];
|
||||
for (const c of result.candidates ?? []) lines.push(` • ${c.name} (${c.strategy}, ${((c.confidence ?? 0) * 100).toFixed(0)}% confidence) → ${c.path}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function resolveExploreDir(target: string): string {
|
||||
if (fs.existsSync(target)) return target;
|
||||
const candidate = path.join('.opencli', 'explore', target);
|
||||
if (fs.existsSync(candidate)) return candidate;
|
||||
throw new Error(`Explore directory not found: ${target}`);
|
||||
}
|
||||
|
||||
export function loadExploreBundle(exploreDir: string): Record<string, any> {
|
||||
return {
|
||||
manifest: JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8')),
|
||||
endpoints: JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8')),
|
||||
capabilities: JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8')),
|
||||
auth: JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8')),
|
||||
};
|
||||
}
|
||||
|
||||
function chooseEndpoint(cap: any, endpoints: any[]): any | null {
|
||||
if (!endpoints.length) return null;
|
||||
// Match by endpoint pattern from capability
|
||||
if (cap.endpoint) {
|
||||
const match = endpoints.find((e: any) => e.pattern === cap.endpoint || e.url?.includes(cap.endpoint));
|
||||
if (match) return match;
|
||||
}
|
||||
return endpoints.sort((a: any, b: any) => (b.score ?? 0) - (a.score ?? 0))[0];
|
||||
}
|
||||
|
||||
// ── URL templating ─────────────────────────────────────────────────────────
|
||||
|
||||
function buildTemplatedUrl(rawUrl: string, cap: any, _endpoint: any): string {
|
||||
try {
|
||||
const u = new URL(rawUrl);
|
||||
const base = `${u.protocol}//${u.host}${u.pathname}`;
|
||||
const params: Array<[string, string]> = [];
|
||||
|
||||
const hasKeyword = cap.recommendedArgs?.some((a: any) => a.name === 'keyword');
|
||||
|
||||
u.searchParams.forEach((v, k) => {
|
||||
// Skip volatile params
|
||||
if (VOLATILE_PARAMS.has(k)) return;
|
||||
|
||||
// Template known param types
|
||||
if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) {
|
||||
params.push([k, '${{ args.keyword }}']);
|
||||
} else if (LIMIT_PARAM_NAMES.has(k)) {
|
||||
params.push([k, '${{ args.limit | default(20) }}']);
|
||||
} else if (PAGE_PARAM_NAMES.has(k)) {
|
||||
params.push([k, '${{ args.page | default(1) }}']);
|
||||
} else {
|
||||
params.push([k, v]);
|
||||
}
|
||||
if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) params.push([k, '${{ args.keyword }}']);
|
||||
else if (LIMIT_PARAM_NAMES.has(k)) params.push([k, '${{ args.limit | default(20) }}']);
|
||||
else if (PAGE_PARAM_NAMES.has(k)) params.push([k, '${{ args.page | default(1) }}']);
|
||||
else params.push([k, v]);
|
||||
});
|
||||
|
||||
if (params.length === 0) return base;
|
||||
return base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&');
|
||||
} catch {
|
||||
return rawUrl;
|
||||
}
|
||||
return params.length ? base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&') : base;
|
||||
} catch { return rawUrl; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a YAML pipeline definition from a capability + endpoint.
|
||||
* Build inline evaluate script for browser-based fetch+parse.
|
||||
* Follows patterns from bilibili/hot.yaml and twitter/trending.yaml.
|
||||
*/
|
||||
function buildEvaluateScript(url: string, itemPath: string, endpoint: any): string {
|
||||
const pathChain = itemPath.split('.').map((p: string) => `?.${p}`).join('');
|
||||
const detectedFields = endpoint?.detectedFields ?? {};
|
||||
const hasFields = Object.keys(detectedFields).length > 0;
|
||||
|
||||
let mapCode = '';
|
||||
if (hasFields) {
|
||||
const mappings = Object.entries(detectedFields)
|
||||
.map(([role, field]) => ` ${role}: item${String(field).split('.').map(p => `?.${p}`).join('')}`)
|
||||
.join(',\n');
|
||||
mapCode = `.map((item) => ({\n${mappings}\n }))`;
|
||||
}
|
||||
|
||||
return [
|
||||
'(async () => {',
|
||||
` const res = await fetch('${url}', {`,
|
||||
` credentials: 'include'`,
|
||||
' });',
|
||||
' const data = await res.json();',
|
||||
` return (data${pathChain} || [])${mapCode};`,
|
||||
'})()\n',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// ── YAML pipeline generation ───────────────────────────────────────────────
|
||||
|
||||
function buildCandidateYaml(site: string, manifest: any, cap: any, endpoint: any): { name: string; yaml: any } {
|
||||
const needsBrowser = cap.strategy !== 'public';
|
||||
const pipeline: any[] = [];
|
||||
const templatedUrl = buildTemplatedUrl(endpoint?.url ?? manifest.target_url, cap, endpoint);
|
||||
|
||||
let domain = '';
|
||||
try { domain = new URL(manifest.target_url).hostname; } catch {}
|
||||
|
||||
// Step 1: Navigate (if browser-based)
|
||||
if (needsBrowser) {
|
||||
// Browser-based: navigate + evaluate (like bilibili/hot.yaml, twitter/trending.yaml)
|
||||
pipeline.push({ navigate: manifest.target_url });
|
||||
const itemPath = cap.itemPath ?? 'data.data.list';
|
||||
pipeline.push({ evaluate: buildEvaluateScript(templatedUrl, itemPath, endpoint) });
|
||||
} else {
|
||||
// Public API: direct fetch (like hackernews/top.yaml)
|
||||
pipeline.push({ fetch: { url: templatedUrl } });
|
||||
if (cap.itemPath) pipeline.push({ select: cap.itemPath });
|
||||
}
|
||||
|
||||
// Step 2: Fetch the API — build a clean URL with templates
|
||||
const rawUrl = endpoint?.url ?? manifest.target_url;
|
||||
const fetchStep: any = { url: buildTemplatedUrl(rawUrl, cap, endpoint) };
|
||||
|
||||
pipeline.push({ fetch: fetchStep });
|
||||
|
||||
// Step 3: Select the item path
|
||||
if (cap.itemPath) {
|
||||
pipeline.push({ select: cap.itemPath });
|
||||
}
|
||||
|
||||
// Step 4: Map fields to columns
|
||||
// Map fields
|
||||
const mapStep: Record<string, string> = {};
|
||||
const columns = cap.recommendedColumns ?? ['title', 'url'];
|
||||
|
||||
// Add a rank column if not doing search
|
||||
if (!cap.recommendedArgs?.some((a: any) => a.name === 'keyword')) {
|
||||
mapStep['rank'] = '${{ index + 1 }}';
|
||||
}
|
||||
|
||||
// Build field mappings from the endpoint's detected fields
|
||||
if (!cap.recommendedArgs?.some((a: any) => a.name === 'keyword')) mapStep['rank'] = '${{ index + 1 }}';
|
||||
const detectedFields = endpoint?.detectedFields ?? {};
|
||||
for (const col of columns) {
|
||||
const fieldPath = detectedFields[col];
|
||||
if (fieldPath) {
|
||||
mapStep[col] = `\${{ item.${fieldPath} }}`;
|
||||
} else {
|
||||
mapStep[col] = `\${{ item.${col} }}`;
|
||||
}
|
||||
mapStep[col] = fieldPath ? `\${{ item.${fieldPath} }}` : `\${{ item.${col} }}`;
|
||||
}
|
||||
|
||||
pipeline.push({ map: mapStep });
|
||||
|
||||
// Step 5: Limit
|
||||
pipeline.push({ limit: '${{ args.limit | default(20) }}' });
|
||||
|
||||
// Build args definition
|
||||
// Args
|
||||
const argsDef: Record<string, any> = {};
|
||||
for (const arg of cap.recommendedArgs ?? []) {
|
||||
const def: any = { type: arg.type ?? 'str' };
|
||||
@@ -172,39 +169,26 @@ function buildCandidateYaml(site: string, manifest: any, cap: any, endpoint: any
|
||||
else if (arg.name === 'page') def.description = 'Page number';
|
||||
argsDef[arg.name] = def;
|
||||
}
|
||||
|
||||
// Ensure limit arg always exists
|
||||
if (!argsDef['limit']) {
|
||||
argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' };
|
||||
}
|
||||
|
||||
const allColumns = Object.keys(mapStep);
|
||||
if (!argsDef['limit']) argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' };
|
||||
|
||||
return {
|
||||
name: cap.name,
|
||||
yaml: {
|
||||
site,
|
||||
name: cap.name,
|
||||
description: `${site} ${cap.name} (auto-generated)`,
|
||||
domain: manifest.final_url ? new URL(manifest.final_url).hostname : undefined,
|
||||
strategy: cap.strategy,
|
||||
browser: needsBrowser,
|
||||
args: argsDef,
|
||||
pipeline,
|
||||
columns: allColumns,
|
||||
site, name: cap.name, description: `${cap.description || site + ' ' + cap.name} (auto-generated)`,
|
||||
domain, strategy: cap.strategy, browser: needsBrowser,
|
||||
args: argsDef, pipeline, columns: Object.keys(mapStep),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function renderSynthesizeSummary(r: any): string {
|
||||
const lines = [
|
||||
'opencli synthesize: OK',
|
||||
`Site: ${r.site}`,
|
||||
`Source: ${r.explore_dir}`,
|
||||
`Candidates: ${r.candidate_count}`,
|
||||
];
|
||||
for (const c of r.candidates ?? []) {
|
||||
lines.push(` • ${c.name} (${c.strategy}, ${(c.confidence * 100).toFixed(0)}% confidence) → ${c.path}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
/** Backward-compatible export for scaffold.ts */
|
||||
export function buildCandidate(site: string, targetUrl: string, cap: any, endpoint: any): any {
|
||||
// Map old-style field names to new ones
|
||||
const normalizedCap = {
|
||||
...cap,
|
||||
recommendedArgs: cap.recommendedArgs ?? cap.recommended_args,
|
||||
recommendedColumns: cap.recommendedColumns ?? cap.recommended_columns,
|
||||
};
|
||||
const manifest = { target_url: targetUrl, final_url: targetUrl };
|
||||
return buildCandidateYaml(site, manifest, normalizedCap, endpoint);
|
||||
}
|
||||
Reference in new issue
Block a user