diff --git a/package.json b/package.json index 0b1f93b..4dcc8a9 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ }, "scripts": { "dev": "tsx src/main.ts", - "build": "tsc && npm run clean-yaml && npm run copy-yaml", + "build": "tsc && npm run clean-yaml && npm run copy-yaml && npm run build-manifest", + "build-manifest": "node dist/build-manifest.js || true", "clean-yaml": "find dist/clis -name '*.yaml' -o -name '*.yml' 2>/dev/null | xargs rm -f", "copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done", "start": "node dist/main.js", diff --git a/src/build-manifest.ts b/src/build-manifest.ts new file mode 100644 index 0000000..121291f --- /dev/null +++ b/src/build-manifest.ts @@ -0,0 +1,130 @@ +#!/usr/bin/env node +/** + * Build-time CLI manifest compiler. + * + * Scans all YAML/TS CLI definitions and pre-compiles them into a single + * manifest.json for instant cold-start registration (no runtime YAML parsing). + * + * Usage: npx tsx src/build-manifest.ts + * Output: dist/cli-manifest.json + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import yaml from 'js-yaml'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const CLIS_DIR = path.resolve(__dirname, 'clis'); +const OUTPUT = path.resolve(__dirname, '..', 'dist', 'cli-manifest.json'); + +interface ManifestEntry { + site: string; + name: string; + description: string; + domain?: string; + strategy: string; + browser: boolean; + args: Array<{ + name: string; + type?: string; + default?: any; + required?: boolean; + help?: string; + choices?: string[]; + }>; + columns?: string[]; + pipeline?: any[]; + timeout?: number; + /** 'yaml' or 'ts' — determines how executeCommand loads the handler */ + type: 'yaml' | 'ts'; + /** Relative path from clis/ dir, e.g. 'bilibili/hot.yaml' or 'bilibili/search.js' */ + modulePath?: string; +} + +function scanYaml(filePath: string, site: string): ManifestEntry | null { + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + const def = yaml.load(raw) as any; + if (!def || typeof def !== 'object') return null; + + const strategyStr = def.strategy ?? (def.browser === false ? 'public' : 'cookie'); + const strategy = strategyStr.toUpperCase(); + const browser = def.browser ?? (strategy !== 'PUBLIC'); + + const args: ManifestEntry['args'] = []; + if (def.args && typeof def.args === 'object') { + for (const [argName, argDef] of Object.entries(def.args as Record)) { + args.push({ + name: argName, + type: argDef?.type ?? 'str', + default: argDef?.default, + required: argDef?.required ?? false, + help: argDef?.description ?? argDef?.help ?? '', + choices: argDef?.choices, + }); + } + } + + return { + site: def.site ?? site, + name: def.name ?? path.basename(filePath, path.extname(filePath)), + description: def.description ?? '', + domain: def.domain, + strategy: strategy.toLowerCase(), + browser, + args, + columns: def.columns, + pipeline: def.pipeline, + timeout: def.timeout, + type: 'yaml', + }; + } catch (err: any) { + process.stderr.write(`Warning: failed to parse ${filePath}: ${err.message}\n`); + return null; + } +} + +function scanTs(filePath: string, site: string): ManifestEntry { + // TS adapters self-register via cli() at import time. + // We record their module path for lazy dynamic import. + const baseName = path.basename(filePath, path.extname(filePath)); + const relativePath = `${site}/${baseName}.js`; + return { + site, + name: baseName, + description: '', + strategy: 'cookie', + browser: true, + args: [], + type: 'ts', + modulePath: relativePath, + }; +} + +// Main +const manifest: ManifestEntry[] = []; + +if (fs.existsSync(CLIS_DIR)) { + for (const site of fs.readdirSync(CLIS_DIR)) { + const siteDir = path.join(CLIS_DIR, site); + if (!fs.statSync(siteDir).isDirectory()) continue; + for (const file of fs.readdirSync(siteDir)) { + const filePath = path.join(siteDir, file); + if (file.endsWith('.yaml') || file.endsWith('.yml')) { + const entry = scanYaml(filePath, site); + if (entry) manifest.push(entry); + } else if (file.endsWith('.ts') && file !== 'index.ts') { + manifest.push(scanTs(filePath, site)); + } + } + } +} + +// Ensure output directory exists +fs.mkdirSync(path.dirname(OUTPUT), { recursive: true }); +fs.writeFileSync(OUTPUT, JSON.stringify(manifest, null, 2)); + +const yamlCount = manifest.filter(e => e.type === 'yaml').length; +const tsCount = manifest.filter(e => e.type === 'ts').length; +console.log(`✅ Manifest compiled: ${manifest.length} entries (${yamlCount} YAML, ${tsCount} TS) → ${OUTPUT}`); diff --git a/src/engine.ts b/src/engine.ts index d9b5cb8..278ec99 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -1,5 +1,11 @@ /** * CLI discovery: finds YAML/TS CLI definitions and registers them. + * + * Supports two modes: + * 1. FAST PATH (manifest): If a pre-compiled cli-manifest.json exists, + * registers all YAML commands instantly without runtime YAML parsing. + * TS modules are loaded lazily only when their command is executed. + * 2. FALLBACK (filesystem scan): Traditional runtime discovery for development. */ import * as fs from 'node:fs'; @@ -9,25 +15,99 @@ import { type CliCommand, type Arg, Strategy, registerCommand } from './registry import type { IPage } from './types.js'; import { executePipeline } from './pipeline.js'; +/** Set of TS module paths that have been loaded */ +const _loadedModules = new Set(); + +/** + * Discover and register CLI commands. + * Uses pre-compiled manifest when available for instant startup. + */ export async function discoverClis(...dirs: string[]): Promise { - const promises: Promise[] = []; + // Fast path: try manifest first (production / post-build) for (const dir of dirs) { - if (!fs.existsSync(dir)) continue; - for (const site of fs.readdirSync(dir)) { - const siteDir = path.join(dir, site); - if (!fs.statSync(siteDir).isDirectory()) continue; - for (const file of fs.readdirSync(siteDir)) { - const filePath = path.join(siteDir, file); - if (file.endsWith('.yaml') || file.endsWith('.yml')) { - registerYamlCli(filePath, site); - } else if (file.endsWith('.js')) { - // Dynamic import of compiled adapter modules - promises.push( - import(`file://${filePath}`).catch((err: any) => { - process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`); - }) - ); - } + const manifestPath = path.resolve(dir, '..', 'cli-manifest.json'); + if (fs.existsSync(manifestPath)) { + loadFromManifest(manifestPath, dir); + continue; // Skip filesystem scan for this directory + } + // Fallback: runtime filesystem scan (development) + await discoverClisFromFs(dir); + } +} + +/** + * Fast-path: register commands from pre-compiled manifest. + * YAML pipelines are inlined — zero YAML parsing at runtime. + * TS modules are deferred — loaded lazily on first execution. + */ +function loadFromManifest(manifestPath: string, clisDir: string): void { + try { + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as any[]; + for (const entry of manifest) { + if (entry.type === 'yaml') { + // YAML pipelines fully inlined in manifest — register directly + const strategy = (Strategy as any)[entry.strategy.toUpperCase()] ?? Strategy.COOKIE; + const cmd: CliCommand = { + site: entry.site, + name: entry.name, + description: entry.description ?? '', + domain: entry.domain, + strategy, + browser: entry.browser, + args: entry.args ?? [], + columns: entry.columns, + pipeline: entry.pipeline, + timeoutSeconds: entry.timeout, + source: `manifest:${entry.site}/${entry.name}`, + }; + registerCommand(cmd); + } else if (entry.type === 'ts' && entry.modulePath) { + // TS adapters: register a lightweight stub. + // The actual module is loaded lazily on first executeCommand(). + const strategy = (Strategy as any)[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE; + const modulePath = path.resolve(clisDir, entry.modulePath); + const cmd: CliCommand = { + site: entry.site, + name: entry.name, + description: entry.description ?? '', + domain: entry.domain, + strategy, + browser: entry.browser ?? true, + args: entry.args ?? [], + columns: entry.columns, + timeoutSeconds: entry.timeout, + source: modulePath, + // Mark as lazy — executeCommand will load the module before running + _lazy: true, + _modulePath: modulePath, + }; + registerCommand(cmd); + } + } + } catch (err: any) { + process.stderr.write(`Warning: failed to load manifest ${manifestPath}: ${err.message}\n`); + } +} + +/** + * Fallback: traditional filesystem scan (used during development with tsx). + */ +async function discoverClisFromFs(dir: string): Promise { + if (!fs.existsSync(dir)) return; + const promises: Promise[] = []; + for (const site of fs.readdirSync(dir)) { + const siteDir = path.join(dir, site); + if (!fs.statSync(siteDir).isDirectory()) continue; + for (const file of fs.readdirSync(siteDir)) { + const filePath = path.join(siteDir, file); + if (file.endsWith('.yaml') || file.endsWith('.yml')) { + registerYamlCli(filePath, site); + } else if (file.endsWith('.js')) { + promises.push( + import(`file://${filePath}`).catch((err: any) => { + process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`); + }) + ); } } } @@ -80,12 +160,38 @@ function registerYamlCli(filePath: string, defaultSite: string): void { } } +/** + * Execute a CLI command. Handles lazy-loading of TS modules. + */ export async function executeCommand( cmd: CliCommand, page: IPage | null, kwargs: Record, debug: boolean = false, ): Promise { + // Lazy-load TS module on first execution + if ((cmd as any)._lazy && (cmd as any)._modulePath) { + const modulePath = (cmd as any)._modulePath; + if (!_loadedModules.has(modulePath)) { + try { + await import(`file://${modulePath}`); + _loadedModules.add(modulePath); + } catch (err: any) { + throw new Error(`Failed to load adapter module ${modulePath}: ${err.message}`); + } + } + // After loading, the module's cli() call will have updated the registry + // with the real func/pipeline. Re-fetch the command. + const { getRegistry, fullName } = await import('./registry.js'); + const updated = getRegistry().get(fullName(cmd)); + if (updated && updated.func) { + return updated.func(page, kwargs, debug); + } + if (updated && updated.pipeline) { + return executePipeline(page, updated.pipeline, { args: kwargs, debug }); + } + } + if (cmd.func) { return cmd.func(page, kwargs, debug); } diff --git a/src/pipeline/steps/fetch.ts b/src/pipeline/steps/fetch.ts index d5d79f5..07dcc3f 100644 --- a/src/pipeline/steps/fetch.ts +++ b/src/pipeline/steps/fetch.ts @@ -56,6 +56,46 @@ async function fetchSingle( `); } +/** + * Batch fetch: send all URLs into the browser as a single evaluate() call. + * This eliminates N-1 cross-process IPC round trips, performing all fetches + * inside the V8 engine and returning results as one JSON array. + */ +async function fetchBatchInBrowser( + page: IPage, urls: string[], method: string, + headers: Record, concurrency: number, +): Promise { + const headersJs = JSON.stringify(headers); + const urlsJs = JSON.stringify(urls); + return page.evaluate(` + async () => { + const urls = ${urlsJs}; + const method = "${method}"; + const headers = ${headersJs}; + const concurrency = ${concurrency}; + + const results = new Array(urls.length); + let idx = 0; + + async function worker() { + while (idx < urls.length) { + const i = idx++; + try { + const resp = await fetch(urls[i], { method, headers, credentials: "include" }); + results[i] = await resp.json(); + } catch (e) { + results[i] = { error: e.message }; + } + } + } + + const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker()); + await Promise.all(workers); + return results; + } + `); +} + export async function stepFetch(page: IPage | null, params: any, data: any, args: Record): Promise { const urlOrObj = typeof params === 'string' ? params : (params?.url ?? ''); const method = params?.method ?? 'GET'; @@ -66,9 +106,31 @@ export async function stepFetch(page: IPage | null, params: any, data: any, args // Per-item fetch when data is array and URL references item if (Array.isArray(data) && urlTemplate.includes('item')) { const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5; + + // Render all URLs upfront + const renderedHeaders: Record = {}; + for (const [k, v] of Object.entries(headers)) renderedHeaders[k] = String(render(v, { args, data })); + const renderedParams: Record = {}; + for (const [k, v] of Object.entries(queryParams)) renderedParams[k] = String(render(v, { args, data })); + + const urls = data.map((item: any, index: number) => { + let url = String(render(urlTemplate, { args, data, item, index })); + if (Object.keys(renderedParams).length > 0) { + const qs = new URLSearchParams(renderedParams).toString(); + url = `${url}${url.includes('?') ? '&' : '?'}${qs}`; + } + return url; + }); + + // BATCH IPC: if browser is available, batch all fetches into a single evaluate() call + if (page !== null) { + return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency); + } + + // Non-browser: use concurrent pool (already optimized) return mapConcurrent(data, concurrency, async (item, index) => { const itemUrl = String(render(urlTemplate, { args, data, item, index })); - return fetchSingle(page, itemUrl, method, queryParams, headers, args, data); + return fetchSingle(null, itemUrl, method, queryParams, headers, args, data); }); } const url = render(urlOrObj, { args, data }); diff --git a/src/registry.ts b/src/registry.ts index fa6d824..c40d887 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -34,6 +34,9 @@ export interface CliCommand { pipeline?: any[]; timeoutSeconds?: number; source?: string; + /** Internal: lazy-loaded TS module support */ + _lazy?: boolean; + _modulePath?: string; } export interface CliOptions {