perf: extreme architectural optimizations

Phase 1: Smart autoScroll with MutationObserver (single IPC, instant DOM change detection)
Phase 2: Build-time CLI manifest compiler + lazy-loading engine (instant cold start)
Phase 3: Batch IPC fetch (N browser fetches in single evaluate() call)

package.json

@@ -12,7 +12,8 @@
   },
   "scripts": {
     "dev": "tsx src/main.ts",
-    "build": "tsc && npm run clean-yaml && npm run copy-yaml",
+    "build": "tsc && npm run clean-yaml && npm run copy-yaml && npm run build-manifest",
+    "build-manifest": "node dist/build-manifest.js || true",
     "clean-yaml": "find dist/clis -name '*.yaml' -o -name '*.yml' 2>/dev/null | xargs rm -f",
     "copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done",
     "start": "node dist/main.js",

src/build-manifest.ts

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+/**
+ * Build-time CLI manifest compiler.
+ *
+ * Scans all YAML/TS CLI definitions and pre-compiles them into a single
+ * manifest.json for instant cold-start registration (no runtime YAML parsing).
+ *
+ * Usage: npx tsx src/build-manifest.ts
+ * Output: dist/cli-manifest.json
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import yaml from 'js-yaml';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const CLIS_DIR = path.resolve(__dirname, 'clis');
+const OUTPUT = path.resolve(__dirname, '..', 'dist', 'cli-manifest.json');
+
+interface ManifestEntry {
+  site: string;
+  name: string;
+  description: string;
+  domain?: string;
+  strategy: string;
+  browser: boolean;
+  args: Array<{
+    name: string;
+    type?: string;
+    default?: any;
+    required?: boolean;
+    help?: string;
+    choices?: string[];
+  }>;
+  columns?: string[];
+  pipeline?: any[];
+  timeout?: number;
+  /** 'yaml' or 'ts' — determines how executeCommand loads the handler */
+  type: 'yaml' | 'ts';
+  /** Relative path from clis/ dir, e.g. 'bilibili/hot.yaml' or 'bilibili/search.js' */
+  modulePath?: string;
+}
+
+function scanYaml(filePath: string, site: string): ManifestEntry | null {
+  try {
+    const raw = fs.readFileSync(filePath, 'utf-8');
+    const def = yaml.load(raw) as any;
+    if (!def || typeof def !== 'object') return null;
+
+    const strategyStr = def.strategy ?? (def.browser === false ? 'public' : 'cookie');
+    const strategy = strategyStr.toUpperCase();
+    const browser = def.browser ?? (strategy !== 'PUBLIC');
+
+    const args: ManifestEntry['args'] = [];
+    if (def.args && typeof def.args === 'object') {
+      for (const [argName, argDef] of Object.entries(def.args as Record<string, any>)) {
+        args.push({
+          name: argName,
+          type: argDef?.type ?? 'str',
+          default: argDef?.default,
+          required: argDef?.required ?? false,
+          help: argDef?.description ?? argDef?.help ?? '',
+          choices: argDef?.choices,
+        });
+      }
+    }
+
+    return {
+      site: def.site ?? site,
+      name: def.name ?? path.basename(filePath, path.extname(filePath)),
+      description: def.description ?? '',
+      domain: def.domain,
+      strategy: strategy.toLowerCase(),
+      browser,
+      args,
+      columns: def.columns,
+      pipeline: def.pipeline,
+      timeout: def.timeout,
+      type: 'yaml',
+    };
+  } catch (err: any) {
+    process.stderr.write(`Warning: failed to parse ${filePath}: ${err.message}\n`);
+    return null;
+  }
+}
+
+function scanTs(filePath: string, site: string): ManifestEntry {
+  // TS adapters self-register via cli() at import time.
+  // We record their module path for lazy dynamic import.
+  const baseName = path.basename(filePath, path.extname(filePath));
+  const relativePath = `${site}/${baseName}.js`;
+  return {
+    site,
+    name: baseName,
+    description: '',
+    strategy: 'cookie',
+    browser: true,
+    args: [],
+    type: 'ts',
+    modulePath: relativePath,
+  };
+}
+
+// Main
+const manifest: ManifestEntry[] = [];
+
+if (fs.existsSync(CLIS_DIR)) {
+  for (const site of fs.readdirSync(CLIS_DIR)) {
+    const siteDir = path.join(CLIS_DIR, site);
+    if (!fs.statSync(siteDir).isDirectory()) continue;
+    for (const file of fs.readdirSync(siteDir)) {
+      const filePath = path.join(siteDir, file);
+      if (file.endsWith('.yaml') || file.endsWith('.yml')) {
+        const entry = scanYaml(filePath, site);
+        if (entry) manifest.push(entry);
+      } else if (file.endsWith('.ts') && file !== 'index.ts') {
+        manifest.push(scanTs(filePath, site));
+      }
+    }
+  }
+}
+
+// Ensure output directory exists
+fs.mkdirSync(path.dirname(OUTPUT), { recursive: true });
+fs.writeFileSync(OUTPUT, JSON.stringify(manifest, null, 2));
+
+const yamlCount = manifest.filter(e => e.type === 'yaml').length;
+const tsCount = manifest.filter(e => e.type === 'ts').length;
+console.log(`✅ Manifest compiled: ${manifest.length} entries (${yamlCount} YAML, ${tsCount} TS) → ${OUTPUT}`);

src/engine.ts

@@ -1,5 +1,11 @@
 /**
  * CLI discovery: finds YAML/TS CLI definitions and registers them.
+ *
+ * Supports two modes:
+ * 1. FAST PATH (manifest): If a pre-compiled cli-manifest.json exists,
+ *    registers all YAML commands instantly without runtime YAML parsing.
+ *    TS modules are loaded lazily only when their command is executed.
+ * 2. FALLBACK (filesystem scan): Traditional runtime discovery for development.
  */
 
 import * as fs from 'node:fs';
@@ -9,25 +15,99 @@ import { type CliCommand, type Arg, Strategy, registerCommand } from './registry
 import type { IPage } from './types.js';
 import { executePipeline } from './pipeline.js';
 
+/** Set of TS module paths that have been loaded */
+const _loadedModules = new Set<string>();
+
+/**
+ * Discover and register CLI commands.
+ * Uses pre-compiled manifest when available for instant startup.
+ */
 export async function discoverClis(...dirs: string[]): Promise<void> {
-  const promises: Promise<any>[] = [];
+  // Fast path: try manifest first (production / post-build)
   for (const dir of dirs) {
-    if (!fs.existsSync(dir)) continue;
-    for (const site of fs.readdirSync(dir)) {
-      const siteDir = path.join(dir, site);
-      if (!fs.statSync(siteDir).isDirectory()) continue;
-      for (const file of fs.readdirSync(siteDir)) {
-        const filePath = path.join(siteDir, file);
-        if (file.endsWith('.yaml') || file.endsWith('.yml')) {
-          registerYamlCli(filePath, site);
-        } else if (file.endsWith('.js')) {
-          // Dynamic import of compiled adapter modules
-          promises.push(
-            import(`file://${filePath}`).catch((err: any) => {
-              process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
-            })
-          );
-        }
+    const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
+    if (fs.existsSync(manifestPath)) {
+      loadFromManifest(manifestPath, dir);
+      continue; // Skip filesystem scan for this directory
+    }
+    // Fallback: runtime filesystem scan (development)
+    await discoverClisFromFs(dir);
+  }
+}
+
+/**
+ * Fast-path: register commands from pre-compiled manifest.
+ * YAML pipelines are inlined — zero YAML parsing at runtime.
+ * TS modules are deferred — loaded lazily on first execution.
+ */
+function loadFromManifest(manifestPath: string, clisDir: string): void {
+  try {
+    const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as any[];
+    for (const entry of manifest) {
+      if (entry.type === 'yaml') {
+        // YAML pipelines fully inlined in manifest — register directly
+        const strategy = (Strategy as any)[entry.strategy.toUpperCase()] ?? Strategy.COOKIE;
+        const cmd: CliCommand = {
+          site: entry.site,
+          name: entry.name,
+          description: entry.description ?? '',
+          domain: entry.domain,
+          strategy,
+          browser: entry.browser,
+          args: entry.args ?? [],
+          columns: entry.columns,
+          pipeline: entry.pipeline,
+          timeoutSeconds: entry.timeout,
+          source: `manifest:${entry.site}/${entry.name}`,
+        };
+        registerCommand(cmd);
+      } else if (entry.type === 'ts' && entry.modulePath) {
+        // TS adapters: register a lightweight stub.
+        // The actual module is loaded lazily on first executeCommand().
+        const strategy = (Strategy as any)[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE;
+        const modulePath = path.resolve(clisDir, entry.modulePath);
+        const cmd: CliCommand = {
+          site: entry.site,
+          name: entry.name,
+          description: entry.description ?? '',
+          domain: entry.domain,
+          strategy,
+          browser: entry.browser ?? true,
+          args: entry.args ?? [],
+          columns: entry.columns,
+          timeoutSeconds: entry.timeout,
+          source: modulePath,
+          // Mark as lazy — executeCommand will load the module before running
+          _lazy: true,
+          _modulePath: modulePath,
+        };
+        registerCommand(cmd);
+      }
+    }
+  } catch (err: any) {
+    process.stderr.write(`Warning: failed to load manifest ${manifestPath}: ${err.message}\n`);
+  }
+}
+
+/**
+ * Fallback: traditional filesystem scan (used during development with tsx).
+ */
+async function discoverClisFromFs(dir: string): Promise<void> {
+  if (!fs.existsSync(dir)) return;
+  const promises: Promise<any>[] = [];
+  for (const site of fs.readdirSync(dir)) {
+    const siteDir = path.join(dir, site);
+    if (!fs.statSync(siteDir).isDirectory()) continue;
+    for (const file of fs.readdirSync(siteDir)) {
+      const filePath = path.join(siteDir, file);
+      if (file.endsWith('.yaml') || file.endsWith('.yml')) {
+        registerYamlCli(filePath, site);
+      } else if (file.endsWith('.js')) {
+        promises.push(
+          import(`file://${filePath}`).catch((err: any) => {
+            process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
+          })
+        );
       }
     }
   }
@@ -80,12 +160,38 @@ function registerYamlCli(filePath: string, defaultSite: string): void {
   }
 }
 
+/**
+ * Execute a CLI command. Handles lazy-loading of TS modules.
+ */
 export async function executeCommand(
   cmd: CliCommand,
   page: IPage | null,
   kwargs: Record<string, any>,
   debug: boolean = false,
 ): Promise<any> {
+  // Lazy-load TS module on first execution
+  if ((cmd as any)._lazy && (cmd as any)._modulePath) {
+    const modulePath = (cmd as any)._modulePath;
+    if (!_loadedModules.has(modulePath)) {
+      try {
+        await import(`file://${modulePath}`);
+        _loadedModules.add(modulePath);
+      } catch (err: any) {
+        throw new Error(`Failed to load adapter module ${modulePath}: ${err.message}`);
+      }
+    }
+    // After loading, the module's cli() call will have updated the registry
+    // with the real func/pipeline. Re-fetch the command.
+    const { getRegistry, fullName } = await import('./registry.js');
+    const updated = getRegistry().get(fullName(cmd));
+    if (updated && updated.func) {
+      return updated.func(page, kwargs, debug);
+    }
+    if (updated && updated.pipeline) {
+      return executePipeline(page, updated.pipeline, { args: kwargs, debug });
+    }
+  }
+
   if (cmd.func) {
     return cmd.func(page, kwargs, debug);
   }

src/pipeline/steps/fetch.ts

@@ -56,6 +56,46 @@ async function fetchSingle(
   `);
 }
 
+/**
+ * Batch fetch: send all URLs into the browser as a single evaluate() call.
+ * This eliminates N-1 cross-process IPC round trips, performing all fetches
+ * inside the V8 engine and returning results as one JSON array.
+ */
+async function fetchBatchInBrowser(
+  page: IPage, urls: string[], method: string,
+  headers: Record<string, string>, concurrency: number,
+): Promise<any[]> {
+  const headersJs = JSON.stringify(headers);
+  const urlsJs = JSON.stringify(urls);
+  return page.evaluate(`
+    async () => {
+      const urls = ${urlsJs};
+      const method = "${method}";
+      const headers = ${headersJs};
+      const concurrency = ${concurrency};
+
+      const results = new Array(urls.length);
+      let idx = 0;
+
+      async function worker() {
+        while (idx < urls.length) {
+          const i = idx++;
+          try {
+            const resp = await fetch(urls[i], { method, headers, credentials: "include" });
+            results[i] = await resp.json();
+          } catch (e) {
+            results[i] = { error: e.message };
+          }
+        }
+      }
+
+      const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
+      await Promise.all(workers);
+      return results;
+    }
+  `);
+}
+
 export async function stepFetch(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any> {
   const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
   const method = params?.method ?? 'GET';
@@ -66,9 +106,31 @@ export async function stepFetch(page: IPage | null, params: any, data: any, args
   // Per-item fetch when data is array and URL references item
   if (Array.isArray(data) && urlTemplate.includes('item')) {
     const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
+
+    // Render all URLs upfront
+    const renderedHeaders: Record<string, string> = {};
+    for (const [k, v] of Object.entries(headers)) renderedHeaders[k] = String(render(v, { args, data }));
+    const renderedParams: Record<string, string> = {};
+    for (const [k, v] of Object.entries(queryParams)) renderedParams[k] = String(render(v, { args, data }));
+
+    const urls = data.map((item: any, index: number) => {
+      let url = String(render(urlTemplate, { args, data, item, index }));
+      if (Object.keys(renderedParams).length > 0) {
+        const qs = new URLSearchParams(renderedParams).toString();
+        url = `${url}${url.includes('?') ? '&' : '?'}${qs}`;
+      }
+      return url;
+    });
+
+    // BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
+    if (page !== null) {
+      return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
+    }
+
+    // Non-browser: use concurrent pool (already optimized)
     return mapConcurrent(data, concurrency, async (item, index) => {
       const itemUrl = String(render(urlTemplate, { args, data, item, index }));
-      return fetchSingle(page, itemUrl, method, queryParams, headers, args, data);
+      return fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
     });
   }
   const url = render(urlOrObj, { args, data });

src/registry.ts

@@ -34,6 +34,9 @@ export interface CliCommand {
   pipeline?: any[];
   timeoutSeconds?: number;
   source?: string;
+  /** Internal: lazy-loaded TS module support */
+  _lazy?: boolean;
+  _modulePath?: string;
 }
 
 export interface CliOptions {