okooooooo/opencli
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add download support for xhs, twi, bilibili, zhihu (#22)

Browse Source 
* feat: add download support for images, videos, and articles

Add comprehensive download functionality to OpenCLI with support for
multiple platforms and content types.

- Add `src/download/index.ts`: HTTP download with progress, yt-dlp
  wrapper for video platforms, cookie export to Netscape format for
  authenticated downloads
- Add `src/download/progress.ts`: Terminal progress bars, multi-file
  download tracker with status summary
- Add `src/pipeline/steps/download.ts`: New `download` pipeline step
  for declarative YAML pipelines

- Register `download` step in executor.ts
- Add template filters: `slugify`, `sanitize`, `ext`, `basename` for
  filename templating

- `xiaohongshu download`: Download images and videos from notes
- `bilibili download`: Download videos using yt-dlp with cookie auth
- `twitter download`: Download media from user timeline or single tweet
- `zhihu download`: Export articles to Markdown with optional image
  download

```yaml
pipeline:
  - download:
      url: ${{ item.imageUrl }}
      dir: ./downloads
      filename: ${{ item.title | sanitize }}.jpg
      concurrency: 5
      skip_existing: true
      use_ytdlp: false
      type: auto  # auto|image|video|document
```

- Concurrent downloads with configurable parallelism
- Progress bars with file size display
- Skip existing files option
- Cookie forwarding for authenticated downloads
- yt-dlp integration for video platforms (YouTube, Bilibili, Twitter)
- HTML to Markdown conversion for article export

- yt-dlp: Required for video downloads from streaming platforms
- ffmpeg: Optional for video format conversion

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* docs: add download support documentation

- Add Download Support section to both README.md and README.zh-CN.md
- Document supported platforms: Xiaohongshu, Bilibili, Twitter, Zhihu
- Include prerequisites (yt-dlp installation)
- Add usage examples for all download commands
- Document the `download` pipeline step for YAML adapters
- Update built-in commands table with new `download` commands

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: preserve zhihu ordered list content

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: jackwener <jakevingoo@gmail.com>
This commit is contained in:
AlexYueClaude Opus 4.5jackwener
 authored and GitHub committed 2026-03-18 22:20:13 +08:00
1 parent 700d970f13
commit 8a355dfd2d
12 files changed
+1849 -69
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/download/index.ts
+395
View File
@@ -0,0 +1,395 @@
/**
* Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
*/
import { spawn, execSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as https from 'node:https';
import * as http from 'node:http';
import * as os from 'node:os';
import { URL } from 'node:url';
import type { ProgressBar } from './progress.js';
export interface DownloadOptions {
cookies?: string;
headers?: Record<string, string>;
timeout?: number;
onProgress?: (received: number, total: number) => void;
}
export interface YtdlpOptions {
cookies?: string;
cookiesFile?: string;
format?: string;
extraArgs?: string[];
onProgress?: (percent: number) => void;
}
/**
* Check if yt-dlp is available in PATH.
*/
export function checkYtdlp(): boolean {
try {
execSync('yt-dlp --version', { encoding: 'utf-8', stdio: 'pipe' });
return true;
} catch {
return false;
}
}
/**
* Check if ffmpeg is available in PATH.
*/
export function checkFfmpeg(): boolean {
try {
execSync('ffmpeg -version', { encoding: 'utf-8', stdio: 'pipe' });
return true;
} catch {
return false;
}
}
/**
* Detect content type from URL and optional headers.
*/
export function detectContentType(url: string, contentType?: string): 'image' | 'video' | 'document' | 'binary' {
// Check content-type header first
if (contentType) {
if (contentType.startsWith('image/')) return 'image';
if (contentType.startsWith('video/')) return 'video';
if (contentType.startsWith('text/') || contentType.includes('json') || contentType.includes('xml')) return 'document';
}
// Detect from URL
const urlLower = url.toLowerCase();
const ext = path.extname(new URL(url).pathname).toLowerCase();
// Image extensions
if (['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.avif'].includes(ext)) {
return 'image';
}
// Video extensions
if (['.mp4', '.webm', '.avi', '.mov', '.mkv', '.flv', '.m3u8', '.ts'].includes(ext)) {
return 'video';
}
// Video platforms (need yt-dlp)
if (urlLower.includes('youtube.com') || urlLower.includes('youtu.be') ||
urlLower.includes('bilibili.com') || urlLower.includes('twitter.com') ||
urlLower.includes('x.com') || urlLower.includes('tiktok.com') ||
urlLower.includes('vimeo.com') || urlLower.includes('twitch.tv')) {
return 'video';
}
// Document extensions
if (['.html', '.htm', '.json', '.xml', '.txt', '.md', '.markdown'].includes(ext)) {
return 'document';
}
return 'binary';
}
/**
* Check if URL requires yt-dlp for download.
*/
export function requiresYtdlp(url: string): boolean {
const urlLower = url.toLowerCase();
return (
urlLower.includes('youtube.com') ||
urlLower.includes('youtu.be') ||
urlLower.includes('bilibili.com/video') ||
urlLower.includes('twitter.com') ||
urlLower.includes('x.com') ||
urlLower.includes('tiktok.com') ||
urlLower.includes('vimeo.com') ||
urlLower.includes('twitch.tv')
);
}
/**
* HTTP download with progress callback.
*/
export async function httpDownload(
url: string,
destPath: string,
options: DownloadOptions = {},
): Promise<{ success: boolean; size: number; error?: string }> {
const { cookies, headers = {}, timeout = 30000, onProgress } = options;
return new Promise((resolve) => {
const parsedUrl = new URL(url);
const protocol = parsedUrl.protocol === 'https:' ? https : http;
const requestHeaders: Record<string, string> = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
...headers,
};
if (cookies) {
requestHeaders['Cookie'] = cookies;
}
// Ensure directory exists
const dir = path.dirname(destPath);
fs.mkdirSync(dir, { recursive: true });
const tempPath = `${destPath}.tmp`;
const file = fs.createWriteStream(tempPath);
const request = protocol.get(url, { headers: requestHeaders, timeout }, (response) => {
// Handle redirects
if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
file.close();
fs.unlinkSync(tempPath);
httpDownload(response.headers.location, destPath, options).then(resolve);
return;
}
if (response.statusCode !== 200) {
file.close();
fs.unlinkSync(tempPath);
resolve({ success: false, size: 0, error: `HTTP ${response.statusCode}` });
return;
}
const totalSize = parseInt(response.headers['content-length'] || '0', 10);
let received = 0;
response.on('data', (chunk: Buffer) => {
received += chunk.length;
if (onProgress) onProgress(received, totalSize);
});
response.pipe(file);
file.on('finish', () => {
file.close();
// Rename temp file to final destination
fs.renameSync(tempPath, destPath);
resolve({ success: true, size: received });
});
});
request.on('error', (err) => {
file.close();
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
resolve({ success: false, size: 0, error: err.message });
});
request.on('timeout', () => {
request.destroy();
file.close();
if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath);
resolve({ success: false, size: 0, error: 'Timeout' });
});
});
}
/**
* Export cookies to Netscape format for yt-dlp.
*/
export function exportCookiesToNetscape(
cookies: Array<{ name: string; value: string; domain: string; path?: string; secure?: boolean; httpOnly?: boolean }>,
filePath: string,
): void {
const lines = [
'# Netscape HTTP Cookie File',
'# https://curl.se/docs/http-cookies.html',
'# This is a generated file! Do not edit.',
'',
];
for (const cookie of cookies) {
const domain = cookie.domain.startsWith('.') ? cookie.domain : `.${cookie.domain}`;
const includeSubdomains = 'TRUE';
const cookiePath = cookie.path || '/';
const secure = cookie.secure ? 'TRUE' : 'FALSE';
const expiry = Math.floor(Date.now() / 1000) + 86400 * 365; // 1 year from now
lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${cookie.name}\t${cookie.value}`);
}
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, lines.join('\n'));
}
/**
* Download video using yt-dlp.
*/
export async function ytdlpDownload(
url: string,
destPath: string,
options: YtdlpOptions = {},
): Promise<{ success: boolean; size: number; error?: string }> {
const { cookiesFile, format = 'best', extraArgs = [], onProgress } = options;
if (!checkYtdlp()) {
return { success: false, size: 0, error: 'yt-dlp not installed. Install with: pip install yt-dlp' };
}
return new Promise((resolve) => {
const dir = path.dirname(destPath);
fs.mkdirSync(dir, { recursive: true });
// Build yt-dlp arguments
const args = [
url,
'-o', destPath,
'-f', format,
'--no-playlist',
'--progress',
];
if (cookiesFile && fs.existsSync(cookiesFile)) {
args.push('--cookies', cookiesFile);
} else {
// Try to use browser cookies
args.push('--cookies-from-browser', 'chrome');
}
args.push(...extraArgs);
const proc = spawn('yt-dlp', args, {
stdio: ['ignore', 'pipe', 'pipe'],
});
let lastPercent = 0;
let errorOutput = '';
proc.stderr.on('data', (data: Buffer) => {
const line = data.toString();
errorOutput += line;
// Parse progress from yt-dlp output
const match = line.match(/(\d+\.?\d*)%/);
if (match && onProgress) {
const percent = parseFloat(match[1]);
if (percent > lastPercent) {
lastPercent = percent;
onProgress(percent);
}
}
});
proc.stdout.on('data', (data: Buffer) => {
const line = data.toString();
const match = line.match(/(\d+\.?\d*)%/);
if (match && onProgress) {
const percent = parseFloat(match[1]);
if (percent > lastPercent) {
lastPercent = percent;
onProgress(percent);
}
}
});
proc.on('close', (code) => {
if (code === 0 && fs.existsSync(destPath)) {
const stats = fs.statSync(destPath);
resolve({ success: true, size: stats.size });
} else {
// Check for common yt-dlp output patterns
const patterns = fs.readdirSync(dir).filter(f => f.startsWith(path.basename(destPath, path.extname(destPath))));
if (patterns.length > 0) {
const actualFile = path.join(dir, patterns[0]);
const stats = fs.statSync(actualFile);
resolve({ success: true, size: stats.size });
} else {
resolve({ success: false, size: 0, error: errorOutput.slice(0, 200) || `Exit code ${code}` });
}
}
});
proc.on('error', (err) => {
resolve({ success: false, size: 0, error: err.message });
});
});
}
/**
* Save document content to file.
*/
export async function saveDocument(
content: string,
destPath: string,
format: 'json' | 'markdown' | 'html' | 'text' = 'markdown',
metadata?: Record<string, any>,
): Promise<{ success: boolean; size: number; error?: string }> {
try {
const dir = path.dirname(destPath);
fs.mkdirSync(dir, { recursive: true });
let output: string;
if (format === 'json') {
output = JSON.stringify({ ...metadata, content }, null, 2);
} else if (format === 'markdown') {
// Add frontmatter if metadata exists
const frontmatter = metadata ? `---\n${Object.entries(metadata).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join('\n')}\n---\n\n` : '';
output = frontmatter + content;
} else {
output = content;
}
fs.writeFileSync(destPath, output, 'utf-8');
return { success: true, size: Buffer.byteLength(output, 'utf-8') };
} catch (err: any) {
return { success: false, size: 0, error: err.message };
}
}
/**
* Sanitize filename by removing invalid characters.
*/
export function sanitizeFilename(name: string, maxLength: number = 200): string {
return name
.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_') // Remove invalid chars
.replace(/\s+/g, '_') // Replace spaces with underscores
.replace(/_+/g, '_') // Collapse multiple underscores
.replace(/^_|_$/g, '') // Trim underscores
.slice(0, maxLength);
}
/**
* Generate filename from URL if not provided.
*/
export function generateFilename(url: string, index: number, extension?: string): string {
try {
const parsedUrl = new URL(url);
const pathname = parsedUrl.pathname;
const basename = path.basename(pathname);
if (basename && basename !== '/' && basename.includes('.')) {
return sanitizeFilename(basename);
}
// Generate from hostname and index
const ext = extension || detectExtension(url);
const hostname = parsedUrl.hostname.replace(/^www\./, '');
return sanitizeFilename(`${hostname}_${index + 1}${ext}`);
} catch {
const ext = extension || '.bin';
return `download_${index + 1}${ext}`;
}
}
/**
* Detect file extension from URL.
*/
function detectExtension(url: string): string {
const type = detectContentType(url);
switch (type) {
case 'image': return '.jpg';
case 'video': return '.mp4';
case 'document': return '.md';
default: return '.bin';
}
}
/**
* Get temp directory for cookie files.
*/
export function getTempDir(): string {
return path.join(os.tmpdir(), 'opencli-download');
}
src/download/progress.ts
+125
View File
@@ -0,0 +1,125 @@
/**
* Download progress display: terminal progress bars, status updates.
*/
import chalk from 'chalk';
export interface ProgressBar {
update(current: number, total: number, label?: string): void;
complete(success: boolean, message?: string): void;
fail(error: string): void;
}
/**
* Format bytes as human-readable string (KB, MB, GB).
*/
export function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
}
/**
* Format milliseconds as human-readable duration.
*/
export function formatDuration(ms: number): string {
if (ms < 1000) return `${ms}ms`;
const seconds = Math.floor(ms / 1000);
if (seconds < 60) return `${seconds}s`;
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60;
return `${minutes}m ${remainingSeconds}s`;
}
/**
* Create a simple progress bar for terminal display.
*/
export function createProgressBar(filename: string, index: number, total: number): ProgressBar {
const prefix = chalk.dim(`[${index + 1}/${total}]`);
const truncatedName = filename.length > 40 ? filename.slice(0, 37) + '...' : filename;
return {
update(current: number, totalBytes: number, label?: string) {
const percent = totalBytes > 0 ? Math.round((current / totalBytes) * 100) : 0;
const bar = createBar(percent);
const size = totalBytes > 0 ? formatBytes(totalBytes) : '';
const extra = label ? ` ${label}` : '';
process.stderr.write(`\r${prefix} ${truncatedName} ${bar} ${percent}% ${size}${extra}`);
},
complete(success: boolean, message?: string) {
const icon = success ? chalk.green('✓') : chalk.red('✗');
const msg = message ? ` ${chalk.dim(message)}` : '';
process.stderr.write(`\r${prefix} ${icon} ${truncatedName}${msg}\n`);
},
fail(error: string) {
process.stderr.write(`\r${prefix} ${chalk.red('✗')} ${truncatedName} ${chalk.red(error)}\n`);
},
};
}
/**
* Create a progress bar string.
*/
function createBar(percent: number, width: number = 20): string {
const filled = Math.round((percent / 100) * width);
const empty = width - filled;
return chalk.cyan('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
}
/**
* Multi-file download progress tracker.
*/
export class DownloadProgressTracker {
private completed = 0;
private failed = 0;
private skipped = 0;
private total: number;
private startTime: number;
private verbose: boolean;
constructor(total: number, verbose: boolean = true) {
this.total = total;
this.startTime = Date.now();
this.verbose = verbose;
}
onFileStart(filename: string, index: number): ProgressBar | null {
if (!this.verbose) return null;
return createProgressBar(filename, index, this.total);
}
onFileComplete(success: boolean, skipped: boolean = false): void {
if (skipped) {
this.skipped++;
} else if (success) {
this.completed++;
} else {
this.failed++;
}
}
getSummary(): string {
const elapsed = formatDuration(Date.now() - this.startTime);
const parts: string[] = [];
if (this.completed > 0) {
parts.push(chalk.green(`${this.completed} downloaded`));
}
if (this.skipped > 0) {
parts.push(chalk.yellow(`${this.skipped} skipped`));
}
if (this.failed > 0) {
parts.push(chalk.red(`${this.failed} failed`));
}
return `${parts.join(', ')} in ${elapsed}`;
}
finish(): void {
if (this.verbose) {
process.stderr.write(`\n${chalk.bold('Download complete:')} ${this.getSummary()}\n`);
}
}
}