From 8e7490407c9cb2ae4594665284e935b30dbace23 Mon Sep 17 00:00:00 2001 From: Ji Date: Mon, 16 Mar 2026 22:26:30 -0700 Subject: [PATCH] feat(youtube): add video metadata and transcript commands (#25) Add two new YouTube adapters: - **youtube video**: fetch metadata (title, views, description, etc.) from ytInitialPlayerResponse and ytInitialData - **youtube transcript**: fetch subtitles via Android InnerTube API to bypass PoToken requirement on Web client caption URLs - Two output modes: --mode grouped (sentence merging, speaker detection, chapter headings) and --mode raw (precise sub-second timestamps) - CJK support with 30s time-window fallback for unpunctuated captions - Language selection with --lang and stderr warning on fallback - URL normalization for watch, youtu.be, shorts, embed, live formats Co-authored-by: Ji Zhang --- src/clis/youtube/transcript-group.test.ts | 108 ++++++++ src/clis/youtube/transcript-group.ts | 287 ++++++++++++++++++++++ src/clis/youtube/transcript.ts | 280 +++++++++++++++++++++ src/clis/youtube/utils.ts | 28 +++ src/clis/youtube/video.ts | 116 +++++++++ 5 files changed, 819 insertions(+) create mode 100644 src/clis/youtube/transcript-group.test.ts create mode 100644 src/clis/youtube/transcript-group.ts create mode 100644 src/clis/youtube/transcript.ts create mode 100644 src/clis/youtube/utils.ts create mode 100644 src/clis/youtube/video.ts diff --git a/src/clis/youtube/transcript-group.test.ts b/src/clis/youtube/transcript-group.test.ts new file mode 100644 index 0000000..509a225 --- /dev/null +++ b/src/clis/youtube/transcript-group.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect } from 'vitest'; +import { groupTranscriptSegments, formatGroupedTranscript } from './transcript-group.js'; + +describe('groupTranscriptSegments', () => { + it('groups segments by sentence boundaries', () => { + const segments = [ + { start: 0, text: 'Hello there.' }, + { start: 2, text: 'How are you doing today?' }, + { start: 5, text: 'I am' }, + { start: 6, text: 'doing well.' }, + ]; + const result = groupTranscriptSegments(segments); + expect(result).toHaveLength(3); + expect(result[0].text).toBe('Hello there.'); + expect(result[1].text).toBe('How are you doing today?'); + expect(result[2].text).toBe('I am doing well.'); + }); + + it('flushes on large time gaps', () => { + const segments = [ + { start: 0, text: 'First part' }, + { start: 2, text: 'still first' }, + { start: 25, text: 'second part after gap' }, + ]; + const result = groupTranscriptSegments(segments); + expect(result).toHaveLength(2); + expect(result[0].text).toBe('First part still first'); + expect(result[1].text).toBe('second part after gap'); + }); + + it('respects 30s max group span for unpunctuated text', () => { + // Simulate CJK captions without punctuation + const segments = Array.from({ length: 20 }, (_, i) => ({ + start: i * 2, + text: `segment${i}`, + })); + const result = groupTranscriptSegments(segments); + // 20 segments * 2s = 40s total, should be split into at least 2 groups + expect(result.length).toBeGreaterThanOrEqual(2); + // No single group should span more than ~30s + for (const g of result) { + const words = g.text.split(' '); + // With 2s per segment and 30s max, each group should have at most ~16 segments + expect(words.length).toBeLessThanOrEqual(16); + } + }); + + it('detects speaker changes via >> markers', () => { + const segments = [ + { start: 0, text: '>> How are you?' }, + { start: 3, text: '>> I am fine.' }, + ]; + const result = groupTranscriptSegments(segments); + expect(result.some(g => g.speakerChange)).toBe(true); + expect(result.some(g => g.speaker !== undefined)).toBe(true); + }); + + it('recognizes CJK sentence-ending punctuation', () => { + const segments = [ + { start: 0, text: '你好世界。' }, + { start: 2, text: '这是测试' }, + { start: 4, text: '内容。' }, + ]; + const result = groupTranscriptSegments(segments); + expect(result).toHaveLength(2); + expect(result[0].text).toBe('你好世界。'); + expect(result[1].text).toBe('这是测试 内容。'); + }); + + it('returns empty array for empty input', () => { + expect(groupTranscriptSegments([])).toEqual([]); + }); +}); + +describe('formatGroupedTranscript', () => { + it('formats timestamps correctly', () => { + const segments = [ + { start: 65, text: 'One minute five.', speakerChange: false }, + { start: 3661, text: 'One hour one minute.', speakerChange: false }, + ]; + const { rows } = formatGroupedTranscript(segments); + expect(rows[0].timestamp).toBe('1:05'); + expect(rows[1].timestamp).toBe('1:01:01'); + }); + + it('inserts chapter headings at correct positions', () => { + const segments = [ + { start: 0, text: 'Intro text.', speakerChange: false }, + { start: 60, text: 'Chapter content.', speakerChange: false }, + ]; + const chapters = [{ title: 'Introduction', start: 0 }, { title: 'Main', start: 50 }]; + const { rows } = formatGroupedTranscript(segments, chapters); + expect(rows[0].text).toBe('[Chapter] Introduction'); + expect(rows[1].text).toBe('Intro text.'); + expect(rows[2].text).toBe('[Chapter] Main'); + expect(rows[3].text).toBe('Chapter content.'); + }); + + it('labels speakers', () => { + const segments = [ + { start: 0, text: 'Hello.', speakerChange: true, speaker: 0 }, + { start: 5, text: 'Hi there.', speakerChange: true, speaker: 1 }, + ]; + const { rows } = formatGroupedTranscript(segments); + expect(rows[0].speaker).toBe('Speaker 1'); + expect(rows[1].speaker).toBe('Speaker 2'); + }); +}); diff --git a/src/clis/youtube/transcript-group.ts b/src/clis/youtube/transcript-group.ts new file mode 100644 index 0000000..221680c --- /dev/null +++ b/src/clis/youtube/transcript-group.ts @@ -0,0 +1,287 @@ +/** + * Transcript grouping: sentence merging, speaker detection, and chapter support. + * Ported and simplified from Defuddle's YouTube extractor. + * + * Raw segments (2-3 second fragments) are grouped into readable paragraphs: + * - Sentence boundaries: merge until sentence-ending punctuation (.!?) + * - Speaker turns: detect ">>" markers from YouTube auto-captions + * - Chapters: optional chapter headings inserted at appropriate timestamps + */ + +// Include CJK sentence-ending punctuation: 。!? (fullwidth: .!?) +const SENTENCE_END = /[.!?\u3002\uFF01\uFF1F\uFF0E]["'\u2019\u201D)]*\s*$/; +const QUESTION_END = /[?\uFF1F]["'\u2019\u201D)]*\s*$/; +const TRANSCRIPT_GROUP_GAP_SECONDS = 20; +const TURN_MERGE_MAX_WORDS = 80; +const TURN_MERGE_MAX_SPAN_SECONDS = 45; +const SHORT_UTTERANCE_MAX_WORDS = 3; +const FIRST_GROUP_MERGE_MIN_WORDS = 8; + +export interface RawSegment { + start: number; + end: number; + text: string; +} + +export interface GroupedSegment { + start: number; + text: string; + speakerChange: boolean; + speaker?: number; +} + +export interface Chapter { + title: string; + start: number; +} + +function countWords(text: string): number { + return text.split(/\s+/).filter(Boolean).length; +} + +/** + * Group raw transcript segments into readable blocks. + * If speaker markers (>>) are present, groups by speaker turn. + * Otherwise, groups by sentence boundaries. + */ +export function groupTranscriptSegments( + segments: { start: number; text: string }[], +): GroupedSegment[] { + if (segments.length === 0) return []; + const hasSpeakerMarkers = segments.some(s => /^>>/.test(s.text)); + return hasSpeakerMarkers ? groupBySpeaker(segments) : groupBySentence(segments); +} + +/** + * Format grouped segments + chapters into a final text output. + */ +export function formatGroupedTranscript( + segments: GroupedSegment[], + chapters: Chapter[] = [], +): { rows: Array<{ timestamp: string; speaker: string; text: string }>; plainText: string } { + const sortedChapters = [...chapters].sort((a, b) => a.start - b.start); + let chapterIdx = 0; + + const rows: Array<{ timestamp: string; speaker: string; text: string }> = []; + const textParts: string[] = []; + + for (const segment of segments) { + // Insert chapter headings + while (chapterIdx < sortedChapters.length && sortedChapters[chapterIdx].start <= segment.start) { + const title = sortedChapters[chapterIdx].title; + rows.push({ timestamp: fmtTime(sortedChapters[chapterIdx].start), speaker: '', text: `[Chapter] ${title}` }); + if (textParts.length > 0) textParts.push(''); + textParts.push(`### ${title}`); + textParts.push(''); + chapterIdx++; + } + + const timestamp = fmtTime(segment.start); + const speaker = segment.speaker !== undefined ? `Speaker ${segment.speaker + 1}` : ''; + + rows.push({ timestamp, speaker, text: segment.text }); + + if (segment.speakerChange && textParts.length > 0) { + textParts.push(''); + } + textParts.push(`${timestamp} ${segment.text}`); + } + + return { rows, plainText: textParts.join('\n') }; +} + +function fmtTime(sec: number): string { + const h = Math.floor(sec / 3600); + const m = Math.floor((sec % 3600) / 60); + const s = Math.floor(sec % 60); + if (h > 0) { + return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`; + } + return `${m}:${String(s).padStart(2, '0')}`; +} + +// ── Sentence grouping ───────────────────────────────────────────────────── + +// Max time span (seconds) for a single group when no sentence boundaries are found. +// Prevents unbounded merging for languages without punctuation (Chinese, etc.). +const MAX_GROUP_SPAN_SECONDS = 30; + +function groupBySentence( + segments: { start: number; text: string }[], +): GroupedSegment[] { + const groups: GroupedSegment[] = []; + let buffer = ''; + let bufferStart = 0; + let lastStart = 0; + + const flush = () => { + if (buffer.trim()) { + groups.push({ start: bufferStart, text: buffer.trim(), speakerChange: false }); + buffer = ''; + } + }; + + for (const seg of segments) { + // Large gap between segments — always flush + if (buffer && seg.start - lastStart > TRANSCRIPT_GROUP_GAP_SECONDS) { + flush(); + } + // Time-based flush: prevent unbounded groups for unpunctuated languages + if (buffer && seg.start - bufferStart > MAX_GROUP_SPAN_SECONDS) { + flush(); + } + if (!buffer) bufferStart = seg.start; + buffer += (buffer ? ' ' : '') + seg.text; + lastStart = seg.start; + if (SENTENCE_END.test(seg.text)) flush(); + } + flush(); + return groups; +} + +// ── Speaker grouping ────────────────────────────────────────────────────── + +function groupBySpeaker( + segments: { start: number; text: string }[], +): GroupedSegment[] { + type Turn = { + start: number; + segments: { start: number; text: string }[]; + speakerChange: boolean; + speaker?: number; + }; + + const turns: Turn[] = []; + let currentTurn: Turn | null = null; + let speakerIndex = -1; + let prevSegText = ''; + + for (const seg of segments) { + const isSpeakerChange = /^>>/.test(seg.text); + const cleanText = seg.text.replace(/^>>\s*/, '').replace(/^-\s+/, ''); + + const prevEndsWithComma = /,\s*$/.test(prevSegText); + const prevEndedSentence = (SENTENCE_END.test(prevSegText) || !prevSegText) && !prevEndsWithComma; + const isRealSpeakerChange = isSpeakerChange && prevEndedSentence; + + if (isRealSpeakerChange) { + if (currentTurn) turns.push(currentTurn); + speakerIndex = (speakerIndex + 1) % 2; + currentTurn = { + start: seg.start, + segments: [{ start: seg.start, text: cleanText }], + speakerChange: true, + speaker: speakerIndex, + }; + } else { + if (!currentTurn) { + currentTurn = { start: seg.start, segments: [], speakerChange: false }; + } + currentTurn.segments.push({ start: seg.start, text: cleanText }); + } + prevSegText = cleanText; + } + if (currentTurn) turns.push(currentTurn); + + splitAffirmativeTurns(turns); + + const groups: GroupedSegment[] = []; + for (const turn of turns) { + const sentenceGroups = turn.speaker === undefined + ? groupBySentence(turn.segments) + : mergeSentenceGroupsWithinTurn(groupBySentence(turn.segments)); + for (let i = 0; i < sentenceGroups.length; i++) { + groups.push({ + ...sentenceGroups[i], + speakerChange: i === 0 && turn.speakerChange, + speaker: turn.speaker, + }); + } + } + return groups; +} + +function splitAffirmativeTurns(turns: Array<{ + start: number; + segments: { start: number; text: string }[]; + speakerChange: boolean; + speaker?: number; +}>): void { + const affirmativePattern = /^(mhm|yeah|yes|yep|right|okay|ok|absolutely|sure|exactly|uh-huh|mm-hmm)[.!,]?\s+/i; + + for (let i = 0; i < turns.length; i++) { + const turn = turns[i]; + if (turn.speaker === undefined || turn.segments.length === 0) continue; + + const firstSeg = turn.segments[0]; + const match = affirmativePattern.exec(firstSeg.text); + if (!match) continue; + if (/,\s*$/.test(match[0])) continue; + + const remainder = firstSeg.text.slice(match[0].length).trim(); + const restSegments = turn.segments.slice(1); + const restWords = countWords(remainder) + restSegments.reduce((sum, s) => sum + countWords(s.text), 0); + if (restWords < 30) continue; + + const affirmativeText = match[0].trimEnd(); + const newRestSegments = remainder + ? [{ start: firstSeg.start, text: remainder }, ...restSegments] + : restSegments; + + turns.splice(i, 1, { + start: turn.start, + segments: [{ start: firstSeg.start, text: affirmativeText }], + speakerChange: turn.speakerChange, + speaker: turn.speaker, + }, { + start: newRestSegments[0].start, + segments: newRestSegments, + speakerChange: true, + speaker: turn.speaker === 0 ? 1 : 0, + }); + i++; + } +} + +function mergeSentenceGroupsWithinTurn(groups: GroupedSegment[]): GroupedSegment[] { + if (groups.length <= 1) return groups; + + const merged: GroupedSegment[] = []; + let current = { ...groups[0] }; + let currentIsFirstInTurn = true; + + for (let i = 1; i < groups.length; i++) { + const next = groups[i]; + if (shouldMergeSentenceGroups(current, next, currentIsFirstInTurn)) { + current.text = `${current.text} ${next.text}`; + continue; + } + merged.push(current); + current = { ...next }; + currentIsFirstInTurn = false; + } + merged.push(current); + return merged; +} + +function shouldMergeSentenceGroups( + current: { start: number; text: string }, + next: { start: number; text: string }, + currentIsFirstInTurn: boolean, +): boolean { + const currentWords = countWords(current.text); + const nextWords = countWords(next.text); + + if (isShortStandaloneUtterance(current.text, currentWords) + || isShortStandaloneUtterance(next.text, nextWords)) return false; + if (currentIsFirstInTurn && currentWords < FIRST_GROUP_MERGE_MIN_WORDS) return false; + if (QUESTION_END.test(current.text) || QUESTION_END.test(next.text)) return false; + if (currentWords + nextWords > TURN_MERGE_MAX_WORDS) return false; + if (next.start - current.start > TURN_MERGE_MAX_SPAN_SECONDS) return false; + return true; +} + +function isShortStandaloneUtterance(text: string, words?: number): boolean { + const w = words ?? countWords(text); + return w > 0 && w <= SHORT_UTTERANCE_MAX_WORDS && SENTENCE_END.test(text); +} diff --git a/src/clis/youtube/transcript.ts b/src/clis/youtube/transcript.ts new file mode 100644 index 0000000..29bf0ce --- /dev/null +++ b/src/clis/youtube/transcript.ts @@ -0,0 +1,280 @@ +/** + * YouTube transcript — uses InnerTube player API with Android client context. + * + * The Web client's caption URLs require a PoToken (proof of origin) generated + * by BotGuard at runtime. The Android client returns caption URLs that work + * without PoToken — same approach used by youtube-transcript-api (Python). + * + * Modes: + * --mode grouped (default): sentences merged, speaker detection, chapters + * --mode raw: every caption segment as-is with precise timestamps + */ +import { cli, Strategy } from '../../registry.js'; +import { parseVideoId } from './utils.js'; +import { + groupTranscriptSegments, + formatGroupedTranscript, + type RawSegment, + type Chapter, +} from './transcript-group.js'; + +cli({ + site: 'youtube', + name: 'transcript', + description: 'Get YouTube video transcript/subtitles', + domain: 'www.youtube.com', + strategy: Strategy.COOKIE, + args: [ + { name: 'url', required: true, help: 'YouTube video URL or video ID' }, + { name: 'lang', required: false, help: 'Language code (e.g. en, zh-Hans). Omit to auto-select' }, + { name: 'mode', required: false, default: 'grouped', help: 'Output mode: grouped (readable paragraphs) or raw (every segment)' }, + ], + // columns intentionally omitted — raw and grouped modes return different schemas, + // so we let the renderer auto-detect columns from the data keys. + func: async (page, kwargs) => { + const videoId = parseVideoId(kwargs.url); + const videoUrl = `https://www.youtube.com/watch?v=${videoId}`; + await page.goto(videoUrl); + await page.wait(3); + + const lang = kwargs.lang || ''; + const mode = kwargs.mode || 'grouped'; + + // Step 1: Get caption track URL via Android InnerTube API + const captionData = await page.evaluate(` + (async () => { + const cfg = window.ytcfg?.data_ || {}; + const apiKey = cfg.INNERTUBE_API_KEY; + if (!apiKey) return { error: 'INNERTUBE_API_KEY not found on page' }; + + const resp = await fetch('/youtubei/v1/player?key=' + apiKey + '&prettyPrint=false', { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + context: { client: { clientName: 'ANDROID', clientVersion: '20.10.38' } }, + videoId: ${JSON.stringify(videoId)} + }) + }); + + if (!resp.ok) return { error: 'InnerTube player API returned HTTP ' + resp.status }; + const data = await resp.json(); + + const renderer = data.captions?.playerCaptionsTracklistRenderer; + if (!renderer?.captionTracks?.length) { + return { error: 'No captions available for this video' }; + } + + const tracks = renderer.captionTracks; + const available = tracks.map(t => t.languageCode + (t.kind === 'asr' ? ' (auto)' : '')); + + const langPref = ${JSON.stringify(lang)}; + let track = null; + if (langPref) { + track = tracks.find(t => t.languageCode === langPref) + || tracks.find(t => t.languageCode.startsWith(langPref)); + } + if (!track) { + track = tracks.find(t => t.kind !== 'asr') || tracks[0]; + } + + return { + captionUrl: track.baseUrl, + language: track.languageCode, + kind: track.kind || 'manual', + available, + requestedLang: langPref || null, + langMatched: !!(langPref && track.languageCode === langPref), + langPrefixMatched: !!(langPref && track.languageCode !== langPref && track.languageCode.startsWith(langPref)) + }; + })() + `); + + if (!captionData || typeof captionData === 'string') { + throw new Error(`Failed to get caption info: ${typeof captionData === 'string' ? captionData : 'null response'}`); + } + if (captionData.error) { + throw new Error(`${captionData.error}${captionData.available ? ' (available: ' + captionData.available.join(', ') + ')' : ''}`); + } + + // Warn if --lang was specified but not matched + if (captionData.requestedLang && !captionData.langMatched && !captionData.langPrefixMatched) { + console.error(`Warning: --lang "${captionData.requestedLang}" not found. Using "${captionData.language}" instead. Available: ${captionData.available.join(', ')}`); + } + + // Step 2: Fetch caption XML and parse segments + const segments: RawSegment[] = await page.evaluate(` + (async () => { + const resp = await fetch(${JSON.stringify(captionData.captionUrl)}); + const xml = await resp.text(); + + if (!xml?.length) { + return { error: 'Caption URL returned empty response' }; + } + + function getAttr(tag, name) { + const needle = name + '="'; + const idx = tag.indexOf(needle); + if (idx === -1) return ''; + const valStart = idx + needle.length; + const valEnd = tag.indexOf('"', valStart); + if (valEnd === -1) return ''; + return tag.substring(valStart, valEnd); + } + + function decodeEntities(s) { + return s + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll(''', "'"); + } + + const isFormat3 = xml.includes('