feat(youtube): add video metadata and transcript commands (#25)
Add two new YouTube adapters: - **youtube video**: fetch metadata (title, views, description, etc.) from ytInitialPlayerResponse and ytInitialData - **youtube transcript**: fetch subtitles via Android InnerTube API to bypass PoToken requirement on Web client caption URLs - Two output modes: --mode grouped (sentence merging, speaker detection, chapter headings) and --mode raw (precise sub-second timestamps) - CJK support with 30s time-window fallback for unpunctuated captions - Language selection with --lang and stderr warning on fallback - URL normalization for watch, youtu.be, shorts, embed, live formats Co-authored-by: Ji Zhang <jizhang.work@gmail.com>
This commit is contained in:
5 files changed
+819
No files matched your search
@@ -0,0 +1,108 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { groupTranscriptSegments, formatGroupedTranscript } from './transcript-group.js';
|
||||
|
||||
describe('groupTranscriptSegments', () => {
|
||||
it('groups segments by sentence boundaries', () => {
|
||||
const segments = [
|
||||
{ start: 0, text: 'Hello there.' },
|
||||
{ start: 2, text: 'How are you doing today?' },
|
||||
{ start: 5, text: 'I am' },
|
||||
{ start: 6, text: 'doing well.' },
|
||||
];
|
||||
const result = groupTranscriptSegments(segments);
|
||||
expect(result).toHaveLength(3);
|
||||
expect(result[0].text).toBe('Hello there.');
|
||||
expect(result[1].text).toBe('How are you doing today?');
|
||||
expect(result[2].text).toBe('I am doing well.');
|
||||
});
|
||||
|
||||
it('flushes on large time gaps', () => {
|
||||
const segments = [
|
||||
{ start: 0, text: 'First part' },
|
||||
{ start: 2, text: 'still first' },
|
||||
{ start: 25, text: 'second part after gap' },
|
||||
];
|
||||
const result = groupTranscriptSegments(segments);
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].text).toBe('First part still first');
|
||||
expect(result[1].text).toBe('second part after gap');
|
||||
});
|
||||
|
||||
it('respects 30s max group span for unpunctuated text', () => {
|
||||
// Simulate CJK captions without punctuation
|
||||
const segments = Array.from({ length: 20 }, (_, i) => ({
|
||||
start: i * 2,
|
||||
text: `segment${i}`,
|
||||
}));
|
||||
const result = groupTranscriptSegments(segments);
|
||||
// 20 segments * 2s = 40s total, should be split into at least 2 groups
|
||||
expect(result.length).toBeGreaterThanOrEqual(2);
|
||||
// No single group should span more than ~30s
|
||||
for (const g of result) {
|
||||
const words = g.text.split(' ');
|
||||
// With 2s per segment and 30s max, each group should have at most ~16 segments
|
||||
expect(words.length).toBeLessThanOrEqual(16);
|
||||
}
|
||||
});
|
||||
|
||||
it('detects speaker changes via >> markers', () => {
|
||||
const segments = [
|
||||
{ start: 0, text: '>> How are you?' },
|
||||
{ start: 3, text: '>> I am fine.' },
|
||||
];
|
||||
const result = groupTranscriptSegments(segments);
|
||||
expect(result.some(g => g.speakerChange)).toBe(true);
|
||||
expect(result.some(g => g.speaker !== undefined)).toBe(true);
|
||||
});
|
||||
|
||||
it('recognizes CJK sentence-ending punctuation', () => {
|
||||
const segments = [
|
||||
{ start: 0, text: '你好世界。' },
|
||||
{ start: 2, text: '这是测试' },
|
||||
{ start: 4, text: '内容。' },
|
||||
];
|
||||
const result = groupTranscriptSegments(segments);
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].text).toBe('你好世界。');
|
||||
expect(result[1].text).toBe('这是测试 内容。');
|
||||
});
|
||||
|
||||
it('returns empty array for empty input', () => {
|
||||
expect(groupTranscriptSegments([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatGroupedTranscript', () => {
|
||||
it('formats timestamps correctly', () => {
|
||||
const segments = [
|
||||
{ start: 65, text: 'One minute five.', speakerChange: false },
|
||||
{ start: 3661, text: 'One hour one minute.', speakerChange: false },
|
||||
];
|
||||
const { rows } = formatGroupedTranscript(segments);
|
||||
expect(rows[0].timestamp).toBe('1:05');
|
||||
expect(rows[1].timestamp).toBe('1:01:01');
|
||||
});
|
||||
|
||||
it('inserts chapter headings at correct positions', () => {
|
||||
const segments = [
|
||||
{ start: 0, text: 'Intro text.', speakerChange: false },
|
||||
{ start: 60, text: 'Chapter content.', speakerChange: false },
|
||||
];
|
||||
const chapters = [{ title: 'Introduction', start: 0 }, { title: 'Main', start: 50 }];
|
||||
const { rows } = formatGroupedTranscript(segments, chapters);
|
||||
expect(rows[0].text).toBe('[Chapter] Introduction');
|
||||
expect(rows[1].text).toBe('Intro text.');
|
||||
expect(rows[2].text).toBe('[Chapter] Main');
|
||||
expect(rows[3].text).toBe('Chapter content.');
|
||||
});
|
||||
|
||||
it('labels speakers', () => {
|
||||
const segments = [
|
||||
{ start: 0, text: 'Hello.', speakerChange: true, speaker: 0 },
|
||||
{ start: 5, text: 'Hi there.', speakerChange: true, speaker: 1 },
|
||||
];
|
||||
const { rows } = formatGroupedTranscript(segments);
|
||||
expect(rows[0].speaker).toBe('Speaker 1');
|
||||
expect(rows[1].speaker).toBe('Speaker 2');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Transcript grouping: sentence merging, speaker detection, and chapter support.
|
||||
* Ported and simplified from Defuddle's YouTube extractor.
|
||||
*
|
||||
* Raw segments (2-3 second fragments) are grouped into readable paragraphs:
|
||||
* - Sentence boundaries: merge until sentence-ending punctuation (.!?)
|
||||
* - Speaker turns: detect ">>" markers from YouTube auto-captions
|
||||
* - Chapters: optional chapter headings inserted at appropriate timestamps
|
||||
*/
|
||||
|
||||
// Include CJK sentence-ending punctuation: 。！？ (fullwidth: ．！？)
|
||||
const SENTENCE_END = /[.!?\u3002\uFF01\uFF1F\uFF0E]["'\u2019\u201D)]*\s*$/;
|
||||
const QUESTION_END = /[?\uFF1F]["'\u2019\u201D)]*\s*$/;
|
||||
const TRANSCRIPT_GROUP_GAP_SECONDS = 20;
|
||||
const TURN_MERGE_MAX_WORDS = 80;
|
||||
const TURN_MERGE_MAX_SPAN_SECONDS = 45;
|
||||
const SHORT_UTTERANCE_MAX_WORDS = 3;
|
||||
const FIRST_GROUP_MERGE_MIN_WORDS = 8;
|
||||
|
||||
export interface RawSegment {
|
||||
start: number;
|
||||
end: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface GroupedSegment {
|
||||
start: number;
|
||||
text: string;
|
||||
speakerChange: boolean;
|
||||
speaker?: number;
|
||||
}
|
||||
|
||||
export interface Chapter {
|
||||
title: string;
|
||||
start: number;
|
||||
}
|
||||
|
||||
function countWords(text: string): number {
|
||||
return text.split(/\s+/).filter(Boolean).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Group raw transcript segments into readable blocks.
|
||||
* If speaker markers (>>) are present, groups by speaker turn.
|
||||
* Otherwise, groups by sentence boundaries.
|
||||
*/
|
||||
export function groupTranscriptSegments(
|
||||
segments: { start: number; text: string }[],
|
||||
): GroupedSegment[] {
|
||||
if (segments.length === 0) return [];
|
||||
const hasSpeakerMarkers = segments.some(s => /^>>/.test(s.text));
|
||||
return hasSpeakerMarkers ? groupBySpeaker(segments) : groupBySentence(segments);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format grouped segments + chapters into a final text output.
|
||||
*/
|
||||
export function formatGroupedTranscript(
|
||||
segments: GroupedSegment[],
|
||||
chapters: Chapter[] = [],
|
||||
): { rows: Array<{ timestamp: string; speaker: string; text: string }>; plainText: string } {
|
||||
const sortedChapters = [...chapters].sort((a, b) => a.start - b.start);
|
||||
let chapterIdx = 0;
|
||||
|
||||
const rows: Array<{ timestamp: string; speaker: string; text: string }> = [];
|
||||
const textParts: string[] = [];
|
||||
|
||||
for (const segment of segments) {
|
||||
// Insert chapter headings
|
||||
while (chapterIdx < sortedChapters.length && sortedChapters[chapterIdx].start <= segment.start) {
|
||||
const title = sortedChapters[chapterIdx].title;
|
||||
rows.push({ timestamp: fmtTime(sortedChapters[chapterIdx].start), speaker: '', text: `[Chapter] ${title}` });
|
||||
if (textParts.length > 0) textParts.push('');
|
||||
textParts.push(`### ${title}`);
|
||||
textParts.push('');
|
||||
chapterIdx++;
|
||||
}
|
||||
|
||||
const timestamp = fmtTime(segment.start);
|
||||
const speaker = segment.speaker !== undefined ? `Speaker ${segment.speaker + 1}` : '';
|
||||
|
||||
rows.push({ timestamp, speaker, text: segment.text });
|
||||
|
||||
if (segment.speakerChange && textParts.length > 0) {
|
||||
textParts.push('');
|
||||
}
|
||||
textParts.push(`${timestamp} ${segment.text}`);
|
||||
}
|
||||
|
||||
return { rows, plainText: textParts.join('\n') };
|
||||
}
|
||||
|
||||
function fmtTime(sec: number): string {
|
||||
const h = Math.floor(sec / 3600);
|
||||
const m = Math.floor((sec % 3600) / 60);
|
||||
const s = Math.floor(sec % 60);
|
||||
if (h > 0) {
|
||||
return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
|
||||
}
|
||||
return `${m}:${String(s).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
// ── Sentence grouping ─────────────────────────────────────────────────────
|
||||
|
||||
// Max time span (seconds) for a single group when no sentence boundaries are found.
|
||||
// Prevents unbounded merging for languages without punctuation (Chinese, etc.).
|
||||
const MAX_GROUP_SPAN_SECONDS = 30;
|
||||
|
||||
function groupBySentence(
|
||||
segments: { start: number; text: string }[],
|
||||
): GroupedSegment[] {
|
||||
const groups: GroupedSegment[] = [];
|
||||
let buffer = '';
|
||||
let bufferStart = 0;
|
||||
let lastStart = 0;
|
||||
|
||||
const flush = () => {
|
||||
if (buffer.trim()) {
|
||||
groups.push({ start: bufferStart, text: buffer.trim(), speakerChange: false });
|
||||
buffer = '';
|
||||
}
|
||||
};
|
||||
|
||||
for (const seg of segments) {
|
||||
// Large gap between segments — always flush
|
||||
if (buffer && seg.start - lastStart > TRANSCRIPT_GROUP_GAP_SECONDS) {
|
||||
flush();
|
||||
}
|
||||
// Time-based flush: prevent unbounded groups for unpunctuated languages
|
||||
if (buffer && seg.start - bufferStart > MAX_GROUP_SPAN_SECONDS) {
|
||||
flush();
|
||||
}
|
||||
if (!buffer) bufferStart = seg.start;
|
||||
buffer += (buffer ? ' ' : '') + seg.text;
|
||||
lastStart = seg.start;
|
||||
if (SENTENCE_END.test(seg.text)) flush();
|
||||
}
|
||||
flush();
|
||||
return groups;
|
||||
}
|
||||
|
||||
// ── Speaker grouping ──────────────────────────────────────────────────────
|
||||
|
||||
function groupBySpeaker(
|
||||
segments: { start: number; text: string }[],
|
||||
): GroupedSegment[] {
|
||||
type Turn = {
|
||||
start: number;
|
||||
segments: { start: number; text: string }[];
|
||||
speakerChange: boolean;
|
||||
speaker?: number;
|
||||
};
|
||||
|
||||
const turns: Turn[] = [];
|
||||
let currentTurn: Turn | null = null;
|
||||
let speakerIndex = -1;
|
||||
let prevSegText = '';
|
||||
|
||||
for (const seg of segments) {
|
||||
const isSpeakerChange = /^>>/.test(seg.text);
|
||||
const cleanText = seg.text.replace(/^>>\s*/, '').replace(/^-\s+/, '');
|
||||
|
||||
const prevEndsWithComma = /,\s*$/.test(prevSegText);
|
||||
const prevEndedSentence = (SENTENCE_END.test(prevSegText) || !prevSegText) && !prevEndsWithComma;
|
||||
const isRealSpeakerChange = isSpeakerChange && prevEndedSentence;
|
||||
|
||||
if (isRealSpeakerChange) {
|
||||
if (currentTurn) turns.push(currentTurn);
|
||||
speakerIndex = (speakerIndex + 1) % 2;
|
||||
currentTurn = {
|
||||
start: seg.start,
|
||||
segments: [{ start: seg.start, text: cleanText }],
|
||||
speakerChange: true,
|
||||
speaker: speakerIndex,
|
||||
};
|
||||
} else {
|
||||
if (!currentTurn) {
|
||||
currentTurn = { start: seg.start, segments: [], speakerChange: false };
|
||||
}
|
||||
currentTurn.segments.push({ start: seg.start, text: cleanText });
|
||||
}
|
||||
prevSegText = cleanText;
|
||||
}
|
||||
if (currentTurn) turns.push(currentTurn);
|
||||
|
||||
splitAffirmativeTurns(turns);
|
||||
|
||||
const groups: GroupedSegment[] = [];
|
||||
for (const turn of turns) {
|
||||
const sentenceGroups = turn.speaker === undefined
|
||||
? groupBySentence(turn.segments)
|
||||
: mergeSentenceGroupsWithinTurn(groupBySentence(turn.segments));
|
||||
for (let i = 0; i < sentenceGroups.length; i++) {
|
||||
groups.push({
|
||||
...sentenceGroups[i],
|
||||
speakerChange: i === 0 && turn.speakerChange,
|
||||
speaker: turn.speaker,
|
||||
});
|
||||
}
|
||||
}
|
||||
return groups;
|
||||
}
|
||||
|
||||
function splitAffirmativeTurns(turns: Array<{
|
||||
start: number;
|
||||
segments: { start: number; text: string }[];
|
||||
speakerChange: boolean;
|
||||
speaker?: number;
|
||||
}>): void {
|
||||
const affirmativePattern = /^(mhm|yeah|yes|yep|right|okay|ok|absolutely|sure|exactly|uh-huh|mm-hmm)[.!,]?\s+/i;
|
||||
|
||||
for (let i = 0; i < turns.length; i++) {
|
||||
const turn = turns[i];
|
||||
if (turn.speaker === undefined || turn.segments.length === 0) continue;
|
||||
|
||||
const firstSeg = turn.segments[0];
|
||||
const match = affirmativePattern.exec(firstSeg.text);
|
||||
if (!match) continue;
|
||||
if (/,\s*$/.test(match[0])) continue;
|
||||
|
||||
const remainder = firstSeg.text.slice(match[0].length).trim();
|
||||
const restSegments = turn.segments.slice(1);
|
||||
const restWords = countWords(remainder) + restSegments.reduce((sum, s) => sum + countWords(s.text), 0);
|
||||
if (restWords < 30) continue;
|
||||
|
||||
const affirmativeText = match[0].trimEnd();
|
||||
const newRestSegments = remainder
|
||||
? [{ start: firstSeg.start, text: remainder }, ...restSegments]
|
||||
: restSegments;
|
||||
|
||||
turns.splice(i, 1, {
|
||||
start: turn.start,
|
||||
segments: [{ start: firstSeg.start, text: affirmativeText }],
|
||||
speakerChange: turn.speakerChange,
|
||||
speaker: turn.speaker,
|
||||
}, {
|
||||
start: newRestSegments[0].start,
|
||||
segments: newRestSegments,
|
||||
speakerChange: true,
|
||||
speaker: turn.speaker === 0 ? 1 : 0,
|
||||
});
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
function mergeSentenceGroupsWithinTurn(groups: GroupedSegment[]): GroupedSegment[] {
|
||||
if (groups.length <= 1) return groups;
|
||||
|
||||
const merged: GroupedSegment[] = [];
|
||||
let current = { ...groups[0] };
|
||||
let currentIsFirstInTurn = true;
|
||||
|
||||
for (let i = 1; i < groups.length; i++) {
|
||||
const next = groups[i];
|
||||
if (shouldMergeSentenceGroups(current, next, currentIsFirstInTurn)) {
|
||||
current.text = `${current.text} ${next.text}`;
|
||||
continue;
|
||||
}
|
||||
merged.push(current);
|
||||
current = { ...next };
|
||||
currentIsFirstInTurn = false;
|
||||
}
|
||||
merged.push(current);
|
||||
return merged;
|
||||
}
|
||||
|
||||
function shouldMergeSentenceGroups(
|
||||
current: { start: number; text: string },
|
||||
next: { start: number; text: string },
|
||||
currentIsFirstInTurn: boolean,
|
||||
): boolean {
|
||||
const currentWords = countWords(current.text);
|
||||
const nextWords = countWords(next.text);
|
||||
|
||||
if (isShortStandaloneUtterance(current.text, currentWords)
|
||||
|| isShortStandaloneUtterance(next.text, nextWords)) return false;
|
||||
if (currentIsFirstInTurn && currentWords < FIRST_GROUP_MERGE_MIN_WORDS) return false;
|
||||
if (QUESTION_END.test(current.text) || QUESTION_END.test(next.text)) return false;
|
||||
if (currentWords + nextWords > TURN_MERGE_MAX_WORDS) return false;
|
||||
if (next.start - current.start > TURN_MERGE_MAX_SPAN_SECONDS) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function isShortStandaloneUtterance(text: string, words?: number): boolean {
|
||||
const w = words ?? countWords(text);
|
||||
return w > 0 && w <= SHORT_UTTERANCE_MAX_WORDS && SENTENCE_END.test(text);
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* YouTube transcript — uses InnerTube player API with Android client context.
|
||||
*
|
||||
* The Web client's caption URLs require a PoToken (proof of origin) generated
|
||||
* by BotGuard at runtime. The Android client returns caption URLs that work
|
||||
* without PoToken — same approach used by youtube-transcript-api (Python).
|
||||
*
|
||||
* Modes:
|
||||
* --mode grouped (default): sentences merged, speaker detection, chapters
|
||||
* --mode raw: every caption segment as-is with precise timestamps
|
||||
*/
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { parseVideoId } from './utils.js';
|
||||
import {
|
||||
groupTranscriptSegments,
|
||||
formatGroupedTranscript,
|
||||
type RawSegment,
|
||||
type Chapter,
|
||||
} from './transcript-group.js';
|
||||
|
||||
cli({
|
||||
site: 'youtube',
|
||||
name: 'transcript',
|
||||
description: 'Get YouTube video transcript/subtitles',
|
||||
domain: 'www.youtube.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'url', required: true, help: 'YouTube video URL or video ID' },
|
||||
{ name: 'lang', required: false, help: 'Language code (e.g. en, zh-Hans). Omit to auto-select' },
|
||||
{ name: 'mode', required: false, default: 'grouped', help: 'Output mode: grouped (readable paragraphs) or raw (every segment)' },
|
||||
],
|
||||
// columns intentionally omitted — raw and grouped modes return different schemas,
|
||||
// so we let the renderer auto-detect columns from the data keys.
|
||||
func: async (page, kwargs) => {
|
||||
const videoId = parseVideoId(kwargs.url);
|
||||
const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||
await page.goto(videoUrl);
|
||||
await page.wait(3);
|
||||
|
||||
const lang = kwargs.lang || '';
|
||||
const mode = kwargs.mode || 'grouped';
|
||||
|
||||
// Step 1: Get caption track URL via Android InnerTube API
|
||||
const captionData = await page.evaluate(`
|
||||
(async () => {
|
||||
const cfg = window.ytcfg?.data_ || {};
|
||||
const apiKey = cfg.INNERTUBE_API_KEY;
|
||||
if (!apiKey) return { error: 'INNERTUBE_API_KEY not found on page' };
|
||||
|
||||
const resp = await fetch('/youtubei/v1/player?key=' + apiKey + '&prettyPrint=false', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
context: { client: { clientName: 'ANDROID', clientVersion: '20.10.38' } },
|
||||
videoId: ${JSON.stringify(videoId)}
|
||||
})
|
||||
});
|
||||
|
||||
if (!resp.ok) return { error: 'InnerTube player API returned HTTP ' + resp.status };
|
||||
const data = await resp.json();
|
||||
|
||||
const renderer = data.captions?.playerCaptionsTracklistRenderer;
|
||||
if (!renderer?.captionTracks?.length) {
|
||||
return { error: 'No captions available for this video' };
|
||||
}
|
||||
|
||||
const tracks = renderer.captionTracks;
|
||||
const available = tracks.map(t => t.languageCode + (t.kind === 'asr' ? ' (auto)' : ''));
|
||||
|
||||
const langPref = ${JSON.stringify(lang)};
|
||||
let track = null;
|
||||
if (langPref) {
|
||||
track = tracks.find(t => t.languageCode === langPref)
|
||||
|| tracks.find(t => t.languageCode.startsWith(langPref));
|
||||
}
|
||||
if (!track) {
|
||||
track = tracks.find(t => t.kind !== 'asr') || tracks[0];
|
||||
}
|
||||
|
||||
return {
|
||||
captionUrl: track.baseUrl,
|
||||
language: track.languageCode,
|
||||
kind: track.kind || 'manual',
|
||||
available,
|
||||
requestedLang: langPref || null,
|
||||
langMatched: !!(langPref && track.languageCode === langPref),
|
||||
langPrefixMatched: !!(langPref && track.languageCode !== langPref && track.languageCode.startsWith(langPref))
|
||||
};
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!captionData || typeof captionData === 'string') {
|
||||
throw new Error(`Failed to get caption info: ${typeof captionData === 'string' ? captionData : 'null response'}`);
|
||||
}
|
||||
if (captionData.error) {
|
||||
throw new Error(`${captionData.error}${captionData.available ? ' (available: ' + captionData.available.join(', ') + ')' : ''}`);
|
||||
}
|
||||
|
||||
// Warn if --lang was specified but not matched
|
||||
if (captionData.requestedLang && !captionData.langMatched && !captionData.langPrefixMatched) {
|
||||
console.error(`Warning: --lang "${captionData.requestedLang}" not found. Using "${captionData.language}" instead. Available: ${captionData.available.join(', ')}`);
|
||||
}
|
||||
|
||||
// Step 2: Fetch caption XML and parse segments
|
||||
const segments: RawSegment[] = await page.evaluate(`
|
||||
(async () => {
|
||||
const resp = await fetch(${JSON.stringify(captionData.captionUrl)});
|
||||
const xml = await resp.text();
|
||||
|
||||
if (!xml?.length) {
|
||||
return { error: 'Caption URL returned empty response' };
|
||||
}
|
||||
|
||||
function getAttr(tag, name) {
|
||||
const needle = name + '="';
|
||||
const idx = tag.indexOf(needle);
|
||||
if (idx === -1) return '';
|
||||
const valStart = idx + needle.length;
|
||||
const valEnd = tag.indexOf('"', valStart);
|
||||
if (valEnd === -1) return '';
|
||||
return tag.substring(valStart, valEnd);
|
||||
}
|
||||
|
||||
function decodeEntities(s) {
|
||||
return s
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll(''', "'");
|
||||
}
|
||||
|
||||
const isFormat3 = xml.includes('<p t="');
|
||||
const marker = isFormat3 ? '<p ' : '<text ';
|
||||
const endMarker = isFormat3 ? '</p>' : '</text>';
|
||||
const results = [];
|
||||
let pos = 0;
|
||||
|
||||
while (true) {
|
||||
const tagStart = xml.indexOf(marker, pos);
|
||||
if (tagStart === -1) break;
|
||||
let contentStart = xml.indexOf('>', tagStart);
|
||||
if (contentStart === -1) break;
|
||||
contentStart += 1;
|
||||
const tagEnd = xml.indexOf(endMarker, contentStart);
|
||||
if (tagEnd === -1) break;
|
||||
|
||||
const attrStr = xml.substring(tagStart + marker.length, contentStart - 1);
|
||||
const content = xml.substring(contentStart, tagEnd);
|
||||
|
||||
let startSec, durSec;
|
||||
if (isFormat3) {
|
||||
startSec = (parseFloat(getAttr(attrStr, 't')) || 0) / 1000;
|
||||
durSec = (parseFloat(getAttr(attrStr, 'd')) || 0) / 1000;
|
||||
} else {
|
||||
startSec = parseFloat(getAttr(attrStr, 'start')) || 0;
|
||||
durSec = parseFloat(getAttr(attrStr, 'dur')) || 0;
|
||||
}
|
||||
|
||||
// Strip inner tags (e.g. <s> in srv3 format) and decode entities
|
||||
const text = decodeEntities(content.replace(/<[^>]+>/g, '')).split('\\\\n').join(' ').trim();
|
||||
if (text) {
|
||||
results.push({ start: startSec, end: startSec + durSec, text });
|
||||
}
|
||||
|
||||
pos = tagEnd + endMarker.length;
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
return { error: 'Parsed 0 segments from caption XML' };
|
||||
}
|
||||
|
||||
return results;
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!Array.isArray(segments)) {
|
||||
throw new Error((segments as any)?.error || 'Failed to parse caption segments');
|
||||
}
|
||||
if (segments.length === 0) {
|
||||
throw new Error('No caption segments found');
|
||||
}
|
||||
|
||||
// Step 3: Fetch chapters (for grouped mode)
|
||||
let chapters: Chapter[] = [];
|
||||
if (mode === 'grouped') {
|
||||
try {
|
||||
const chapterData = await page.evaluate(`
|
||||
(async () => {
|
||||
const cfg = window.ytcfg?.data_ || {};
|
||||
const apiKey = cfg.INNERTUBE_API_KEY;
|
||||
if (!apiKey) return [];
|
||||
|
||||
const resp = await fetch('/youtubei/v1/next?key=' + apiKey + '&prettyPrint=false', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
context: { client: { clientName: 'WEB', clientVersion: '2.20240101.00.00' } },
|
||||
videoId: ${JSON.stringify(videoId)}
|
||||
})
|
||||
});
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json();
|
||||
|
||||
const chapters = [];
|
||||
|
||||
// Try chapterRenderer from player bar
|
||||
const panels = data.playerOverlays?.playerOverlayRenderer
|
||||
?.decoratedPlayerBarRenderer?.decoratedPlayerBarRenderer
|
||||
?.playerBar?.multiMarkersPlayerBarRenderer?.markersMap;
|
||||
|
||||
if (Array.isArray(panels)) {
|
||||
for (const panel of panels) {
|
||||
const markers = panel.value?.chapters;
|
||||
if (!Array.isArray(markers)) continue;
|
||||
for (const marker of markers) {
|
||||
const ch = marker.chapterRenderer;
|
||||
if (!ch) continue;
|
||||
const title = ch.title?.simpleText || '';
|
||||
const startMs = ch.timeRangeStartMillis;
|
||||
if (title && typeof startMs === 'number') {
|
||||
chapters.push({ title, start: startMs / 1000 });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (chapters.length > 0) return chapters;
|
||||
|
||||
// Fallback: macroMarkersListItemRenderer from engagement panels
|
||||
const engPanels = data.engagementPanels;
|
||||
if (!Array.isArray(engPanels)) return [];
|
||||
for (const ep of engPanels) {
|
||||
const content = ep.engagementPanelSectionListRenderer?.content;
|
||||
const items = content?.macroMarkersListRenderer?.contents;
|
||||
if (!Array.isArray(items)) continue;
|
||||
for (const item of items) {
|
||||
const renderer = item.macroMarkersListItemRenderer;
|
||||
if (!renderer) continue;
|
||||
const t = renderer.title?.simpleText || '';
|
||||
const ts = renderer.timeDescription?.simpleText || '';
|
||||
if (!t || !ts) continue;
|
||||
const parts = ts.split(':').map(Number);
|
||||
let secs = null;
|
||||
if (parts.length === 3 && parts.every(n => !isNaN(n))) secs = parts[0]*3600 + parts[1]*60 + parts[2];
|
||||
else if (parts.length === 2 && parts.every(n => !isNaN(n))) secs = parts[0]*60 + parts[1];
|
||||
if (secs !== null) chapters.push({ title: t, start: secs });
|
||||
}
|
||||
}
|
||||
return chapters;
|
||||
})()
|
||||
`);
|
||||
if (Array.isArray(chapterData)) {
|
||||
chapters = chapterData;
|
||||
}
|
||||
} catch {
|
||||
// Chapters are optional — proceed without them
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Format output based on mode
|
||||
if (mode === 'raw') {
|
||||
// Precise timestamps in seconds with decimals, matching bilibili/subtitle format
|
||||
return segments.map((seg, i) => ({
|
||||
index: i + 1,
|
||||
start: Number(seg.start).toFixed(2) + 's',
|
||||
end: Number(seg.end).toFixed(2) + 's',
|
||||
text: seg.text,
|
||||
}));
|
||||
}
|
||||
|
||||
// Grouped mode: merge sentences, detect speakers, insert chapters
|
||||
const grouped = groupTranscriptSegments(
|
||||
segments.map(s => ({ start: s.start, text: s.text })),
|
||||
);
|
||||
const { rows } = formatGroupedTranscript(grouped, chapters);
|
||||
return rows;
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Shared YouTube utilities — URL parsing, video ID extraction, etc.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extract a YouTube video ID from a URL or bare video ID string.
|
||||
* Supports: watch?v=, youtu.be/, /shorts/, /embed/, /live/, /v/
|
||||
*/
|
||||
export function parseVideoId(input: string): string {
|
||||
if (!input.startsWith('http')) return input;
|
||||
|
||||
try {
|
||||
const parsed = new URL(input);
|
||||
if (parsed.searchParams.has('v')) {
|
||||
return parsed.searchParams.get('v')!;
|
||||
}
|
||||
if (parsed.hostname === 'youtu.be') {
|
||||
return parsed.pathname.slice(1).split('/')[0];
|
||||
}
|
||||
// Handle /shorts/xxx, /embed/xxx, /live/xxx, /v/xxx
|
||||
const pathMatch = parsed.pathname.match(/^\/(shorts|embed|live|v)\/([^/?]+)/);
|
||||
if (pathMatch) return pathMatch[2];
|
||||
} catch {
|
||||
// Not a valid URL — treat entire input as video ID
|
||||
}
|
||||
|
||||
return input;
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* YouTube video metadata — read ytInitialPlayerResponse + ytInitialData from video page.
|
||||
*/
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { parseVideoId } from './utils.js';
|
||||
|
||||
cli({
|
||||
site: 'youtube',
|
||||
name: 'video',
|
||||
description: 'Get YouTube video metadata (title, views, description, etc.)',
|
||||
domain: 'www.youtube.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'url', required: true, help: 'YouTube video URL or video ID' },
|
||||
],
|
||||
columns: ['field', 'value'],
|
||||
func: async (page, kwargs) => {
|
||||
const videoId = parseVideoId(kwargs.url);
|
||||
const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
|
||||
await page.goto(videoUrl);
|
||||
await page.wait(3);
|
||||
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const player = window.ytInitialPlayerResponse;
|
||||
const yt = window.ytInitialData;
|
||||
if (!player) return { error: 'ytInitialPlayerResponse not found' };
|
||||
|
||||
const details = player.videoDetails || {};
|
||||
const microformat = player.microformat?.playerMicroformatRenderer || {};
|
||||
|
||||
// Try to get full description from ytInitialData
|
||||
let fullDescription = details.shortDescription || '';
|
||||
try {
|
||||
const contents = yt?.contents?.twoColumnWatchNextResults
|
||||
?.results?.results?.contents;
|
||||
if (contents) {
|
||||
for (const c of contents) {
|
||||
const desc = c.videoSecondaryInfoRenderer?.attributedDescription?.content;
|
||||
if (desc) { fullDescription = desc; break; }
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Get like count if available
|
||||
let likes = '';
|
||||
try {
|
||||
const contents = yt?.contents?.twoColumnWatchNextResults
|
||||
?.results?.results?.contents;
|
||||
if (contents) {
|
||||
for (const c of contents) {
|
||||
const buttons = c.videoPrimaryInfoRenderer?.videoActions
|
||||
?.menuRenderer?.topLevelButtons;
|
||||
if (buttons) {
|
||||
for (const b of buttons) {
|
||||
const toggle = b.segmentedLikeDislikeButtonViewModel
|
||||
?.likeButtonViewModel?.likeButtonViewModel?.toggleButtonViewModel
|
||||
?.toggleButtonViewModel?.defaultButtonViewModel?.buttonViewModel;
|
||||
if (toggle?.title) { likes = toggle.title; break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Get publish date
|
||||
const publishDate = microformat.publishDate
|
||||
|| microformat.uploadDate
|
||||
|| details.publishDate || '';
|
||||
|
||||
// Get category
|
||||
const category = microformat.category || '';
|
||||
|
||||
// Get channel subscriber count if available
|
||||
let subscribers = '';
|
||||
try {
|
||||
const contents = yt?.contents?.twoColumnWatchNextResults
|
||||
?.results?.results?.contents;
|
||||
if (contents) {
|
||||
for (const c of contents) {
|
||||
const owner = c.videoSecondaryInfoRenderer?.owner
|
||||
?.videoOwnerRenderer?.subscriberCountText?.simpleText;
|
||||
if (owner) { subscribers = owner; break; }
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
return {
|
||||
title: details.title || '',
|
||||
channel: details.author || '',
|
||||
channelId: details.channelId || '',
|
||||
videoId: details.videoId || '',
|
||||
views: details.viewCount || '',
|
||||
likes,
|
||||
subscribers,
|
||||
duration: details.lengthSeconds ? details.lengthSeconds + 's' : '',
|
||||
publishDate,
|
||||
category,
|
||||
description: fullDescription,
|
||||
keywords: (details.keywords || []).join(', '),
|
||||
isLive: details.isLiveContent || false,
|
||||
thumbnail: details.thumbnail?.thumbnails?.slice(-1)?.[0]?.url || '',
|
||||
};
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!data || typeof data !== 'object') throw new Error('Failed to extract video metadata from page');
|
||||
if (data.error) throw new Error(data.error);
|
||||
|
||||
// Return as field/value pairs for table display
|
||||
return Object.entries(data).map(([field, value]) => ({
|
||||
field,
|
||||
value: String(value),
|
||||
}));
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user