feat(snapshot): filter contentinfo subtrees, bilibili ad URLs, boilerplate buttons

- Add contentinfo to subtree-level noise filtering (biggest single win)
  - Reuters: 51% → 62%, Google: 57% → 70%, Netflix: 48% → 60%
- Add cm.bilibili.com/cm/api/fees/ ad URL pattern
- Add 广告 keyword to ad detection
- Add back-to-top / 回到顶部 boilerplate button filtering
- Unify ad/boilerplate/contentinfo into single subtree-skip mechanism
- Add vitest config and comprehensive test suite (33 tests)
- Fixture tests skip gracefully when snapshot files are absent

Bump to v0.7.1
This commit is contained in:
jackwener committed 2026-03-17 01:26:49 +08:00
1 parent c2a5cbe90e
commit fbec2f6f5d
4 files changed
+998 -17

No files matched your search

+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@jackwener/opencli",
"version": "0.7.0",
"version": "0.7.1",
"publishConfig": {
"access": "public"
},
+579
View File
@@ -0,0 +1,579 @@
/**
* Tests for snapshotFormatter.ts: Playwright MCP snapshot tree filtering.
*
* Uses sanitized excerpts from real websites (GitHub, Bilibili, Twitter)
* to validate noise filtering, annotation stripping, and output quality.
*/
import { describe, it, expect } from 'vitest';
import { formatSnapshot } from './snapshotFormatter.js';
// ---------------------------------------------------------------------------
// Fixtures: sanitized excerpts from real Playwright MCP snapshots
// ---------------------------------------------------------------------------
/** GitHub dashboard navigation bar (generic-heavy, refs, /url: lines) */
const GITHUB_NAV = `\
- generic [ref=e2]:
- region
- generic [ref=e3]:
- link "Skip to content" [ref=e4] [cursor=pointer]:
- /url: "#start-of-content"
- banner "Global Navigation Menu" [ref=e8]:
- generic [ref=e9]:
- generic [ref=e10]:
- button "Open menu" [ref=e12] [cursor=pointer]:
- img [ref=e13]
- link "Homepage" [ref=e15] [cursor=pointer]:
- /url: /
- img [ref=e16]
- generic [ref=e18]:
- navigation "Breadcrumbs" [ref=e19]:
- list [ref=e20]:
- listitem [ref=e21]:
- link "Dashboard" [ref=e22] [cursor=pointer]:
- /url: https://github.com/
- generic [ref=e23]: Dashboard
- button "Search or jump to…" [ref=e26] [cursor=pointer]:
- generic [ref=e27]:
- generic:
- img
- generic [ref=e28]:
- generic:
- text: Type
- generic: /
- text: to search`;
/** GitHub repo list sidebar (repetitive structure) */
const GITHUB_REPOS = `\
- navigation "Repositories" [ref=e79]:
- generic [ref=e80]:
- generic [ref=e81]:
- heading "Top repositories" [level=2] [ref=e82]
- link "New" [ref=e83] [cursor=pointer]:
- /url: /new
- generic [ref=e84]:
- generic:
- img
- generic [ref=e85]: New
- search "Top repositories" [ref=e86]:
- textbox "Find a repository…" [ref=e87]
- list [ref=e88]:
- listitem [ref=e89]:
- generic [ref=e90]:
- link "Repository" [ref=e91] [cursor=pointer]:
- /url: /jackwener/twitter-cli
- img "Repository" [ref=e92]
- link "jackwener/twitter-cli" [ref=e94] [cursor=pointer]:
- /url: /jackwener/twitter-cli
- listitem [ref=e95]:
- generic [ref=e96]:
- link "Repository" [ref=e97] [cursor=pointer]:
- /url: /jackwener/opencli
- img "Repository" [ref=e98]
- link "jackwener/opencli" [ref=e100] [cursor=pointer]:
- /url: /jackwener/opencli`;
/** Bilibili nav bar (Chinese text, multiple link categories) */
const BILIBILI_NAV = `\
- generic [ref=e3]:
- generic [ref=e4]:
- generic [ref=e5]:
- list [ref=e6]:
- listitem [ref=e7]:
- link "首页" [ref=e8] [cursor=pointer]:
- /url: //www.bilibili.com
- img [ref=e9]
- generic [ref=e11]: 首页
- listitem [ref=e12]:
- link "番剧" [ref=e13] [cursor=pointer]:
- /url: //www.bilibili.com/anime/
- listitem [ref=e14]:
- link "直播" [ref=e15] [cursor=pointer]:
- /url: //live.bilibili.com
- generic [ref=e32]:
- textbox "冷知识 金廷26年胜率100%" [ref=e34]
- img [ref=e36] [cursor=pointer]`;
/** Bilibili video card (deeply nested generic wrappers, view counts) */
const BILIBILI_VIDEO = `\
- generic [ref=e363]:
- link "超酷时刻 即将到来 3.3万 40 16:24" [ref=e364] [cursor=pointer]:
- /url: https://www.bilibili.com/video/BV1zVw5zoEFt
- generic [ref=e365]:
- img "超酷时刻 即将到来" [ref=e368]
- generic:
- generic:
- generic:
- generic:
- img
- generic: 3.3万
- generic:
- img
- generic: "40"
- generic: 16:24
- generic [ref=e370]:
- heading "超酷时刻 即将到来" [level=3] [ref=e371]:
- link "超酷时刻 即将到来" [ref=e372] [cursor=pointer]:
- /url: https://www.bilibili.com/video/BV1zVw5zoEFt
- link "Tesla特斯拉中国 · 13小时前" [ref=e374] [cursor=pointer]:
- /url: //space.bilibili.com/491190876
- img [ref=e375]
- generic "Tesla特斯拉中国" [ref=e379]
- generic [ref=e380]: · 13小时前`;
/** Empty paragraph blocks (Bilibili bottom section) */
const BILIBILI_EMPTY = `\
- generic [ref=e576]:
- generic:
- generic:
- generic:
- paragraph
- paragraph
- paragraph
- generic [ref=e577]:
- generic:
- generic:
- generic:
- paragraph
- paragraph
- paragraph`;
/** Twitter-style feed item (simulated based on common patterns) */
const TWITTER_TWEET = `\
- main [ref=e100]:
- region "Timeline" [ref=e101]:
- article [ref=e200]:
- generic [ref=e201]:
- generic [ref=e202]:
- link "@elonmusk" [ref=e203] [cursor=pointer]:
- /url: /elonmusk
- img "@elonmusk" [ref=e204]
- generic [ref=e205]:
- generic [ref=e206]: Elon Musk
- generic [ref=e207]: @elonmusk
- generic [ref=e208]:
- generic [ref=e209]: This is a very long tweet that goes on and on about various things including technology, space, and other random topics that make this text exceed any reasonable length limit we might want to set for display purposes in a CLI interface.
- generic [ref=e210]:
- button "Reply" [ref=e211] [cursor=pointer]:
- img [ref=e212]
- generic [ref=e213]: "42"
- button "Retweet" [ref=e214] [cursor=pointer]:
- img [ref=e215]
- generic [ref=e216]: "1.2K"
- button "Like" [ref=e217] [cursor=pointer]:
- img [ref=e218]
- generic [ref=e219]: "5.3K"
- separator [ref=e300]`;
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('formatSnapshot', () => {
describe('basic behavior', () => {
it('returns empty string for empty/null input', () => {
expect(formatSnapshot('')).toBe('');
expect(formatSnapshot(null as any)).toBe('');
expect(formatSnapshot(undefined as any)).toBe('');
});
it('strips [ref=...] and [cursor=...] annotations', () => {
const input = '- button "Click me" [ref=e42] [cursor=pointer]';
const result = formatSnapshot(input);
expect(result).not.toContain('[ref=');
expect(result).not.toContain('[cursor=');
expect(result).toContain('button "Click me"');
});
it('removes /url: metadata lines', () => {
const input = `\
- link "Home" [ref=e1] [cursor=pointer]:
- /url: https://example.com
- generic [ref=e2]: Home`;
const result = formatSnapshot(input);
expect(result).not.toContain('/url:');
expect(result).not.toContain('https://example.com');
});
it('assigns sequential [@N] refs to interactive elements', () => {
const input = `\
- button "Save" [ref=e1]
- link "Cancel" [ref=e2]
- textbox "Name" [ref=e3]`;
const result = formatSnapshot(input);
expect(result).toContain('[@1] button "Save"');
expect(result).toContain('[@2] link "Cancel"');
expect(result).toContain('[@3] textbox "Name"');
});
});
describe('noise filtering', () => {
it('removes generic nodes without text', () => {
const input = `\
- generic [ref=e1]:
- generic [ref=e2]:
- button "Click" [ref=e3]`;
const result = formatSnapshot(input);
expect(result).not.toMatch(/^generic/m);
expect(result).toContain('button "Click"');
});
it('keeps generic nodes WITH text content', () => {
const input = '- generic [ref=e23]: Dashboard';
const result = formatSnapshot(input);
expect(result).toContain('generic: Dashboard');
});
it('removes img nodes without alt text', () => {
const input = `\
- img [ref=e13]
- img "Profile photo" [ref=e14]`;
const result = formatSnapshot(input);
expect(result).not.toContain('img\n');
expect(result).toContain('img "Profile photo"');
});
it('removes separator nodes', () => {
const input = '- separator [ref=e304]';
const result = formatSnapshot(input);
expect(result).toBe('');
});
it('removes presentation/none roles', () => {
const input = `\
- presentation [ref=e1]
- none [ref=e2]
- button "OK" [ref=e3]`;
const result = formatSnapshot(input);
expect(result).not.toContain('presentation');
expect(result).not.toContain('none');
expect(result).toContain('button "OK"');
});
});
describe('empty container pruning', () => {
it('prunes containers with no visible children', () => {
const input = `\
- list [ref=e88]:
- listitem [ref=e89]:
- generic [ref=e90]:
- img [ref=e91]`;
// After filtering: generic (no text) → removed, img (no alt) → removed
// listitem becomes empty → pruned, list becomes empty → pruned
const result = formatSnapshot(input);
expect(result).toBe('');
});
it('keeps containers with visible children', () => {
const input = `\
- list [ref=e1]:
- listitem [ref=e2]:
- link "Home" [ref=e3]`;
const result = formatSnapshot(input);
expect(result).toContain('list');
expect(result).toContain('listitem');
expect(result).toContain('link "Home"');
});
});
describe('maxDepth option', () => {
it('limits output to specified depth', () => {
const input = `\
- main [ref=e1]:
- heading "Dashboard" [ref=e2]
- navigation [ref=e3]:
- list [ref=e4]:
- link "Deep link" [ref=e5]`;
const result = formatSnapshot(input, { maxDepth: 2 });
expect(result).toContain('main');
expect(result).toContain('heading "Dashboard"');
// navigation is pruned: its only child list is empty after link is excluded by maxDepth
expect(result).not.toContain('navigation');
expect(result).not.toContain('Deep link');
});
it('handles maxDepth=0 correctly (was a bug)', () => {
const input = `\
- heading "Title" [ref=e1]
- link "Sub" [ref=e2]`;
const result = formatSnapshot(input, { maxDepth: 0 });
expect(result).toContain('heading "Title"');
expect(result).not.toContain('Sub');
});
});
describe('interactive mode', () => {
it('keeps interactive elements and landmarks', () => {
const result = formatSnapshot(GITHUB_NAV, { interactive: true });
// Interactive elements should be present
expect(result).toContain('button');
expect(result).toContain('link');
// Landmarks preserved
expect(result).toContain('banner');
expect(result).toContain('navigation');
});
it('filters non-interactive, non-landmark, textless nodes', () => {
const input = `\
- main [ref=e1]:
- generic [ref=e2]:
- generic [ref=e3]:
- button "Save" [ref=e4]
- generic [ref=e5]: some text content`;
const result = formatSnapshot(input, { interactive: true });
expect(result).toContain('main');
expect(result).toContain('button "Save"');
// generic with text is kept
expect(result).toContain('generic: some text content');
});
});
describe('compact mode', () => {
it('strips bracket annotations and collapses whitespace', () => {
const input = '- button "Save" [ref=e1] [cursor=pointer] [level=2]';
const result = formatSnapshot(input, { compact: true });
// ref/cursor already stripped, but [level=...] should also go in compact
expect(result).not.toContain('[level=');
expect(result).toContain('button');
});
});
describe('maxTextLength option', () => {
it('truncates long content lines', () => {
const input = '- heading "This is a very long heading that should be truncated at some point" [ref=e1]';
const result = formatSnapshot(input, { maxTextLength: 30 });
expect(result.length).toBeLessThanOrEqual(35); // some tolerance for ellipsis
expect(result).toContain('…');
});
});
// ---------------------------------------------------------------------------
// Real-world snapshot integration tests
// ---------------------------------------------------------------------------
describe('GitHub snapshot', () => {
it('drastically reduces nav bar output', () => {
const raw = GITHUB_NAV;
const rawLineCount = raw.split('\n').length;
const result = formatSnapshot(raw);
const resultLineCount = result.split('\n').length;
// Should significantly reduce line count
expect(resultLineCount).toBeLessThan(rawLineCount);
// Key content preserved
expect(result).toContain('link "Skip to content"');
expect(result).toContain('banner "Global Navigation Menu"');
expect(result).toContain('link "Dashboard"');
expect(result).toContain('button "Search or jump to…"');
// Noise removed
expect(result).not.toContain('[ref=');
expect(result).not.toContain('/url:');
});
it('preserves repo list structure', () => {
const result = formatSnapshot(GITHUB_REPOS);
expect(result).toContain('navigation "Repositories"');
expect(result).toContain('heading "Top repositories"');
expect(result).toContain('textbox "Find a repository…"');
expect(result).toContain('link "jackwener/twitter-cli"');
expect(result).toContain('link "jackwener/opencli"');
expect(result).toContain('img "Repository"');
// No refs or urls
expect(result).not.toContain('[ref=');
expect(result).not.toContain('/url:');
});
});
describe('Bilibili snapshot', () => {
it('cleans nav bar with Chinese text', () => {
const result = formatSnapshot(BILIBILI_NAV);
expect(result).toContain('link "首页"');
expect(result).toContain('link "番剧"');
expect(result).toContain('link "直播"');
expect(result).toContain('textbox "冷知识 金廷26年胜率100%"');
expect(result).not.toContain('[ref=');
});
it('handles video card with deeply nested wrappers', () => {
const result = formatSnapshot(BILIBILI_VIDEO);
expect(result).toContain('link "超酷时刻 即将到来 3.3万 40 16:24"');
expect(result).toContain('heading "超酷时刻 即将到来"');
expect(result).toContain('generic "Tesla特斯拉中国"');
// Deeply nested view count generics with text are kept
expect(result).toContain('3.3万');
});
it('prunes empty paragraph blocks', () => {
const result = formatSnapshot(BILIBILI_EMPTY);
// All content is generic (no text) and empty paragraphs
// After noise filtering, everything should be pruned
expect(result.trim()).toBe('');
});
});
describe('Twitter snapshot', () => {
it('preserves tweet structure', () => {
const result = formatSnapshot(TWITTER_TWEET);
expect(result).toContain('main');
expect(result).toContain('region "Timeline"');
expect(result).toContain('link "@elonmusk"');
expect(result).toContain('button "Reply"');
expect(result).toContain('button "Like"');
expect(result).not.toContain('separator');
});
it('truncates long tweet text with maxTextLength', () => {
const result = formatSnapshot(TWITTER_TWEET, { maxTextLength: 60 });
// The long tweet text should be truncated
expect(result).toContain('…');
// But short elements are unaffected
expect(result).toContain('button "Reply"');
});
it('interactive mode keeps only buttons and links', () => {
const result = formatSnapshot(TWITTER_TWEET, { interactive: true });
expect(result).toContain('link "@elonmusk"');
expect(result).toContain('button "Reply"');
expect(result).toContain('button "Retweet"');
expect(result).toContain('button "Like"');
// Structural landmarks kept
expect(result).toContain('main');
expect(result).toContain('region "Timeline"');
expect(result).toContain('article');
});
it('combined options: interactive + maxDepth', () => {
// With maxDepth: 2 and interactive, depth > 2 is filtered.
// article at depth 2 has only generic children (noise-filtered),
// so article gets pruned by container pruning, which cascades up.
const result = formatSnapshot(TWITTER_TWEET, { interactive: true, maxDepth: 2 });
expect(result).toContain('main');
expect(result).not.toContain('button "Reply"');
expect(result).not.toContain('link "@elonmusk"');
});
});
describe('reduction ratios on real data', () => {
it('achieves significant reduction on GitHub nav', () => {
const rawLines = GITHUB_NAV.split('\n').length;
const formatted = formatSnapshot(GITHUB_NAV);
const formattedLines = formatted.split('\n').filter(l => l.trim()).length;
// Expect at least 40% reduction
expect(formattedLines).toBeLessThan(rawLines * 0.6);
});
it('achieves significant reduction on Bilibili video card', () => {
const rawLines = BILIBILI_VIDEO.split('\n').length;
const formatted = formatSnapshot(BILIBILI_VIDEO);
const formattedLines = formatted.split('\n').filter(l => l.trim()).length;
// Expect at least 30% reduction
expect(formattedLines).toBeLessThan(rawLines * 0.7);
});
});
// ---------------------------------------------------------------------------
// Full-page snapshot fixture tests (loaded from __fixtures__/)
// ---------------------------------------------------------------------------
describe('full-page snapshots from fixtures', () => {
const fs = require('node:fs');
const path = require('node:path');
const fixturesDir = path.join(__dirname, '__fixtures__');
function loadFixture(name: string): string | null {
const p = path.join(fixturesDir, name);
if (!fs.existsSync(p)) return null;
return fs.readFileSync(p, 'utf-8');
}
it('GitHub: significant reduction and clean output', () => {
const raw = loadFixture('snapshot_github.txt');
if (!raw) return;
const rawLines = raw.split('\n').length;
const result = formatSnapshot(raw);
const resultLines = result.split('\n').filter((l: string) => l.trim()).length;
// Should achieve > 50% reduction on GitHub dashboard (heavy generic noise)
expect(resultLines).toBeLessThan(rawLines * 0.5);
// No annotations remain
expect(result).not.toContain('[ref=');
expect(result).not.toContain('[cursor=');
expect(result).not.toContain('/url:');
// Key content preserved
expect(result).toContain('link "Skip to content"');
expect(result).toContain('banner "Global Navigation Menu"');
expect(result).toContain('heading "Dashboard"');
});
it('Bilibili: significant reduction and Chinese text preserved', () => {
const raw = loadFixture('snapshot_bilibili.txt');
if (!raw) return;
const rawLines = raw.split('\n').length;
const result = formatSnapshot(raw);
const resultLines = result.split('\n').filter((l: string) => l.trim()).length;
// Should achieve > 40% reduction on Bilibili (lots of imgs and generics)
expect(resultLines).toBeLessThan(rawLines * 0.6);
// No annotations remain
expect(result).not.toContain('[ref=');
expect(result).not.toContain('[cursor=');
// Chinese text preserved
expect(result).toContain('link "首页"');
expect(result).toContain('link "番剧"');
});
it('Twitter/X: significant reduction and tweet structure preserved', () => {
const raw = loadFixture('snapshot_twitter.txt');
if (!raw) return;
const rawLines = raw.split('\n').length;
const result = formatSnapshot(raw);
const resultLines = result.split('\n').filter((l: string) => l.trim()).length;
// Should achieve > 40% reduction on Twitter/X
expect(resultLines).toBeLessThan(rawLines * 0.6);
// No annotations remain
expect(result).not.toContain('[ref=');
expect(result).not.toContain('[cursor=');
expect(result).not.toContain('/url:');
// Key structure preserved
expect(result).toContain('main');
});
it('GitHub interactive mode: drastic reduction', () => {
const raw = loadFixture('snapshot_github.txt');
if (!raw) return;
const result = formatSnapshot(raw, { interactive: true });
const resultLines = result.split('\n').filter((l: string) => l.trim()).length;
// Interactive mode should be much more aggressive
expect(resultLines).toBeLessThan(200);
// Interactive elements still present
expect(result).toContain('button');
expect(result).toContain('link');
expect(result).toContain('textbox');
});
it('Bilibili maxDepth=3: shallow view', () => {
const raw = loadFixture('snapshot_bilibili.txt');
if (!raw) return;
const result = formatSnapshot(raw, { maxDepth: 3 });
const resultLines = result.split('\n').filter((l: string) => l.trim()).length;
// Depth-limited should be very compact
expect(resultLines).toBeLessThan(50);
});
});
});
+403 -15
View File
@@ -1,35 +1,270 @@
/**
* Aria snapshot formatter: parses Playwright MCP snapshot text into clean format.
*
* Multi-pass pipeline:
* 1. Parse & filter: strip annotations, metadata, noise roles, ads, decorators
* 2. Deduplicate: generic/text child matching parent label
* 3. Deduplicate: heading + link with identical labels
* 4. Deduplicate: nested identical links
* 5. Prune: empty containers (iterative bottom-up)
* 6. Collapse: single-child containers
*/
export interface FormatOptions {
interactive?: boolean;
compact?: boolean;
maxDepth?: number;
maxTextLength?: number;
}
const DEFAULT_MAX_TEXT_LENGTH = 200;
// Roles that are pure noise and should always be filtered
const NOISE_ROLES = new Set([
'none', 'presentation', 'separator', 'paragraph', 'tooltip', 'status',
]);
// Roles whose entire subtree should be removed (footer boilerplate, etc.)
const SUBTREE_NOISE_ROLES = new Set([
'contentinfo',
]);
// Roles considered interactive (clickable/typeable)
const INTERACTIVE_ROLES = new Set([
'button', 'link', 'textbox', 'checkbox', 'radio',
'combobox', 'tab', 'menuitem', 'option', 'switch',
'slider', 'spinbutton', 'searchbox',
]);
// Structural landmark roles kept even in interactive mode
const LANDMARK_ROLES = new Set([
'main', 'navigation', 'banner', 'heading', 'search',
'region', 'list', 'listitem', 'article', 'complementary',
'group', 'toolbar', 'tablist',
]);
// Container roles eligible for pruning and collapse
const CONTAINER_ROLES = new Set([
'list', 'listitem', 'group', 'toolbar', 'tablist',
'navigation', 'region', 'complementary',
'search', 'article', 'paragraph', 'figure',
]);
// Decorator / separator text that adds no semantic value
const DECORATOR_TEXT = new Set(['•', '·', '|', '—', '-', '/', '\\']);
// Ad-related URL patterns
const AD_URL_PATTERNS = [
'googleadservices.com/pagead/',
'alb.reddit.com/cr?',
'doubleclick.net/',
'cm.bilibili.com/cm/api/fees/',
];
// Boilerplate button labels to filter (back-to-top, etc.)
const BOILERPLATE_LABELS = [
'回到顶部', 'back to top', 'scroll to top', 'go to top',
];
/**
* Parse role and text from a trimmed snapshot line.
* Handles quoted labels and trailing text after colon correctly,
* including lines wrapped in single quotes by Playwright.
*/
function parseLine(trimmed: string): { role: string; text: string; hasText: boolean; trailingText: string } {
// Unwrap outer single quotes if present (Playwright wraps lines with special chars)
let line = trimmed;
if (line.startsWith("'") && line.endsWith("':")) {
line = line.slice(1, -2) + ':';
} else if (line.startsWith("'") && line.endsWith("'")) {
line = line.slice(1, -1);
}
// Role is the first word
const roleMatch = line.match(/^([a-zA-Z]+)\b/);
const role = roleMatch ? roleMatch[1].toLowerCase() : '';
// Extract quoted text content (the semantic label)
const textMatch = line.match(/"([^"]*)"/);
const text = textMatch ? textMatch[1] : '';
// For trailing text: strip annotations and quoted strings first, then check after last colon
// This avoids matching colons inside quoted labels like "Account: user@email.com"
let stripped = line;
// Remove all quoted strings
stripped = stripped.replace(/"[^"]*"/g, '""');
// Remove all bracket annotations
stripped = stripped.replace(/\[[^\]]*\]/g, '');
const colonIdx = stripped.lastIndexOf(':');
let trailingText = '';
if (colonIdx !== -1) {
const afterColon = stripped.slice(colonIdx + 1).trim();
if (afterColon.length > 0) {
// Get the actual trailing text from original line at same position
const origColonIdx = line.lastIndexOf(':');
if (origColonIdx !== -1) {
trailingText = line.slice(origColonIdx + 1).trim();
}
}
}
return { role, text, hasText: text.length > 0 || trailingText.length > 0, trailingText };
}
/**
* Strip ALL bracket annotations from a content line, preserving quoted strings.
* Handles both double-quoted and outer single-quoted lines from Playwright.
*/
function stripAnnotations(content: string): string {
// Unwrap outer single quotes first
let line = content;
let hadOuterQuotes = false;
if (line.startsWith("'") && (line.endsWith("':") || line.endsWith("'"))) {
hadOuterQuotes = true;
if (line.endsWith("':")) {
line = line.slice(1, -2) + ':';
} else {
line = line.slice(1, -1);
}
}
// Split by double quotes to protect quoted content
const parts = line.split('"');
for (let i = 0; i < parts.length; i += 2) {
// Only strip annotations from non-quoted parts (even indices)
parts[i] = parts[i].replace(/\s*\[[^\]]*\]/g, '');
}
let result = parts.join('"').replace(/\s{2,}/g, ' ').trim();
return result;
}
/**
* Check if a line is a metadata-only line (like /url: ...).
*/
function isMetadataLine(trimmed: string): boolean {
return /^\/[a-zA-Z]+:/.test(trimmed);
}
/**
* Check if text content is purely decorative (separators, dots, etc.)
*/
function isDecoratorText(text: string): boolean {
return DECORATOR_TEXT.has(text.trim());
}
/**
* Check if a node is ad-related based on its text content.
*/
function isAdNode(text: string, trailingText: string): boolean {
const t = (text + ' ' + trailingText).toLowerCase();
if (t.includes('sponsored') || t.includes('advertisement')) return true;
if (t.includes('广告')) return true;
// Check for ad tracking URLs in the label
for (const pattern of AD_URL_PATTERNS) {
if (text.includes(pattern) || trailingText.includes(pattern)) return true;
}
return false;
}
/**
* Check if a node is boilerplate UI (back-to-top, etc.)
*/
function isBoilerplateNode(text: string): boolean {
const t = text.toLowerCase();
return BOILERPLATE_LABELS.some(label => t.includes(label));
}
/**
* Check if a role is noise that should be filtered.
*/
function isNoiseNode(role: string, hasText: boolean, text: string, trailingText: string): boolean {
if (NOISE_ROLES.has(role)) return true;
// generic without text is a wrapper
if (role === 'generic' && !hasText) return true;
// img without alt text is noise
if (role === 'img' && !hasText) return true;
// Decorator-only text nodes
if ((role === 'generic' || role === 'text') && hasText) {
const content = trailingText || text;
if (isDecoratorText(content)) return true;
}
return false;
}
interface Entry {
depth: number;
content: string;
role: string;
text: string;
trailingText: string;
isInteractive: boolean;
isLandmark: boolean;
isSubtreeSkip: boolean; // ad nodes or boilerplate — skip entire subtree
}
export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
if (!raw || typeof raw !== 'string') return '';
const lines = raw.split('\n');
const result: string[] = [];
let refCounter = 0;
for (const line of lines) {
const maxTextLen = opts.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH;
const lines = raw.split('\n');
// === Pass 1: Parse, filter, and collect entries ===
const entries: Entry[] = [];
let refCounter = 0;
let skipUntilDepth = -1; // When >= 0, skip all nodes at depth > this value
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (!line.trim()) continue;
const indent = line.length - line.trimStart().length;
const depth = Math.floor(indent / 2);
if (opts.maxDepth && depth > opts.maxDepth) continue;
// If we're in a subtree skip zone, check depth
if (skipUntilDepth >= 0) {
if (depth > skipUntilDepth) continue; // still inside subtree
skipUntilDepth = -1; // exited subtree
}
let content = line.trimStart();
// Skip non-interactive elements in interactive mode
if (opts.interactive) {
const interactiveRoles = ['button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'tab', 'menuitem', 'option'];
const role = content.split(/[\s[]/)[0]?.toLowerCase() ?? '';
if (!interactiveRoles.some(r => role.includes(r)) && depth > 1) continue;
// Strip leading "- "
if (content.startsWith('- ')) {
content = content.slice(2);
}
// Compact: strip verbose role descriptions
// Skip metadata lines
if (isMetadataLine(content)) continue;
// Apply maxDepth filter
if (opts.maxDepth !== undefined && depth > opts.maxDepth) continue;
const { role, text, hasText, trailingText } = parseLine(content);
// Skip noise nodes
if (isNoiseNode(role, hasText, text, trailingText)) continue;
// Skip subtree noise roles (contentinfo footer, etc.) — skip entire subtree
if (SUBTREE_NOISE_ROLES.has(role)) {
skipUntilDepth = depth;
continue;
}
// Strip annotations
content = stripAnnotations(content);
// Check if node should trigger subtree skip (ads, boilerplate)
const isSubtreeSkip = isAdNode(text, trailingText) || isBoilerplateNode(text);
// Interactive mode filter
const isInteractive = INTERACTIVE_ROLES.has(role);
const isLandmark = LANDMARK_ROLES.has(role);
if (opts.interactive && !isInteractive && !isLandmark && !hasText) continue;
// Compact mode
if (opts.compact) {
content = content
.replace(/\s*\[.*?\]\s*/g, ' ')
@@ -37,15 +272,168 @@ export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
.trim();
}
// Text truncation
if (maxTextLen > 0 && content.length > maxTextLen) {
content = content.slice(0, maxTextLen) + '…';
}
// Assign refs to interactive elements
const interactivePattern = /^(button|link|textbox|checkbox|radio|combobox|tab|menuitem|option)\b/i;
if (interactivePattern.test(content)) {
if (isInteractive) {
refCounter++;
content = `[@${refCounter}] ${content}`;
}
result.push(' '.repeat(depth) + content);
entries.push({ depth, content, role, text, trailingText, isInteractive, isLandmark, isSubtreeSkip });
}
return result.join('\n');
// === Pass 2: Remove subtree-skip nodes (ads, boilerplate, contentinfo) ===
let noAds: Entry[] = [];
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
if (entry.isSubtreeSkip) {
const skipDepth = entry.depth;
i++;
while (i < entries.length && entries[i].depth > skipDepth) {
i++;
}
i--;
continue;
}
noAds.push(entry);
}
// === Pass 3: Deduplicate child generic/text matching parent label ===
let deduped: Entry[] = [];
for (let i = 0; i < noAds.length; i++) {
const entry = noAds[i];
if (entry.role === 'generic' || entry.role === 'text') {
let parent: Entry | undefined;
for (let j = deduped.length - 1; j >= 0; j--) {
if (deduped[j].depth < entry.depth) {
parent = deduped[j];
break;
}
if (deduped[j].depth === entry.depth) break;
}
if (parent) {
const childText = entry.trailingText || entry.text;
if (childText && parent.text && childText === parent.text) {
continue;
}
}
}
deduped.push(entry);
}
// === Pass 4: Deduplicate heading + child link with identical label ===
// Pattern: heading "Title": → link "Title": (same text) → skip the link
const deduped2: Entry[] = [];
for (let i = 0; i < deduped.length; i++) {
const entry = deduped[i];
if (entry.role === 'heading' && entry.text) {
const next = deduped[i + 1];
if (next && next.role === 'link' && next.text === entry.text && next.depth === entry.depth + 1) {
// Keep the heading, skip the link. But preserve link's children re-parented.
deduped2.push(entry);
i++; // skip the link
continue;
}
}
deduped2.push(entry);
}
// === Pass 5: Deduplicate nested identical links ===
const deduped3: Entry[] = [];
for (let i = 0; i < deduped2.length; i++) {
const entry = deduped2[i];
if (entry.role === 'link' && entry.text) {
const next = deduped2[i + 1];
if (next && next.role === 'link' && next.text === entry.text && next.depth === entry.depth + 1) {
continue; // Skip parent, keep child
}
}
deduped3.push(entry);
}
// === Pass 6: Iteratively prune empty containers (bottom-up) ===
let current = deduped3;
let changed = true;
while (changed) {
changed = false;
const next: Entry[] = [];
for (let i = 0; i < current.length; i++) {
const entry = current[i];
if (CONTAINER_ROLES.has(entry.role) && !entry.text && !entry.trailingText) {
let hasChildren = false;
for (let j = i + 1; j < current.length; j++) {
if (current[j].depth <= entry.depth) break;
if (current[j].depth > entry.depth) {
hasChildren = true;
break;
}
}
if (!hasChildren) {
changed = true;
continue;
}
}
next.push(entry);
}
current = next;
}
// === Pass 7: Collapse single-child containers ===
const collapsed: Entry[] = [];
for (let i = 0; i < current.length; i++) {
const entry = current[i];
if (CONTAINER_ROLES.has(entry.role) && !entry.text && !entry.trailingText) {
let childCount = 0;
let childIdx = -1;
for (let j = i + 1; j < current.length; j++) {
if (current[j].depth <= entry.depth) break;
if (current[j].depth === entry.depth + 1) {
childCount++;
if (childCount === 1) childIdx = j;
}
}
if (childCount === 1 && childIdx !== -1) {
const child = current[childIdx];
let hasGrandchildren = false;
for (let j = childIdx + 1; j < current.length; j++) {
if (current[j].depth <= child.depth) break;
if (current[j].depth > child.depth) {
hasGrandchildren = true;
break;
}
}
if (!hasGrandchildren) {
const mergedContent = entry.content.replace(/:$/, '') + ' > ' + child.content;
collapsed.push({
...entry,
content: mergedContent,
role: child.role,
text: child.text,
trailingText: child.trailingText,
isInteractive: child.isInteractive,
});
i++;
continue;
}
}
}
collapsed.push(entry);
}
return collapsed.map(e => ' '.repeat(e.depth) + e.content).join('\n');
}
+15 -1
View File
@@ -2,6 +2,20 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
include: ['src/**/*.test.ts', 'tests/**/*.test.ts'],
projects: [
{
test: {
name: 'unit',
include: ['src/**/*.test.ts'],
},
},
{
test: {
name: 'e2e',
include: ['tests/**/*.test.ts'],
maxWorkers: 2,
},
},
],
},
});