From 7700704923f7887fcf0d0210b902a961aa3fd932 Mon Sep 17 00:00:00 2001 From: Kasumi <125308847+KasumiChen@users.noreply.github.com> Date: Fri, 20 Mar 2026 09:35:30 -0700 Subject: [PATCH] feat: add Bloomberg adapter (#145) * feat: add Bloomberg adapter with RSS feeds and article extraction * refactor: improve Bloomberg adapter review fixes - news.ts: reorder flow (goto before loadStory), increase wait times for slow hydration, add clarity comments - utils.ts: clarify validateBloombergLink regex (use non-capturing group) - build-manifest.ts: log warning on scanTs parse failure (match scanYaml pattern) - public-commands.test.ts: use it.each for section RSS tests (better isolation & reporting) --------- Co-authored-by: ByteYue --- README.md | 3 + README.zh-CN.md | 3 + docs/.vitepress/config.mts | 1 + docs/adapters/browser/bloomberg.md | 70 +++++ docs/adapters/index.md | 1 + src/build-manifest.ts | 44 +-- src/clis/bloomberg/businessweek.ts | 18 ++ src/clis/bloomberg/economics.ts | 18 ++ src/clis/bloomberg/feeds.ts | 16 ++ src/clis/bloomberg/industries.ts | 18 ++ src/clis/bloomberg/main.ts | 18 ++ src/clis/bloomberg/markets.ts | 18 ++ src/clis/bloomberg/news.ts | 136 +++++++++ src/clis/bloomberg/opinions.ts | 18 ++ src/clis/bloomberg/politics.ts | 18 ++ src/clis/bloomberg/tech.ts | 18 ++ src/clis/bloomberg/utils.test.ts | 135 +++++++++ src/clis/bloomberg/utils.ts | 429 +++++++++++++++++++++++++++++ tests/e2e/browser-public.test.ts | 25 ++ tests/e2e/public-commands.test.ts | 45 +++ 20 files changed, 1033 insertions(+), 19 deletions(-) create mode 100644 docs/adapters/browser/bloomberg.md create mode 100644 src/clis/bloomberg/businessweek.ts create mode 100644 src/clis/bloomberg/economics.ts create mode 100644 src/clis/bloomberg/feeds.ts create mode 100644 src/clis/bloomberg/industries.ts create mode 100644 src/clis/bloomberg/main.ts create mode 100644 src/clis/bloomberg/markets.ts create mode 100644 src/clis/bloomberg/news.ts create mode 100644 src/clis/bloomberg/opinions.ts create mode 100644 src/clis/bloomberg/politics.ts create mode 100644 src/clis/bloomberg/tech.ts create mode 100644 src/clis/bloomberg/utils.test.ts create mode 100644 src/clis/bloomberg/utils.ts diff --git a/README.md b/README.md index 196115d..5425bc5 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ Run `opencli list` for the live registry. | **boss** | `search` `detail` | Browser | | **coupang** | `search` `add-to-cart` | Browser | | **bbc** | `news` | Public | +| **bloomberg** | `main` `markets` `economics` `industries` `tech` `politics` `businessweek` `opinions` `feeds` `news` | Public / Browser | | **ctrip** | `search` | Browser | | **github** | `search` | Public | | **arxiv** | `search` `paper` | Public | @@ -150,6 +151,8 @@ Run `opencli list` for the live registry. | **yahoo-finance** | `quote` | Browser | | **sinafinance** | `news` | 🌐 Public | +> **Bloomberg note**: The RSS-backed Bloomberg listing commands (`main`, section feeds, `feeds`) work without a browser. `bloomberg news` is for standard Bloomberg story/article pages that your current Chrome session can already access. Audio and some other non-standard pages may fail, and OpenCLI does not bypass Bloomberg paywall or entitlement checks. + ### External CLI Hub OpenCLI acts as a universal hub for your existing command-line tools. It provides unified discovery, automatic installation, and pure passthrough execution. diff --git a/README.zh-CN.md b/README.zh-CN.md index 80f5e49..1161d80 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -138,6 +138,7 @@ npm install -g @jackwener/opencli@latest | **boss** | `search` `detail` | 浏览器 | | **coupang** | `search` `add-to-cart` | 浏览器 | | **bbc** | `news` | 公共 API | +| **bloomberg** | `main` `markets` `economics` `industries` `tech` `politics` `businessweek` `opinions` `feeds` `news` | 公共 API / 浏览器 | | **ctrip** | `search` | 浏览器 | | **github** | `search` | 公共 API | | **arxiv** | `search` `paper` | 公开 | @@ -150,6 +151,8 @@ npm install -g @jackwener/opencli@latest | **yahoo-finance** | `quote` | 浏览器 | | **sinafinance** | `news` | 🌐 公开 | +> **Bloomberg 说明**：Bloomberg 的 RSS 列表命令（`main`、各栏目 feed、`feeds`）无需浏览器即可使用。`bloomberg news` 适用于当前 Chrome 会话本身就能访问的标准 Bloomberg 文章页。音频页和部分非标准页面可能失败，OpenCLI 也不会绕过 Bloomberg 的付费墙、登录或权限校验。 + ### 桌面应用适配器 每个桌面适配器都有自己详细的文档说明，包括命令参考、启动配置与使用示例： diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 79d9ba9..baf9c92 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -54,6 +54,7 @@ export default defineConfig({ { text: 'YouTube', link: '/adapters/browser/youtube' }, { text: 'Xueqiu', link: '/adapters/browser/xueqiu' }, { text: 'V2EX', link: '/adapters/browser/v2ex' }, + { text: 'Bloomberg', link: '/adapters/browser/bloomberg' }, { text: 'LinkedIn', link: '/adapters/browser/linkedin' }, { text: 'Coupang', link: '/adapters/browser/coupang' }, { text: 'BOSS Zhipin', link: '/adapters/browser/boss' }, diff --git a/docs/adapters/browser/bloomberg.md b/docs/adapters/browser/bloomberg.md new file mode 100644 index 0000000..28c87a0 --- /dev/null +++ b/docs/adapters/browser/bloomberg.md @@ -0,0 +1,70 @@ +# Bloomberg + +**Mode**: 🌐 / 🔐 Mixed · **Domains**: `feeds.bloomberg.com`, `www.bloomberg.com` + +## Commands + +| Command | Description | +|---------|-------------| +| `opencli bloomberg main` | Bloomberg homepage top stories from RSS | +| `opencli bloomberg markets` | Bloomberg Markets top stories from RSS | +| `opencli bloomberg economics` | Bloomberg Economics top stories from RSS | +| `opencli bloomberg industries` | Bloomberg Industries top stories from RSS | +| `opencli bloomberg tech` | Bloomberg Tech top stories from RSS | +| `opencli bloomberg politics` | Bloomberg Politics top stories from RSS | +| `opencli bloomberg businessweek` | Bloomberg Businessweek top stories from RSS | +| `opencli bloomberg opinions` | Bloomberg Opinion top stories from RSS | +| `opencli bloomberg feeds` | List the RSS feed aliases used by the adapter | +| `opencli bloomberg news ` | Read a standard Bloomberg story/article page and return title, summary, media links, and article text | + +## What works today + +- RSS-backed listing commands work without a browser: + - `main` + - `markets` + - `economics` + - `industries` + - `tech` + - `politics` + - `businessweek` + - `opinions` + - `feeds` +- `bloomberg news` works on standard Bloomberg story/article pages that expose `#__NEXT_DATA__` and are accessible to your current Chrome session. + +## Current limitations + +- Audio pages and some other non-standard Bloomberg URLs may fail. +- Some Bloomberg pages can return bot-protection or access-gated responses instead of article data. +- This adapter is for data retrieval/extraction only. It does **not** bypass Bloomberg paywall, login, entitlement, or other access checks. + +## Usage Examples + +```bash +# List supported RSS feed aliases +opencli bloomberg feeds + +# Fetch Bloomberg homepage headlines +opencli bloomberg main --limit 5 + +# Fetch a section feed as JSON +opencli bloomberg tech --limit 3 -f json + +# Read a standard article page +opencli bloomberg news https://www.bloomberg.com/news/articles/2026-03-19/example -f json + +# Relative article paths also work +opencli bloomberg news /news/articles/2026-03-19/example +``` + +## Prerequisites + +- RSS commands do not require Chrome. +- `bloomberg news` requires: + - Chrome running + - a Chrome session that can already access the target Bloomberg article page + - the [Browser Bridge extension](/guide/browser-bridge) + +## Notes + +- RSS commands support `--limit` with a maximum of 20 items. +- If `bloomberg news` fails on a page from RSS, try a different standard story/article link first; not every Bloomberg URL in feeds is a normal article page. diff --git a/docs/adapters/index.md b/docs/adapters/index.md index b52935d..590b744 100644 --- a/docs/adapters/index.md +++ b/docs/adapters/index.md @@ -14,6 +14,7 @@ Run `opencli list` for the live registry. | **[xueqiu](/adapters/browser/xueqiu)** | `feed` `hot-stock` `hot` `search` `stock` `watchlist` | 🔐 Browser | | **[youtube](/adapters/browser/youtube)** | `search` `video` `transcript` | 🔐 Browser | | **[v2ex](/adapters/browser/v2ex)** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 / 🔐 | +| **[bloomberg](/adapters/browser/bloomberg)** | `main` `markets` `economics` `industries` `tech` `politics` `businessweek` `opinions` `feeds` `news` | 🌐 / 🔐 | | **[weibo](/adapters/browser/weibo)** | `hot` | 🔐 Browser | | **[linkedin](/adapters/browser/linkedin)** | `search` | 🔐 Browser | | **[coupang](/adapters/browser/coupang)** | `search` `add-to-cart` | 🔐 Browser | diff --git a/src/build-manifest.ts b/src/build-manifest.ts index aa82e68..1e17947 100644 --- a/src/build-manifest.ts +++ b/src/build-manifest.ts @@ -196,26 +196,29 @@ function scanYaml(filePath: string, site: string): ManifestEntry | null { } } -function scanTs(filePath: string, site: string): ManifestEntry { +function scanTs(filePath: string, site: string): ManifestEntry | null { // TS adapters self-register via cli() at import time. // We statically parse the source to extract metadata for the manifest stub. const baseName = path.basename(filePath, path.extname(filePath)); const relativePath = `${site}/${baseName}.js`; - const entry: ManifestEntry = { - site, - name: baseName, - description: '', - strategy: 'cookie', - browser: true, - args: [], - type: 'ts', - modulePath: relativePath, - }; - try { const src = fs.readFileSync(filePath, 'utf-8'); + // Helper/test modules should not appear as CLI commands in the manifest. + if (!/\bcli\s*\(/.test(src)) return null; + + const entry: ManifestEntry = { + site, + name: baseName, + description: '', + strategy: 'cookie', + browser: true, + args: [], + type: 'ts', + modulePath: relativePath, + }; + // Extract description const descMatch = src.match(/description\s*:\s*['"`]([^'"`]*)['"`]/); if (descMatch) entry.description = descMatch[1]; @@ -244,11 +247,13 @@ function scanTs(filePath: string, site: string): ManifestEntry { if (argsBlock) { entry.args = parseTsArgsBlock(argsBlock); } - } catch { - // If parsing fails, fall back to empty metadata — module will self-register at runtime - } - return entry; + return entry; + } catch (err: any) { + // If parsing fails, log a warning (matching scanYaml behaviour) and skip the entry. + process.stderr.write(`Warning: failed to scan ${filePath}: ${err.message}\n`); + return null; + } } export function buildManifest(): ManifestEntry[] { @@ -264,10 +269,11 @@ export function buildManifest(): ManifestEntry[] { const entry = scanYaml(filePath, site); if (entry) manifest.push(entry); } else if ( - (file.endsWith('.ts') && !file.endsWith('.d.ts') && file !== 'index.ts') || - (file.endsWith('.js') && !file.endsWith('.d.js') && file !== 'index.js') + (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts') && file !== 'index.ts') || + (file.endsWith('.js') && !file.endsWith('.d.js') && !file.endsWith('.test.js') && file !== 'index.js') ) { - manifest.push(scanTs(filePath, site)); + const entry = scanTs(filePath, site); + if (entry) manifest.push(entry); } } } diff --git a/src/clis/bloomberg/businessweek.ts b/src/clis/bloomberg/businessweek.ts new file mode 100644 index 0000000..eafa9be --- /dev/null +++ b/src/clis/bloomberg/businessweek.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'businessweek', + description: 'Bloomberg Businessweek top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('businessweek', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/economics.ts b/src/clis/bloomberg/economics.ts new file mode 100644 index 0000000..adfb5b4 --- /dev/null +++ b/src/clis/bloomberg/economics.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'economics', + description: 'Bloomberg Economics top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('economics', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/feeds.ts b/src/clis/bloomberg/feeds.ts new file mode 100644 index 0000000..974d9c5 --- /dev/null +++ b/src/clis/bloomberg/feeds.ts @@ -0,0 +1,16 @@ +import { cli, Strategy } from '../../registry.js'; +import { BLOOMBERG_FEEDS } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'feeds', + description: 'List the Bloomberg RSS feed aliases used by the adapter', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [], + columns: ['name', 'url'], + func: async () => { + return Object.entries(BLOOMBERG_FEEDS).map(([name, url]) => ({ name, url })); + }, +}); diff --git a/src/clis/bloomberg/industries.ts b/src/clis/bloomberg/industries.ts new file mode 100644 index 0000000..852b3b4 --- /dev/null +++ b/src/clis/bloomberg/industries.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'industries', + description: 'Bloomberg Industries top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('industries', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/main.ts b/src/clis/bloomberg/main.ts new file mode 100644 index 0000000..0d15d2e --- /dev/null +++ b/src/clis/bloomberg/main.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'main', + description: 'Bloomberg homepage top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('main', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/markets.ts b/src/clis/bloomberg/markets.ts new file mode 100644 index 0000000..25cd8f4 --- /dev/null +++ b/src/clis/bloomberg/markets.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'markets', + description: 'Bloomberg Markets top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('markets', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/news.ts b/src/clis/bloomberg/news.ts new file mode 100644 index 0000000..3ce84a4 --- /dev/null +++ b/src/clis/bloomberg/news.ts @@ -0,0 +1,136 @@ +import { cli, Strategy } from '../../registry.js'; +import { CliError } from '../../errors.js'; +import { + extractStoryMediaLinks, + renderStoryBody, + validateBloombergLink, + type BloombergStory, +} from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'news', + description: 'Read a Bloomberg story/article page and return title, full content, and media links', + domain: 'www.bloomberg.com', + strategy: Strategy.COOKIE, + browser: true, + args: [ + { name: 'link', positional: true, required: true, help: 'Bloomberg story/article URL or relative Bloomberg path' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks', 'content'], + func: async (page, kwargs) => { + const url = validateBloombergLink(kwargs.link); + + // Navigate and wait for the page to hydrate before extracting story data. + await page.goto(url); + await page.wait(5); + + const loadStory = async () => page.evaluate(`(() => { + const isRobot = /Are you a robot/i.test(document.title) + || /unusual activity/i.test(document.body.innerText) + || /click the box below to let us know you're not a robot/i.test(document.body.innerText); + + if (isRobot) { + return { + errorCode: 'ROBOT_PAGE', + title: document.title, + preview: document.body.innerText.slice(0, 400), + }; + } + + const raw = document.querySelector('#__NEXT_DATA__')?.textContent; + if (!raw) { + return { + errorCode: 'NO_NEXT_DATA', + title: document.title, + preview: document.body.innerText.slice(0, 400), + }; + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (err) { + return { + errorCode: 'BAD_NEXT_DATA', + title: document.title, + preview: document.body.innerText.slice(0, 400), + message: String(err), + }; + } + + const story = parsed?.props?.pageProps?.story; + if (!story) { + return { + errorCode: 'NO_STORY', + title: document.title, + preview: document.body.innerText.slice(0, 400), + }; + } + + return { + story: { + headline: story.headline || story.seoHeadline || story.seoTitle || document.querySelector('h1')?.textContent?.trim() || document.title, + summary: story.summary || story.socialDescription || story.seoDescription || document.querySelector('meta[name="description"]')?.getAttribute('content') || '', + url: story.url || story.readingUrl || location.href, + body: story.body || null, + lede: story.lede || null, + ledeImageUrl: story.ledeImageUrl || null, + socialImageUrl: story.socialImageUrl || null, + imageAttachments: story.imageAttachments || {}, + videoAttachments: story.videoAttachments || {}, + } + }; + })()`); + + let result: any = await loadStory(); + + // Retry once — Bloomberg pages sometimes hydrate slowly. + if (result?.errorCode === 'NO_NEXT_DATA' || result?.errorCode === 'NO_STORY') { + await page.wait(4); + result = await loadStory(); + } + + if (result?.errorCode === 'ROBOT_PAGE') { + throw new CliError( + 'FETCH_ERROR', + 'Bloomberg served the bot-protection page instead of article content', + 'Try again later or open the article in a regular Chrome session first, then rerun the command. This command uses your current Bloomberg access and does not bypass paywall or entitlement checks.', + ); + } + + if (result?.errorCode) { + throw new CliError( + 'PARSE_ERROR', + `Bloomberg page did not expose article story data (${result.errorCode})`, + 'This command currently works on standard Bloomberg story/article pages that expose __NEXT_DATA__. Audio, video, newsletter, or other non-standard/blocked pages may not work. Access still depends on your current Bloomberg session.', + ); + } + + const story = result?.story as BloombergStory | undefined; + if (!story) { + throw new CliError( + 'PARSE_ERROR', + 'Failed to extract Bloomberg story data', + 'Bloomberg may have changed the page structure.', + ); + } + + const content = renderStoryBody(story.body); + if (!content) { + throw new CliError( + 'PARSE_ERROR', + 'Bloomberg article body was empty after parsing', + 'Bloomberg may have changed the story-body format, the URL may not point to a standard article page, or the page may not be accessible in your current Bloomberg session.', + ); + } + + return [{ + title: story.headline || '', + summary: story.summary || '', + link: story.url || url, + mediaLinks: extractStoryMediaLinks(story), + content, + }]; + }, +}); diff --git a/src/clis/bloomberg/opinions.ts b/src/clis/bloomberg/opinions.ts new file mode 100644 index 0000000..c1bb3fb --- /dev/null +++ b/src/clis/bloomberg/opinions.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'opinions', + description: 'Bloomberg Opinion top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('opinions', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/politics.ts b/src/clis/bloomberg/politics.ts new file mode 100644 index 0000000..131d054 --- /dev/null +++ b/src/clis/bloomberg/politics.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'politics', + description: 'Bloomberg Politics top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('politics', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/tech.ts b/src/clis/bloomberg/tech.ts new file mode 100644 index 0000000..218c14b --- /dev/null +++ b/src/clis/bloomberg/tech.ts @@ -0,0 +1,18 @@ +import { cli, Strategy } from '../../registry.js'; +import { fetchBloombergFeed } from './utils.js'; + +cli({ + site: 'bloomberg', + name: 'tech', + description: 'Bloomberg Tech top stories (RSS)', + domain: 'feeds.bloomberg.com', + strategy: Strategy.PUBLIC, + browser: false, + args: [ + { name: 'limit', type: 'int', default: 1, help: 'Number of feed items to return (max 20)' }, + ], + columns: ['title', 'summary', 'link', 'mediaLinks'], + func: async (_page, kwargs) => { + return fetchBloombergFeed('tech', kwargs.limit ?? 1); + }, +}); diff --git a/src/clis/bloomberg/utils.test.ts b/src/clis/bloomberg/utils.test.ts new file mode 100644 index 0000000..be581c7 --- /dev/null +++ b/src/clis/bloomberg/utils.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; +import { extractStoryMediaLinks, parseBloombergRss, renderStoryBody } from './utils.js'; + +describe('Bloomberg utils', () => { + it('parses Bloomberg RSS items with summary, link, and deduped media links', () => { + const xml = ` + + + + <![CDATA[Headline One]]> + One & more]]> + https://www.bloomberg.com/news/articles/2026-03-19/example-one + + + + + + Headline Two + Summary Two + https://www.bloomberg.com/news/articles/2026-03-19/example-two + + + + `; + + const items = parseBloombergRss(xml); + expect(items).toHaveLength(2); + expect(items[0]).toEqual({ + title: 'Headline One', + summary: 'Summary One & more', + link: 'https://www.bloomberg.com/news/articles/2026-03-19/example-one', + mediaLinks: ['https://assets.bwbx.io/example-one.jpg'], + }); + expect(items[1]).toEqual({ + title: 'Headline Two', + summary: 'Summary Two', + link: 'https://www.bloomberg.com/news/articles/2026-03-19/example-two', + mediaLinks: ['https://assets.bwbx.io/example-two.png'], + }); + }); + + it('renders Bloomberg story rich-text body into readable text', () => { + const body = { + type: 'document', + content: [ + { type: 'inline-newsletter', data: { position: 'top' }, content: [] }, + { + type: 'paragraph', + data: {}, + content: [ + { type: 'text', value: 'Lead paragraph with ' }, + { type: 'entity', content: [{ type: 'text', value: 'linked text' }] }, + { type: 'text', value: '.' }, + ], + }, + { + type: 'heading', + data: { level: 2 }, + content: [{ type: 'text', value: 'Key Points' }], + }, + { + type: 'list', + data: { style: 'unordered' }, + content: [ + { + type: 'list-item', + content: [ + { type: 'paragraph', content: [{ type: 'text', value: 'Point one' }] }, + ], + }, + { + type: 'list-item', + content: [ + { type: 'paragraph', content: [{ type: 'text', value: 'Point two' }] }, + ], + }, + ], + }, + { + type: 'blockquote', + content: [{ type: 'text', value: 'Quoted line' }], + }, + { + type: 'media', + data: { + attachment: { + caption: '

Chart caption

', + }, + }, + }, + { type: 'ad', data: { num: 1 }, content: [] }, + ], + }; + + expect(renderStoryBody(body)).toBe([ + 'Lead paragraph with linked text.', + '## Key Points', + '- Point one\n- Point two', + '> Quoted line', + 'Chart caption', + ].join('\n\n')); + }); + + it('collects deduped story media links from lede, attachments, and body media', () => { + const story = { + ledeImageUrl: 'https://assets.bwbx.io/lede.webp', + lede: { url: 'https://assets.bwbx.io/lede.webp' }, + socialImageUrl: 'https://assets.bwbx.io/social.png', + imageAttachments: { + one: { url: 'https://assets.bwbx.io/figure.jpg' }, + }, + body: { + content: [ + { + type: 'media', + data: { + chart: { + src: 'https://resource.bloomberg.com/images/chart.png', + fallback: 'https://assets.bwbx.io/chart-fallback.png', + }, + }, + }, + ], + }, + }; + + expect(extractStoryMediaLinks(story)).toEqual([ + 'https://assets.bwbx.io/lede.webp', + 'https://assets.bwbx.io/social.png', + 'https://assets.bwbx.io/figure.jpg', + 'https://resource.bloomberg.com/images/chart.png', + 'https://assets.bwbx.io/chart-fallback.png', + ]); + }); +}); diff --git a/src/clis/bloomberg/utils.ts b/src/clis/bloomberg/utils.ts new file mode 100644 index 0000000..67a3c5a --- /dev/null +++ b/src/clis/bloomberg/utils.ts @@ -0,0 +1,429 @@ +import { CliError } from '../../errors.js'; + +export const BLOOMBERG_FEEDS = { + main: 'https://feeds.bloomberg.com/news.rss', + markets: 'https://feeds.bloomberg.com/markets/news.rss', + economics: 'https://feeds.bloomberg.com/economics/news.rss', + industries: 'https://feeds.bloomberg.com/industries/news.rss', + tech: 'https://feeds.bloomberg.com/technology/news.rss', + politics: 'https://feeds.bloomberg.com/politics/news.rss', + businessweek: 'https://feeds.bloomberg.com/businessweek/news.rss', + opinions: 'https://feeds.bloomberg.com/bview/news.rss', +} as const; + +const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; opencli)'; + +export type BloombergFeedName = keyof typeof BLOOMBERG_FEEDS; + +export interface BloombergFeedItem { + title: string; + summary: string; + link: string; + mediaLinks: string[]; +} + +export interface BloombergStory { + headline?: string; + summary?: string; + url?: string; + body?: any; + lede?: any; + ledeImageUrl?: string; + socialImageUrl?: string; + imageAttachments?: Record; + videoAttachments?: Record; +} + +export async function fetchBloombergFeed(name: BloombergFeedName, limit: number = 1): Promise { + const feedUrl = BLOOMBERG_FEEDS[name]; + if (!feedUrl) { + throw new CliError('ARGUMENT', `Unknown Bloomberg feed: ${name}`); + } + + const resp = await fetch(feedUrl, { + headers: { 'User-Agent': DEFAULT_USER_AGENT }, + }); + + if (!resp.ok) { + throw new CliError( + 'FETCH_ERROR', + `Bloomberg RSS HTTP ${resp.status}`, + 'Bloomberg may be temporarily unavailable; try again later.', + ); + } + + const xml = await resp.text(); + const items = parseBloombergRss(xml); + if (!items.length) { + throw new CliError( + 'NOT_FOUND', + 'Bloomberg RSS feed returned no items', + 'Bloomberg may have changed the feed format.', + ); + } + + const count = Math.max(1, Math.min(Number(limit) || 1, 20)); + return items.slice(0, count); +} + +export function parseBloombergRss(xml: string): BloombergFeedItem[] { + const items: BloombergFeedItem[] = []; + const itemRegex = /]*>([\s\S]*?)<\/item>/gi; + let match: RegExpExecArray | null; + + while ((match = itemRegex.exec(xml))) { + const block = match[1]; + const title = extractTagText(block, 'title'); + const summary = extractTagText(block, 'description'); + const link = extractTagText(block, 'link') || extractTagText(block, 'guid'); + const mediaLinks = extractMediaLinksFromRssItem(block); + + if (!title || !link) continue; + items.push({ + title, + summary, + link, + mediaLinks, + }); + } + + return items; +} + +export function normalizeBloombergLink(input: string): string { + const raw = String(input || '').trim(); + if (!raw) { + throw new CliError('ARGUMENT', 'A Bloomberg link is required'); + } + if (raw.startsWith('/')) return `https://www.bloomberg.com${raw}`; + return raw; +} + +export function validateBloombergLink(input: string): string { + const normalized = normalizeBloombergLink(input); + let url: URL; + try { + url = new URL(normalized); + } catch { + throw new CliError( + 'ARGUMENT', + `Invalid Bloomberg link: ${input}`, + 'Pass a full https://www.bloomberg.com/... URL or a relative Bloomberg path.', + ); + } + + if (!/(?:\.|^)bloomberg\.com$/i.test(url.hostname)) { + throw new CliError( + 'ARGUMENT', + `Expected a bloomberg.com link, got: ${url.hostname}`, + 'Pass a Bloomberg article URL from bloomberg.com.', + ); + } + + return url.toString(); +} + +export function renderStoryBody(body: any): string { + const blocks = Array.isArray(body?.content) ? body.content : []; + const parts = blocks + .map((block: any) => renderBlock(block, 0)) + .map((part: string) => normalizeBlockText(part)) + .filter(Boolean); + + return parts.join('\n\n').replace(/\n{3,}/g, '\n\n').trim(); +} + +export function extractStoryMediaLinks(story: BloombergStory): string[] { + const urls = new Set(); + collectMediaUrls(story?.ledeImageUrl, urls); + collectMediaUrls(story?.socialImageUrl, urls); + collectMediaUrls(story?.lede, urls); + collectMediaUrls(story?.imageAttachments, urls); + collectMediaUrls(story?.videoAttachments, urls); + + const mediaBlocks = Array.isArray(story?.body?.content) + ? story.body.content.filter((block: any) => block?.type === 'media') + : []; + collectMediaUrls(mediaBlocks, urls); + + return [...urls]; +} + +function renderBlock(block: any, depth: number): string { + if (!block || typeof block !== 'object') return ''; + + switch (block.type) { + case 'paragraph': + return renderInlineNodes(block.content || []); + case 'heading': { + const text = renderInlineNodes(block.content || []); + if (!text) return ''; + const level = Number(block.data?.level ?? block.data?.weight ?? 2); + const prefix = level <= 1 ? '# ' : level === 2 ? '## ' : '### '; + return `${prefix}${text}`; + } + case 'blockquote': { + const text = renderInlineNodes(block.content || []); + if (!text) return ''; + return text.split('\n').map((line: string) => line ? `> ${line}` : '>').join('\n'); + } + case 'list': + return renderListBlock(block, depth); + case 'tabularData': + return renderTabularDataBlock(block); + case 'media': + return renderMediaBlock(block); + case 'inline-newsletter': + case 'newsletter': + case 'ad': + return ''; + default: { + if (Array.isArray(block.content) && block.content.length > 0) { + const inlineText = renderInlineNodes(block.content); + if (inlineText) return inlineText; + const nested = block.content.map((child: any) => renderBlock(child, depth + 1)).filter(Boolean); + if (nested.length) return nested.join('\n'); + } + return extractGenericText(block); + } + } +} + +function renderInlineNodes(nodes: any[]): string { + return nodes.map((node) => renderInlineNode(node)).join(''); +} + +function renderInlineNode(node: any): string { + if (node == null) return ''; + if (typeof node === 'string') return decodeXmlEntities(node); + + switch (node.type) { + case 'text': + return decodeXmlEntities(node.value || ''); + case 'linebreak': + return '\n'; + case 'link': + case 'entity': + case 'strong': + case 'emphasis': + case 'italic': + case 'underline': + case 'span': + if (Array.isArray(node.content) && node.content.length > 0) { + return renderInlineNodes(node.content); + } + return decodeXmlEntities(node.value || ''); + default: + if (Array.isArray(node.content) && node.content.length > 0) { + return renderInlineNodes(node.content); + } + if (typeof node.value === 'string') return decodeXmlEntities(node.value); + return ''; + } +} + +function renderListBlock(block: any, depth: number): string { + const items = Array.isArray(block.content) ? block.content : []; + if (!items.length) return ''; + + const listStyle = String(block.subType || block.data?.style || block.data?.listType || ''); + const ordered = /\bordered\b|\bnumber(?:ed)?\b/i.test(listStyle); + let index = 1; + + return items + .map((item: any) => { + const prefix = ordered ? `${index++}. ` : '- '; + return renderListItem(item, prefix, depth); + }) + .filter(Boolean) + .join('\n'); +} + +function renderListItem(item: any, prefix: string, depth: number): string { + const indent = ' '.repeat(depth); + const body = normalizeBlockText(renderListItemBody(item, depth + 1)); + if (!body) return ''; + + const lines = body.split('\n'); + const head = `${indent}${prefix}${lines[0]}`; + if (lines.length === 1) return head; + + const continuationIndent = `${indent}${' '.repeat(prefix.length)}`; + const tail = lines.slice(1).map((line) => `${continuationIndent}${line}`).join('\n'); + return `${head}\n${tail}`; +} + +function renderListItemBody(item: any, depth: number): string { + if (!item || typeof item !== 'object') return ''; + + if (item.type === 'list-item' && Array.isArray(item.content)) { + const parts = item.content + .map((child: any) => child?.type === 'paragraph' + ? renderInlineNodes(child.content || []) + : renderBlock(child, depth)) + .map((part: string) => normalizeBlockText(part)) + .filter(Boolean); + return parts.join('\n'); + } + + return renderBlock(item, depth); +} + +function renderTabularDataBlock(block: any): string { + const rows = block?.data?.rows ?? block?.data?.table?.rows ?? block?.content; + if (!Array.isArray(rows) || !rows.length) { + return extractGenericText(block.data || block.content || block); + } + + const lines = rows + .map((row: any) => extractGenericText(row)) + .map((line) => normalizeBlockText(line)) + .filter(Boolean); + + return lines.join('\n'); +} + +function renderMediaBlock(block: any): string { + const candidates = [ + block?.data?.chart?.caption, + block?.data?.attachment?.caption, + block?.data?.attachment?.title, + block?.data?.attachment?.subtitle, + block?.data?.video?.caption, + ]; + + const caption = candidates + .map((value) => normalizeBlockText(stripHtml(String(value || '')))) + .find(Boolean); + + return caption || ''; +} + +function extractGenericText(value: any): string { + const parts: string[] = []; + collectText(value, parts); + return parts.join(' ').replace(/\s+/g, ' ').trim(); +} + +function collectText(value: any, out: string[]): void { + if (value == null) return; + if (typeof value === 'string') { + const text = normalizeBlockText(stripHtml(decodeXmlEntities(value))); + if (text) out.push(text); + return; + } + if (Array.isArray(value)) { + for (const item of value) collectText(item, out); + return; + } + if (typeof value === 'object') { + if (typeof value.value === 'string') { + const text = normalizeBlockText(stripHtml(decodeXmlEntities(value.value))); + if (text) out.push(text); + return; + } + if (Array.isArray(value.content)) { + collectText(value.content, out); + return; + } + for (const entry of Object.values(value)) collectText(entry, out); + } +} + +function extractTagText(block: string, tag: string): string { + const safeTag = escapeRegExp(tag); + const match = block.match(new RegExp(`<${safeTag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${safeTag}>`, 'i')); + if (!match) return ''; + return normalizeBlockText(stripHtml(decodeXmlEntities(stripCdata(match[1])))); +} + +function extractMediaLinksFromRssItem(block: string): string[] { + const links = new Set(); + const mediaRegex = /<(?:media:content|media:thumbnail|enclosure)\b[^>]*\burl="([^"]+)"[^>]*>/gi; + let match: RegExpExecArray | null; + while ((match = mediaRegex.exec(block))) { + const url = decodeXmlEntities(match[1] || '').trim(); + if (url) links.add(url); + } + return [...links]; +} + +function collectMediaUrls(value: any, out: Set, seen = new WeakSet()): void { + if (value == null) return; + + if (typeof value === 'string') { + const normalized = normalizeMediaUrl(value); + if (normalized) out.add(normalized); + return; + } + + if (Array.isArray(value)) { + for (const item of value) collectMediaUrls(item, out, seen); + return; + } + + if (typeof value === 'object') { + if (seen.has(value)) return; + seen.add(value); + + for (const key of ['url', 'src', 'fallback', 'poster']) { + const candidate = (value as Record)[key]; + if (typeof candidate === 'string') { + const normalized = normalizeMediaUrl(candidate); + if (normalized) out.add(normalized); + } + } + + for (const entry of Object.values(value)) { + collectMediaUrls(entry, out, seen); + } + } +} + +function normalizeMediaUrl(value: string): string | null { + const url = decodeXmlEntities(String(value || '')).trim(); + if (!/^https?:\/\//i.test(url)) return null; + if (!looksLikeMediaUrl(url)) return null; + return url; +} + +function looksLikeMediaUrl(url: string): boolean { + return /(?:assets\.bwbx\.io|resource\.bloomberg\.com|media\.bloomberg\.com)/i.test(url) + || /\.(?:jpg|jpeg|png|webp|gif|svg|mp4|m3u8)(?:[?#].*)?$/i.test(url); +} + +function stripCdata(value: string): string { + const match = value.match(/^$/); + return match ? match[1] : value; +} + +function stripHtml(value: string): string { + return String(value || '').replace(/<[^>]+>/g, ' '); +} + +function decodeXmlEntities(value: string): string { + return String(value || '') + .replace(//g, '$1') + .replace(/&#(\d+);/g, (_m, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_m, code) => String.fromCodePoint(parseInt(code, 16))) + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/ /g, ' '); +} + +function normalizeBlockText(value: string): string { + return String(value || '') + .replace(/\r/g, '') + .replace(/[ \t]+\n/g, '\n') + .replace(/\n[ \t]+/g, '\n') + .replace(/[ \t]{2,}/g, ' ') + .trim(); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/tests/e2e/browser-public.test.ts b/tests/e2e/browser-public.test.ts index 55988e0..67b477f 100644 --- a/tests/e2e/browser-public.test.ts +++ b/tests/e2e/browser-public.test.ts @@ -46,6 +46,31 @@ describe('browser public-data commands E2E', () => { } }, 60_000); + it('bloomberg news returns article detail when the article page is accessible', async () => { + const feedResult = await runCli(['bloomberg', 'tech', '--limit', '1', '-f', 'json']); + if (feedResult.code !== 0) { + console.warn('bloomberg news: skipped — could not load Bloomberg tech feed'); + return; + } + + const feedItems = parseJsonOutput(feedResult.stdout); + const link = Array.isArray(feedItems) ? feedItems[0]?.link : null; + if (!link) { + console.warn('bloomberg news: skipped — tech feed returned no link'); + return; + } + + const data = await tryBrowserCommand(['bloomberg', 'news', link, '-f', 'json']); + expectDataOrSkip(data, 'bloomberg news'); + if (data) { + expect(data[0]).toHaveProperty('title'); + expect(data[0]).toHaveProperty('summary'); + expect(data[0]).toHaveProperty('link'); + expect(data[0]).toHaveProperty('mediaLinks'); + expect(data[0]).toHaveProperty('content'); + } + }, 60_000); + // ── v2ex daily (browser: true) ── it('v2ex daily returns topics', async () => { const data = await tryBrowserCommand(['v2ex', 'daily', '--limit', '3', '-f', 'json']); diff --git a/tests/e2e/public-commands.test.ts b/tests/e2e/public-commands.test.ts index f9fb601..fc5ca22 100644 --- a/tests/e2e/public-commands.test.ts +++ b/tests/e2e/public-commands.test.ts @@ -15,6 +15,51 @@ function isExpectedChineseSiteRestriction(code: number, stderr: string): boolean const isExpectedXiaoyuzhouRestriction = isExpectedChineseSiteRestriction; describe('public commands E2E', () => { + // ── bloomberg (RSS-backed, browser: false) ── + it('bloomberg main returns structured headline data', async () => { + const { stdout, code } = await runCli(['bloomberg', 'main', '--limit', '1', '-f', 'json']); + expect(code).toBe(0); + const data = parseJsonOutput(stdout); + expect(Array.isArray(data)).toBe(true); + expect(data.length).toBe(1); + expect(data[0]).toHaveProperty('title'); + expect(data[0]).toHaveProperty('summary'); + expect(data[0]).toHaveProperty('link'); + expect(data[0]).toHaveProperty('mediaLinks'); + expect(Array.isArray(data[0].mediaLinks)).toBe(true); + }, 30_000); + + it.each(['markets', 'economics', 'industries', 'tech', 'politics', 'businessweek', 'opinions'])( + 'bloomberg %s returns structured RSS items', + async (section) => { + const { stdout, code } = await runCli(['bloomberg', section, '--limit', '1', '-f', 'json']); + expect(code).toBe(0); + const data = parseJsonOutput(stdout); + expect(Array.isArray(data)).toBe(true); + expect(data.length).toBe(1); + expect(data[0]).toHaveProperty('title'); + expect(data[0]).toHaveProperty('summary'); + expect(data[0]).toHaveProperty('link'); + expect(data[0]).toHaveProperty('mediaLinks'); + }, + 30_000, + ); + + it('bloomberg feeds lists the supported RSS aliases', async () => { + const { stdout, code } = await runCli(['bloomberg', 'feeds', '-f', 'json']); + expect(code).toBe(0); + const data = parseJsonOutput(stdout); + expect(Array.isArray(data)).toBe(true); + expect(data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'main' }), + expect.objectContaining({ name: 'markets' }), + expect.objectContaining({ name: 'tech' }), + expect.objectContaining({ name: 'opinions' }), + ]), + ); + }, 30_000); + // ── apple-podcasts ── it('apple-podcasts search returns structured podcast results', async () => { const { stdout, code } = await runCli(['apple-podcasts', 'search', 'technology', '--limit', '3', '-f', 'json']);