fix(data): 清除标题里的 Markdown 强调标记
问题:部分商家把 Markdown 直接写进帖子标题,标记泄漏到界面,例如 **25G Unmetered** 30% lifetime discount - Dedicated Servers **10G FULL UNMETERED ** - 3 Months Half Price! - from $87.00 影响:title 与由标题派生的 provider 各 2 处,共 4 处。 修复:在解析层新增 strip_md_markers,处理成对标记 (**x** / __x__ / ~~x~~)及未配对的裸标记,并清理多余空格。 刻意不动单个 * 与 _ —— 它们可能是正常内容(通配符、型号、 servers_guru 这类用户名),测试已覆盖这些反例。 放在解析层而非渲染层:provider 由 title 派生,在源头清理可 让两个字段同时干净,也避免后续新增字段再次踩到。 验证:146 测试通过(新增 8 个);实际数据中标记数 4 -> 0。
This commit is contained in:
1 parent
1fd4d80301
commit
27bce3f694
3 files changed
+80
-3
No files matched your search
+3
-3
@@ -18,7 +18,7 @@ import xml.etree.ElementTree as ET
|
||||
|
||||
from . import config
|
||||
from .http import fetch
|
||||
from .textutil import strip_html
|
||||
from .textutil import strip_html, strip_md_markers
|
||||
|
||||
DC_CREATOR = "{http://purl.org/dc/elements/1.1/}creator"
|
||||
CONTENT_ENCODED = "{http://purl.org/rss/1.0/modules/content/}encoded"
|
||||
@@ -68,7 +68,7 @@ def parse_let_feed(xml_text: str) -> list[dict]:
|
||||
"thread_id": thread_id,
|
||||
"source": "let",
|
||||
"slug": _slug_from_link(link),
|
||||
"title": _text(it, "title").strip(),
|
||||
"title": strip_md_markers(_text(it, "title")).strip(),
|
||||
"link": link,
|
||||
"pubDate": _text(it, "pubDate").strip(),
|
||||
"author": _text(it, DC_CREATOR).strip(),
|
||||
@@ -132,7 +132,7 @@ def parse_leb_feed(xml_text: str, *, deals_only: bool = True) -> list[dict]:
|
||||
"thread_id": f"leb-{slug}",
|
||||
"source": "leb",
|
||||
"slug": slug,
|
||||
"title": _text(it, "title").strip(),
|
||||
"title": strip_md_markers(_text(it, "title")).strip(),
|
||||
"link": link,
|
||||
"pubDate": _text(it, "pubDate").strip(),
|
||||
"author": _text(it, DC_CREATOR).strip(),
|
||||
|
||||
@@ -67,6 +67,27 @@ def strip_html(s: str, *, drop_quotes: bool = False) -> str:
|
||||
return s.strip()
|
||||
|
||||
|
||||
# 有些商家直接把 Markdown 写进帖子标题,例如
|
||||
# "**25G Unmetered** 30% lifetime discount"。
|
||||
RE_MD_PAIRED = re.compile(r"\*\*(.+?)\*\*|__(.+?)__|~~(.+?)~~", re.S)
|
||||
RE_MD_LEFTOVER = re.compile(r"\*{2,}|_{2,}|~{2,}")
|
||||
|
||||
|
||||
def strip_md_markers(s: str) -> str:
|
||||
"""去掉 Markdown 强调标记,保留其中的文字。
|
||||
|
||||
只处理**成对**标记(**x** / __x__ / ~~x~~)以及没有配对的裸标记。
|
||||
刻意不动单个 * 或 _ —— 它们可能是正常内容(型号、下划线、通配符)。
|
||||
"""
|
||||
if not s:
|
||||
return ""
|
||||
out = RE_MD_PAIRED.sub(
|
||||
lambda m: m.group(1) or m.group(2) or m.group(3) or "", s)
|
||||
out = RE_MD_LEFTOVER.sub("", out)
|
||||
out = re.sub(r"[ \t]+", " ", out)
|
||||
return out.strip()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 单位换算
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
from letdeals.textutil import (
|
||||
parse_flat_tables,
|
||||
strip_html,
|
||||
strip_md_markers,
|
||||
strip_quotes_depth_aware,
|
||||
to_gb,
|
||||
to_tb,
|
||||
@@ -162,3 +163,58 @@ def test_parse_flat_tables_requires_vcpu_and_size():
|
||||
"""只有名字没有规格时不应产出套餐。"""
|
||||
body = "\n".join(["Just A Name", "Some text here", "More text"])
|
||||
assert parse_flat_tables(body) == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Markdown 标记清理
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_strip_md_markers_bold():
|
||||
"""商家把 Markdown 写进标题,标记应被去掉但保留文字。
|
||||
|
||||
实测来自 LowEndTalk 的真实标题:
|
||||
"**25G Unmetered** 30% lifetime discount - Dedicated Servers"
|
||||
"""
|
||||
assert strip_md_markers(
|
||||
"**25G Unmetered** 30% lifetime discount"
|
||||
) == "25G Unmetered 30% lifetime discount"
|
||||
|
||||
|
||||
def test_strip_md_markers_unpaired_trailing():
|
||||
"""未配对的裸标记也要清掉(如结尾多一个 **)。"""
|
||||
assert strip_md_markers(
|
||||
"**10G FULL UNMETERED ** - 3 Months Half Price!"
|
||||
) == "10G FULL UNMETERED - 3 Months Half Price!"
|
||||
|
||||
|
||||
def test_strip_md_markers_other_forms():
|
||||
assert strip_md_markers("__bold__ text") == "bold text"
|
||||
assert strip_md_markers("~~strike~~ here") == "strike here"
|
||||
assert strip_md_markers("a **b** c __d__ e") == "a b c d e"
|
||||
|
||||
|
||||
def test_strip_md_markers_keeps_single_asterisk():
|
||||
"""单个 * 可能是正常内容(通配符、乘法),不能动。"""
|
||||
assert strip_md_markers("2 * 4 = 8") == "2 * 4 = 8"
|
||||
assert strip_md_markers("plan*") == "plan*"
|
||||
|
||||
|
||||
def test_strip_md_markers_keeps_underscores_in_words():
|
||||
"""下划线可能是型号或变量名的一部分,单个不能动。"""
|
||||
assert strip_md_markers("servers_guru deals") == "servers_guru deals"
|
||||
assert strip_md_markers("Xeon_E5_2680") == "Xeon_E5_2680"
|
||||
|
||||
|
||||
def test_strip_md_markers_noop_on_plain_text():
|
||||
plain = "Normal Title Without Markers"
|
||||
assert strip_md_markers(plain) == plain
|
||||
|
||||
|
||||
def test_strip_md_markers_empty():
|
||||
assert strip_md_markers("") == ""
|
||||
assert strip_md_markers(None) == ""
|
||||
|
||||
|
||||
def test_strip_md_markers_collapses_extra_spaces():
|
||||
"""去掉标记后不应留下多余空格。"""
|
||||
assert strip_md_markers("**A** **B**") == "A B"
|
||||
Reference in new issue
Block a user