gungun88
ca102a3398
数据源 - LowEndTalk Offers RSS（含完整正文） - LowEndBox RSS（按 category + 正文规格密度过滤非优惠文章） - 跨源去重（厂商+价格+配置指纹；数据不足时拒绝合并） 抽取内核（确定性正则，无 LLM） - 规格/价格/折扣/活动性质/地点/CPU 型号 - 拍平 HTML 表格重建（按列序位置推断，解决裸容量无语义问题） - 库存判定：商家回帖表态 > 帖文关键词 > 用户提及（弱证据） - 语境校验：疑问句/否定句/非库存对象/促销资格四类误判防线 - 末页优先抓取回帖（实测商家表态集中在回帖流 75%-100% 位置） 工程化 - 包结构分工明确，配置支持环境变量（可绑自有域名） - 124 个 pytest 用例；BERO 十周年帖为黄金样本 - GitHub Actions 每 6 小时抓取 + 定时提交 + 可选 rsync 部署 - 两种部署方案（Actions+自有服务器 / 纯服务器 cron），均含 「记录数为 0 则拒绝发布」安全网 - legacy/ 保留重构前脚本，供回归比对（19 字段零回归差异） 零运行时依赖（仅 Python 标准库）；产物为纯静态文件。
91 lines
2.9 KiB
Python
91 lines
2.9 KiB
Python
"""带重试与退避的 HTTP 抓取。
|
||
|
||
比原型的裸 urlopen 多了：
|
||
· 自动重试（网络抖动 / 5xx / Cloudflare 偶发 403）
|
||
· 指数退避，避免被当成攻击
|
||
· gzip 解压
|
||
· 明确区分「永久失败(404)」和「可重试失败」
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import gzip
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
|
||
from . import config
|
||
|
||
# 这些状态码重试无意义
|
||
PERMANENT = {400, 401, 404, 410, 451}
|
||
|
||
|
||
class FetchError(Exception):
|
||
"""抓取失败（已耗尽重试）。"""
|
||
|
||
def __init__(self, url: str, reason: str, status: int | None = None):
|
||
self.url = url
|
||
self.reason = reason
|
||
self.status = status
|
||
super().__init__(f"{url}: {reason}" + (f" (HTTP {status})" if status else ""))
|
||
|
||
|
||
def _decode(raw: bytes, encoding_header: str | None) -> str:
|
||
if encoding_header == "gzip":
|
||
try:
|
||
raw = gzip.decompress(raw)
|
||
except Exception:
|
||
pass
|
||
return raw.decode("utf-8", errors="replace")
|
||
|
||
|
||
def fetch(
|
||
url: str,
|
||
*,
|
||
accept: str = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
timeout: int | None = None,
|
||
retries: int | None = None,
|
||
verbose: bool = False,
|
||
) -> str | None:
|
||
"""抓取 URL 并返回文本。
|
||
|
||
返回 None 表示**永久失败**（如 404）——调用方应据此跳过而非重试。
|
||
抛 FetchError 表示**可重试失败已耗尽**。
|
||
"""
|
||
timeout = timeout or config.HTTP_TIMEOUT
|
||
retries = config.HTTP_RETRIES if retries is None else retries
|
||
|
||
last_reason = "unknown"
|
||
last_status: int | None = None
|
||
|
||
for attempt in range(retries + 1):
|
||
req = urllib.request.Request(url, headers={
|
||
"User-Agent": config.USER_AGENT,
|
||
"Accept": accept,
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
"Accept-Encoding": "gzip, deflate",
|
||
"Connection": "keep-alive",
|
||
})
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
raw = resp.read()
|
||
return _decode(raw, resp.headers.get("Content-Encoding"))
|
||
except urllib.error.HTTPError as e:
|
||
last_status = e.code
|
||
last_reason = f"HTTP {e.code} {e.reason}"
|
||
if e.code in PERMANENT:
|
||
return None # 永久失败，不重试
|
||
except urllib.error.URLError as e:
|
||
last_reason = f"URLError {e.reason}"
|
||
except Exception as e: # noqa: BLE001 - 网络层兜底
|
||
last_reason = f"{type(e).__name__} {e}"
|
||
|
||
if attempt < retries:
|
||
delay = config.HTTP_BACKOFF ** attempt
|
||
if verbose:
|
||
print(f" [retry {attempt + 1}/{retries}] {last_reason} -> "
|
||
f"{delay:.1f}s 后重试")
|
||
time.sleep(delay)
|
||
|
||
raise FetchError(url, last_reason, last_status)
|