shy3130
ecb49cbaca
合并 0.3.6 分支全部内容 (36 提交, 详细历史见 0.3.6 分支): ■ 数据与插件 - TdxQuant 数据源: 通达信量化接入 (实时/五档/日K/复权/分钟), tdxaidata 收进默认依赖 - 扩展表两级拉取配方: 概念/行业出厂即带完整可拉配置, 可作自定义接入参考示例 - 能力门禁日志带生效源; 标的池解析只走 instruments 维表 ■ 页面扩展化 - 11 个官方页面全量迁移为单文件扩展 (custom/*/extension.tsx), 删目录即卸载 - 运行时页面包系统: 打包产物上传即挂载, 免重建宿主 - 新装用户扩展页默认隐藏, 老用户升级菜单无感 ■ AI - Responses 协议接入; 会话历史与长任务跨设备同步 (服务端注册表) - 助手头部快捷入口 + 悬浮球拖拽 zoom 坐标系修复 + 系统设置开关 ■ 界面 - 设置「关于」子菜单: 版权/更新/打赏 + 开源许可弹窗与合规声明双保险 - 侧栏两态收起、数据/AI 助手快捷行 (同步呼吸光晕)、核心/扩展 ext 标记 - 界面缩放默认 90%; 菜单设置徽标 Swiss 化; 策略池「叠加」分组 - 情绪评分口径可配置; 财务画像 5 张子表手动拉取 ■ 文档与其他 - README 重构「核心+扩展」主线 + 架构图重画; 开放端点数核实为 65 - 版本号 bump 0.3.6; tdx 物化库测试夹具补 macOS dylib
1439 lines
55 KiB
Python
1439 lines
55 KiB
Python
"""扩展数据 API — CRUD + 文件上传 + JSON 写入 + 定时拉取 + schema 发现。"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
import shutil
|
||
import tempfile
|
||
import time
|
||
from datetime import date, datetime
|
||
from pathlib import Path
|
||
from typing import Literal
|
||
|
||
import polars as pl
|
||
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
|
||
from pydantic import BaseModel, Field
|
||
|
||
from app.market_time import CN_TZ, cn_today
|
||
from app.services.ext_data import (
|
||
ExtConfig,
|
||
ExtConfigStore,
|
||
ExtField,
|
||
PullConfig,
|
||
apply_config_mapping,
|
||
detect_symbol_candidates,
|
||
ensure_utf8_csv,
|
||
ext_api_key_field,
|
||
fix_symbol_format,
|
||
get_ext_api_key,
|
||
infer_fields_from_df,
|
||
parse_upload_file,
|
||
rows_to_parquet,
|
||
write_ext_parquet,
|
||
)
|
||
from app.services.ext_pull import _request_json, fetch_and_ingest, pull_scheduler
|
||
|
||
logger = logging.getLogger(__name__)
|
||
router = APIRouter(prefix="/api/ext-data", tags=["ext-data"])
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Pydantic 模型
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class FieldDef(BaseModel):
|
||
name: str
|
||
dtype: str = "string" # string | int | float | bool
|
||
label: str = ""
|
||
|
||
|
||
class CreateExtReq(BaseModel):
|
||
id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_]+$")
|
||
label: str = Field(..., min_length=1, max_length=64)
|
||
mode: Literal["snapshot", "timeseries"]
|
||
fields: list[FieldDef] = Field(..., min_length=1)
|
||
description: str = ""
|
||
symbol_map: dict = {} # {"type": "mapped", "col": "..."} 或 {"type": "computed", "from": "code", "method": "append_exchange"}
|
||
code_map: dict = {} # {"type": "mapped", "col": "..."} 或 {"type": "computed", "from": "symbol", "method": "strip_exchange"}
|
||
market_level: bool = False # 市场级表: 行=全市场每日一条, 无 symbol 列 (如择时状态序列)
|
||
|
||
|
||
class UpdateExtReq(BaseModel):
|
||
label: str | None = None
|
||
fields: list[FieldDef] | None = None
|
||
description: str | None = None
|
||
symbol_map: dict | None = None
|
||
code_map: dict | None = None
|
||
market_level: bool | None = None
|
||
|
||
|
||
class IngestReq(BaseModel):
|
||
"""JSON 批量写入请求。"""
|
||
date: str | None = None # YYYY-MM-DD，不传默认今天
|
||
rows: list[dict] = Field(..., min_length=1)
|
||
|
||
|
||
class PullAuthReq(BaseModel):
|
||
"""拉取接口鉴权方式 (与自定义行情源 AuthConfig 同口径)。
|
||
|
||
Key 本体存 secrets_store (secrets.json), 不写入 config.json。
|
||
"""
|
||
type: Literal["none", "bearer", "header", "query"] = "none"
|
||
header: str = Field("Authorization", min_length=1, max_length=64) # bearer/header 用
|
||
param: str = Field("token", min_length=1, max_length=64) # query 用
|
||
|
||
|
||
class PullConfigReq(BaseModel):
|
||
"""定时拉取配置请求。"""
|
||
url: str = Field(..., min_length=1)
|
||
method: str = "GET"
|
||
headers: dict[str, str] | None = None
|
||
body: str | None = None
|
||
response_path: str = "" # dot-path to rows array
|
||
field_map: dict[str, str] | None = None # external → internal field name
|
||
schedule_minutes: int = Field(1440, ge=1)
|
||
enabled: bool = False
|
||
time_window_start: str | None = None # "HH:MM", None=不限
|
||
time_window_end: str | None = None # "HH:MM", None=不限
|
||
# 接口按日查询的参数名 (如 "date"): 配置后支持历史回补, 且当日拉取也带日期参数
|
||
date_param: str | None = Field(None, min_length=1, max_length=16, pattern=r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||
# 日期参数值的格式: iso/compact/ts_s/ts_ms (时间戳=该交易日北京时间 00:00:00), 缺省 iso
|
||
date_format: str = "iso"
|
||
# 日内序列表时间列名 (字段映射后的列名, 如 "ts"): 配置后同 symbol 允许多行,
|
||
# 按 [symbol, time_field] 去重; 留空 = 每日快照表 (按 symbol 去重)
|
||
time_field: str | None = Field(None, max_length=32)
|
||
# 鉴权方式; 请求中缺省 (None) = 保留现有配置, {"type":"none"} = 关闭鉴权
|
||
auth: PullAuthReq | None = None
|
||
# 单次拉取请求超时 (秒), 默认 30 与历史行为一致; 大响应接口可调高
|
||
timeout_seconds: int = Field(30, ge=5, le=300)
|
||
# 两级拉取配方 (目录 → 逐条目明细); 缺省 (None) = 保留现有配置
|
||
fanout: dict | None = None
|
||
|
||
|
||
class ApiKeyReq(BaseModel):
|
||
"""设置拉取接口 API Key; 空串 = 清除。"""
|
||
key: str = Field(..., max_length=4096)
|
||
|
||
|
||
class DetectUrlReq(BaseModel):
|
||
"""URL 探测请求，不依赖已存在的扩展配置。"""
|
||
url: str = Field(..., min_length=1)
|
||
method: str = "GET"
|
||
headers: dict[str, str] | None = None
|
||
body: str | None = None
|
||
response_path: str = ""
|
||
field_map: dict[str, str] | None = None
|
||
# 探测超时; 与 PullConfig.timeout_seconds 同口径 (默认 30 与历史行为一致),
|
||
# 大响应接口 (如全量集合竞价 /day ~77s) 探测时需要更高超时
|
||
timeout_seconds: int = Field(30, ge=5, le=300)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 辅助
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _store(request: Request) -> ExtConfigStore:
|
||
return ExtConfigStore(request.app.state.repo.store.data_dir)
|
||
|
||
|
||
def _data_dir(request: Request) -> Path:
|
||
return request.app.state.repo.store.data_dir
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CRUD
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _apply_mapping(df: pl.DataFrame, config: ExtConfig, data_dir: Path) -> pl.DataFrame:
|
||
return apply_config_mapping(df, config, data_dir)
|
||
|
||
|
||
def _clean_col_names(df: pl.DataFrame) -> pl.DataFrame:
|
||
"""清洗列名：去掉所有 (...) 及其内容，避免时间戳导致列名不稳定。"""
|
||
import re
|
||
renames = {col: re.sub(r"\([^)]*\)", "", col).strip() for col in df.columns}
|
||
# 去重：如果清洗后重名，加序号后缀
|
||
seen: dict[str, int] = {}
|
||
final = {}
|
||
for old, new in renames.items():
|
||
if new in seen:
|
||
seen[new] += 1
|
||
final[old] = f"{new}_{seen[new]}"
|
||
else:
|
||
seen[new] = 0
|
||
final[old] = new
|
||
return df.rename(final)
|
||
|
||
|
||
_DIMENSION_SEPARATOR_CLASS = r"、,，;；|/\s-"
|
||
|
||
|
||
def _filter_dimension_member_rows(df: pl.DataFrame, field: str, value: str) -> pl.DataFrame:
|
||
"""按分隔后的完整标签匹配成员，避免“人工智能”误命中“人工智能体”。"""
|
||
if field not in df.columns:
|
||
raise HTTPException(400, f"字段 '{field}' 不存在")
|
||
normalized = value.strip()
|
||
if not normalized:
|
||
raise HTTPException(400, "标签值不能为空")
|
||
pattern = rf"(^|[{_DIMENSION_SEPARATOR_CLASS}]){re.escape(normalized)}($|[{_DIMENSION_SEPARATOR_CLASS}])"
|
||
return df.filter(
|
||
pl.col(field)
|
||
.cast(pl.String, strict=False)
|
||
.fill_null("")
|
||
.str.contains(pattern)
|
||
)
|
||
|
||
|
||
def _ext_data_dir(config: ExtConfig, data_dir: Path) -> Path:
|
||
"""返回扩展数据的数据目录。
|
||
|
||
- snapshot: data/ext_data/{id}/（part.parquet 与 config.json 同级）
|
||
- timeseries: data/ext_data/{id}/timeseries/
|
||
"""
|
||
cfg_dir = data_dir / "ext_data" / config.id
|
||
if config.mode == "timeseries":
|
||
return cfg_dir / "timeseries"
|
||
return cfg_dir
|
||
|
||
|
||
def _parquet_glob(config: ExtConfig, data_dir: Path) -> str:
|
||
"""返回该扩展配置下所有 parquet 文件的 glob 模式。
|
||
|
||
snapshot: 'data/ext_data/{id}/*.parquet'（只有 part.parquet）
|
||
timeseries: 'data/ext_data/{id}/timeseries/**/*.parquet'
|
||
"""
|
||
cfg_dir = data_dir / "ext_data" / config.id
|
||
if config.mode == "snapshot":
|
||
return str(cfg_dir / "*.parquet")
|
||
return str(cfg_dir / "timeseries" / "**" / "*.parquet")
|
||
|
||
|
||
def _safe_json_value(value):
|
||
if isinstance(value, float) and not math.isfinite(value):
|
||
return None
|
||
if isinstance(value, (date, datetime)):
|
||
return value.isoformat()
|
||
return value
|
||
|
||
|
||
def _partition_date(raw: str) -> str:
|
||
"""把 `date` 入参规范成 `YYYY-MM-DD` 分区名。
|
||
|
||
这个值直接拼进分区目录名 (`timeseries/date=<value>`), 所以非法值不只是格式问题:
|
||
`date=x/../../../../kline_daily` 会让读取路径离开 `ext_data/<id>/timeseries/`。
|
||
同一文件的 `/sync`、`/ingest`、`/backfill` 都先 `date.fromisoformat` 再用, 只有
|
||
`/rows` 和 `/dimension-members` 走的这条路把原始字符串直接拼进了路径。
|
||
"""
|
||
try:
|
||
return date.fromisoformat(raw).isoformat()
|
||
except ValueError as e:
|
||
raise HTTPException(400, f"日期格式错误: {raw}") from e
|
||
|
||
|
||
def _read_ext_dataframe(
|
||
config: ExtConfig,
|
||
data_dir: Path,
|
||
snapshot_date: str | None = None,
|
||
start_date: str | None = None,
|
||
end_date: str | None = None,
|
||
) -> tuple[pl.DataFrame, str | None]:
|
||
"""读取扩展表数据。
|
||
|
||
- snapshot: 当前快照 (不支持日期范围)。
|
||
- timeseries + snapshot_date: 单日分区 (历史行为)。
|
||
- timeseries + start_date/end_date (可只给一端): 范围内全部分区 concat,
|
||
active_date 返回 "首..末"。date 与范围同时给出时 date 优先。
|
||
"""
|
||
cfg_dir = data_dir / "ext_data" / config.id
|
||
|
||
if config.mode == "snapshot":
|
||
if start_date or end_date:
|
||
raise HTTPException(400, "快照模式无历史分区, 不支持日期范围查询")
|
||
path = cfg_dir / "part.parquet"
|
||
if not path.exists():
|
||
return pl.DataFrame(), None
|
||
return pl.read_parquet(path), _latest_sync_date(config, data_dir)
|
||
|
||
base = cfg_dir / "timeseries"
|
||
if not base.exists():
|
||
return pl.DataFrame(), None
|
||
|
||
if snapshot_date:
|
||
day = _partition_date(snapshot_date)
|
||
path = base / f"date={day}" / "part.parquet"
|
||
if not path.exists():
|
||
return pl.DataFrame(), day
|
||
return pl.read_parquet(path), day
|
||
|
||
partitions = sorted(
|
||
d for d in base.iterdir()
|
||
if d.is_dir() and d.name.startswith("date=") and (d / "part.parquet").exists()
|
||
)
|
||
if not partitions:
|
||
return pl.DataFrame(), None
|
||
|
||
if start_date or end_date:
|
||
lo = _partition_date(start_date) if start_date else None
|
||
hi = _partition_date(end_date) if end_date else None
|
||
if lo and hi and lo > hi:
|
||
raise HTTPException(400, f"start_date {lo} 晚于 end_date {hi}")
|
||
# ISO 日期字符串字典序 == 时间序, 分区名直接比较
|
||
selected = [
|
||
d for d in partitions
|
||
if (not lo or d.name[5:] >= lo) and (not hi or d.name[5:] <= hi)
|
||
]
|
||
if not selected:
|
||
return pl.DataFrame(), None
|
||
# 每分区补行级日期列 (分区名): 表自带 date 列则用 data_date, 两者都有不再加。
|
||
# 没有它, 范围合并后消费方无法区分行来自哪一天。
|
||
frames = []
|
||
for d in selected:
|
||
f = pl.read_parquet(d / "part.parquet")
|
||
day = d.name[5:]
|
||
if "date" not in f.columns:
|
||
f = f.with_columns(pl.lit(day).alias("date"))
|
||
elif "data_date" not in f.columns:
|
||
f = f.with_columns(pl.lit(day).alias("data_date"))
|
||
frames.append(f)
|
||
# diagonal: 各分区 schema 可能不一致 (配置中途加字段 → 早期分区少列),
|
||
# 缺列自动补 null 并取超类型; 单分区时无需 concat
|
||
df = pl.concat(frames, how="diagonal") if len(frames) > 1 else frames[0]
|
||
return df, f"{selected[0].name[5:]}..{selected[-1].name[5:]}"
|
||
|
||
latest = partitions[-1]
|
||
latest_date = latest.name[5:]
|
||
return pl.read_parquet(latest / "part.parquet"), latest_date
|
||
|
||
|
||
def _with_instrument_name(df: pl.DataFrame, data_dir: Path) -> pl.DataFrame:
|
||
if df.is_empty() or "symbol" not in df.columns or "name" in df.columns:
|
||
return df
|
||
path = data_dir / "instruments" / "instruments.parquet"
|
||
if not path.exists():
|
||
return df
|
||
try:
|
||
inst = pl.read_parquet(path)
|
||
if "symbol" in inst.columns and "name" in inst.columns:
|
||
inst = inst.select(["symbol", "name"]).unique(subset=["symbol"], keep="last")
|
||
return df.join(inst, on="symbol", how="left")
|
||
except Exception:
|
||
return df
|
||
return df
|
||
|
||
|
||
def _latest_sync_date(config: ExtConfig, data_dir: Path) -> str | None:
|
||
"""扫描数据文件，返回该扩展配置的最新同步时间（北京墙钟, 含时分秒）。
|
||
|
||
- snapshot: 直接取 ext_data/{id}/part.parquet 的 mtime
|
||
- timeseries: 扫描 ext_data/{id}/timeseries/date=xxx 分区目录
|
||
|
||
用北京时间而非宿主机时钟: 前端 ExtDataStatCard 原样展示这串裸时间,
|
||
容器默认 UTC 时会比同一页拉取面板里的 pull.last_run(带时区 ISO,
|
||
浏览器按本地时区渲染)整整差一个时区。
|
||
"""
|
||
from datetime import datetime
|
||
|
||
if config.mode == "snapshot":
|
||
# 快照: part.parquet 与 config.json 同级
|
||
p = data_dir / "ext_data" / config.id / "part.parquet"
|
||
if p.exists():
|
||
ts = datetime.fromtimestamp(p.stat().st_mtime, tz=CN_TZ).strftime("%Y-%m-%d %H:%M:%S")
|
||
return ts
|
||
# 兼容旧路径
|
||
old = data_dir / "instruments_ext"
|
||
if old.exists():
|
||
return _latest_sync_from_partitions(old)
|
||
return None
|
||
|
||
# 时序: 扫描 timeseries/date=xxx
|
||
base = _ext_data_dir(config, data_dir)
|
||
if not base.exists():
|
||
# 兼容旧路径
|
||
base = data_dir / "kline_ext"
|
||
if not base.exists():
|
||
return None
|
||
return _latest_sync_from_partitions(base)
|
||
|
||
|
||
def _latest_sync_from_partitions(base: Path) -> str | None:
|
||
"""从 date=xxx 分区目录中找到最新分区的修改时间 (北京墙钟)。"""
|
||
from datetime import datetime
|
||
latest_ts: float = 0
|
||
latest_date: str | None = None
|
||
for d in base.iterdir():
|
||
if d.is_dir() and d.name.startswith("date="):
|
||
for f in d.glob("*.parquet"):
|
||
mtime = f.stat().st_mtime
|
||
if mtime > latest_ts:
|
||
latest_ts = mtime
|
||
latest_date = d.name[5:]
|
||
if latest_date and latest_ts > 0:
|
||
ts = datetime.fromtimestamp(latest_ts, tz=CN_TZ).strftime("%H:%M:%S")
|
||
return f"{latest_date} {ts}"
|
||
return latest_date
|
||
|
||
|
||
def _date_range(config: ExtConfig, data_dir: Path) -> list[str] | None:
|
||
"""返回时序型扩展数据的日期范围 [最早, 最新]。"""
|
||
if config.mode != "timeseries":
|
||
return None
|
||
base = _ext_data_dir(config, data_dir)
|
||
if not base.exists():
|
||
# 兼容旧路径
|
||
base = data_dir / "kline_ext"
|
||
if not base.exists():
|
||
return None
|
||
dates: list[str] = []
|
||
for d in base.iterdir():
|
||
if d.is_dir() and d.name.startswith("date="):
|
||
dates.append(d.name[5:])
|
||
if len(dates) < 1:
|
||
return None
|
||
dates.sort()
|
||
return [dates[0], dates[-1]]
|
||
|
||
|
||
@router.get("")
|
||
def list_configs(request: Request):
|
||
"""列出所有扩展数据配置。"""
|
||
configs = _store(request).load_all()
|
||
data_dir = _data_dir(request)
|
||
items = []
|
||
for c in configs:
|
||
d = c.to_dict()
|
||
d["latest_sync_date"] = _latest_sync_date(c, data_dir)
|
||
d["date_range"] = _date_range(c, data_dir)
|
||
items.append(d)
|
||
return {"items": items}
|
||
|
||
|
||
@router.post("")
|
||
def create_config(request: Request, body: CreateExtReq):
|
||
"""创建扩展数据配置。"""
|
||
store = _store(request)
|
||
if store.get(body.id):
|
||
raise HTTPException(400, f"配置 '{body.id}' 已存在")
|
||
config = ExtConfig(
|
||
id=body.id,
|
||
label=body.label,
|
||
mode=body.mode,
|
||
fields=[ExtField(f.name, f.dtype, f.label) for f in body.fields],
|
||
description=body.description,
|
||
symbol_map=body.symbol_map,
|
||
code_map=body.code_map,
|
||
market_level=body.market_level,
|
||
)
|
||
store.upsert(config)
|
||
_refresh_views(request)
|
||
return config.to_dict()
|
||
|
||
|
||
@router.put("/{config_id}")
|
||
def update_config(request: Request, config_id: str, body: UpdateExtReq):
|
||
"""更新扩展数据配置。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
if body.label is not None:
|
||
config.label = body.label
|
||
if body.fields is not None:
|
||
config.fields = [ExtField(f.name, f.dtype, f.label) for f in body.fields]
|
||
if body.description is not None:
|
||
config.description = body.description
|
||
if body.symbol_map is not None:
|
||
config.symbol_map = body.symbol_map
|
||
if body.code_map is not None:
|
||
config.code_map = body.code_map
|
||
if body.market_level is not None:
|
||
config.market_level = body.market_level
|
||
store.upsert(config)
|
||
_refresh_views(request)
|
||
return config.to_dict()
|
||
|
||
|
||
@router.delete("/{config_id}")
|
||
def delete_config(request: Request, config_id: str):
|
||
"""删除扩展数据配置。"""
|
||
store = _store(request)
|
||
if not store.delete(config_id):
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
# 同步清掉 secrets.json 里残留的拉取 API Key, 避免同名重建配置时误用旧 Key
|
||
from app import secrets_store
|
||
|
||
secrets_store.clear(ext_api_key_field(config_id))
|
||
_refresh_views(request)
|
||
return {"status": "deleted"}
|
||
|
||
|
||
def _apply_row_filters(df: pl.DataFrame, filters: list[str] | None) -> pl.DataFrame:
|
||
"""等值过滤: 每个条目 `field:v1|v2` (值间 OR); 多条目间 AND; 按字符串比较。
|
||
|
||
数值列的等值匹配走字符串口径 (1 → "1", 1.0 → "1.0"), 覆盖二开最常见的
|
||
"维度取值筛选"场景; 区间/模糊匹配暂不做, 留给真实需求出现。
|
||
"""
|
||
if not filters:
|
||
return df
|
||
for entry in filters:
|
||
field, sep, raw_values = entry.partition(":")
|
||
field = field.strip()
|
||
values = [v for v in (raw_values.split("|") if sep else []) if v != ""]
|
||
if not field or not values:
|
||
raise HTTPException(400, f"过滤条件格式错误 (应为 字段:值1|值2): {entry}")
|
||
if field not in df.columns:
|
||
raise HTTPException(400, f"过滤字段不存在: {field} (可用: {df.columns})")
|
||
df = df.filter(pl.col(field).cast(pl.Utf8).is_in(values))
|
||
return df
|
||
|
||
|
||
def _apply_row_sort(df: pl.DataFrame, sort: str | None) -> pl.DataFrame:
|
||
"""排序: `field` 升序 / `field:desc` 降序; 未知字段 400。"""
|
||
if not sort:
|
||
return df
|
||
field, _, direction = sort.partition(":")
|
||
field = field.strip()
|
||
descending = direction.strip().lower() == "desc"
|
||
if field not in df.columns:
|
||
raise HTTPException(400, f"排序字段不存在: {field} (可用: {df.columns})")
|
||
return df.sort(field, descending=descending, nulls_last=True)
|
||
|
||
|
||
# 模块级常量避免 B008 (参数默认值内的函数调用); 语义与行内 Query(...) 完全一致
|
||
_ROW_FILTER_QUERY = Query(None, description="过滤 条件`字段:值1|值2`, 可重复多条 (AND)")
|
||
|
||
|
||
@router.get("/{config_id}/rows")
|
||
def list_rows(
|
||
request: Request,
|
||
config_id: str,
|
||
snapshot_date: str | None = Query(None, alias="date"),
|
||
columns: str | None = Query(None, description="逗号分隔的字段列表"),
|
||
limit: int = Query(1000, ge=1, le=20000),
|
||
offset: int = Query(0, ge=0, le=1_000_000, description="分页偏移 (配合 limit/total)"),
|
||
filter: list[str] | None = _ROW_FILTER_QUERY,
|
||
sort: str | None = Query(None, description="排序字段, 降序加 :desc"),
|
||
start_date: str | None = Query(None, description="时序起始日期 YYYY-MM-DD (含)"),
|
||
end_date: str | None = Query(None, description="时序结束日期 YYYY-MM-DD (含)"),
|
||
):
|
||
"""读取扩展数据明细。
|
||
|
||
- snapshot: 返回当前快照。
|
||
- timeseries: 默认返回最新日期分区; date=YYYY-MM-DD 指定单日;
|
||
start_date/end_date 取日期范围 (可只给一端), 全部分区合并返回。
|
||
- filter/sort/offset: 服务端过滤/排序/分页, 二开方无需一次拉全量自行筛。
|
||
"""
|
||
config = _store(request).get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
data_dir = _data_dir(request)
|
||
df, active_date = _read_ext_dataframe(config, data_dir, snapshot_date, start_date, end_date)
|
||
df = _with_instrument_name(df, data_dir)
|
||
# 日内序列表: 按时间列升序, 保证分页/截断取到的是最早的盘
|
||
tf = config.pull.time_field if config.pull else None
|
||
if tf and tf in df.columns:
|
||
df = df.sort(tf)
|
||
df = _apply_row_filters(df, filter)
|
||
df = _apply_row_sort(df, sort)
|
||
total = len(df)
|
||
df = df.slice(offset, limit)
|
||
requested = [c.strip() for c in (columns or "").split(",") if c.strip()]
|
||
if requested:
|
||
keep = [c for c in ["symbol", "code", "name", *requested] if c in df.columns]
|
||
if keep:
|
||
df = df.select(list(dict.fromkeys(keep)))
|
||
|
||
rows = []
|
||
for row in df.to_dicts():
|
||
rows.append({k: _safe_json_value(v) for k, v in row.items()})
|
||
|
||
return {
|
||
"id": config.id,
|
||
"label": config.label,
|
||
"mode": config.mode,
|
||
"date": active_date,
|
||
"total": total,
|
||
"offset": offset,
|
||
"limit": limit,
|
||
"fields": [f.to_dict() for f in config.fields],
|
||
"rows": rows,
|
||
}
|
||
|
||
|
||
@router.get("/{config_id}/values")
|
||
def field_values(
|
||
request: Request,
|
||
config_id: str,
|
||
field: str = Query(..., min_length=1, description="要枚举去重的字段名"),
|
||
snapshot_date: str | None = Query(None, alias="date"),
|
||
start_date: str | None = Query(None),
|
||
end_date: str | None = Query(None),
|
||
limit: int = Query(200, ge=1, le=1000),
|
||
):
|
||
"""字段取值枚举 (去重 + 计数, 按出现次数降序)。
|
||
|
||
`/rows?filter=` 的配套端点: 二开方先取某字段的候选值做筛选下拉,
|
||
再带 filter 查明细。日期语义与 /rows 相同 (date 单日 / 范围 / 默认最新)。
|
||
"""
|
||
config = _store(request).get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
data_dir = _data_dir(request)
|
||
df, active_date = _read_ext_dataframe(config, data_dir, snapshot_date, start_date, end_date)
|
||
if field not in df.columns:
|
||
raise HTTPException(400, f"字段不存在: {field} (可用: {df.columns})")
|
||
counted = (
|
||
df.group_by(field).len().sort(["len", field], descending=[True, False]).head(limit)
|
||
)
|
||
values = [
|
||
{"value": _safe_json_value(row[field]), "count": row["len"]}
|
||
for row in counted.to_dicts()
|
||
]
|
||
return {
|
||
"id": config.id,
|
||
"field": field,
|
||
"date": active_date,
|
||
"total": df.height,
|
||
"distinct": df[field].n_unique(),
|
||
"values": values,
|
||
}
|
||
|
||
|
||
@router.get("/{config_id}/dimension-members")
|
||
def dimension_members(
|
||
request: Request,
|
||
config_id: str,
|
||
field: str = Query(..., min_length=1),
|
||
value: str = Query(..., min_length=1),
|
||
snapshot_date: str | None = Query(None, alias="date"),
|
||
limit: int = Query(1000, ge=1, le=10000),
|
||
):
|
||
"""按扩展字段的完整标签值返回成分股，不绑定具体概念/行业数据源。"""
|
||
config = _store(request).get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
data_dir = _data_dir(request)
|
||
df, active_date = _read_ext_dataframe(config, data_dir, snapshot_date)
|
||
df = _with_instrument_name(df, data_dir)
|
||
matched = _filter_dimension_member_rows(df, field, value)
|
||
total = len(matched)
|
||
|
||
columns = ["symbol", "code", "name", "股票代码", "股票简称", field]
|
||
for mapping in (config.symbol_map, config.code_map):
|
||
if isinstance(mapping, dict) and mapping.get("type") == "mapped" and mapping.get("col"):
|
||
columns.append(str(mapping["col"]))
|
||
selected = [column for column in dict.fromkeys(columns) if column in matched.columns]
|
||
if selected:
|
||
matched = matched.select(selected)
|
||
if total > limit:
|
||
matched = matched.head(limit)
|
||
|
||
symbol_columns = ["symbol", "code", "股票代码", "代码"]
|
||
name_columns = ["name", "股票简称", "名称"]
|
||
for mapping in (config.symbol_map, config.code_map):
|
||
if isinstance(mapping, dict) and mapping.get("type") == "mapped" and mapping.get("col"):
|
||
symbol_columns.append(str(mapping["col"]))
|
||
|
||
rows = []
|
||
for raw in matched.to_dicts():
|
||
row = {key: _safe_json_value(item) for key, item in raw.items()}
|
||
if not row.get("symbol"):
|
||
row["symbol"] = next((str(row[column]) for column in symbol_columns if row.get(column)), "")
|
||
if not row.get("name"):
|
||
row["name"] = next((str(row[column]) for column in name_columns if row.get(column)), "")
|
||
rows.append(row)
|
||
return {
|
||
"id": config.id,
|
||
"label": config.label,
|
||
"date": active_date,
|
||
"field": field,
|
||
"value": value.strip(),
|
||
"total": total,
|
||
"limit": limit,
|
||
"rows": rows,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 板块分时 (dimension intraday)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# 点击触发 + 60s 进程内缓存: 分钟分区是滚动底座 (minute_refresh / 盘后分钟同步),
|
||
# 不做后台预计算 — 板块基数大而单次聚合仅几十毫秒。
|
||
_DIMENSION_INTRADAY_CACHE: dict[tuple[str, str, str, str | None], tuple[float, dict]] = {}
|
||
_DIMENSION_INTRADAY_CACHE_TTL_S = 60.0
|
||
# 成分股网格化 ffill 上限: 超大板块退化为逐时间戳可得均值 (内存保护)。
|
||
_DIMENSION_INTRADAY_FFILL_CAP = 2000
|
||
|
||
|
||
def _bare_symbol_expr(col: str = "symbol") -> pl.Expr:
|
||
"""'000001.SZ' → '000001'; 已是裸代码则原样。"""
|
||
return pl.col(col).cast(pl.String).str.strip_chars().str.split(".").list.first()
|
||
|
||
|
||
def _dimension_member_bares(matched: pl.DataFrame, config: ExtConfig) -> list[str]:
|
||
"""成分股裸代码集合 (symbol 列优先级与 dimension-members 端点一致)。"""
|
||
if matched.is_empty():
|
||
return []
|
||
symbol_columns = ["symbol", "code", "股票代码", "代码"]
|
||
for mapping in (config.symbol_map, config.code_map):
|
||
if isinstance(mapping, dict) and mapping.get("type") == "mapped" and mapping.get("col"):
|
||
symbol_columns.append(str(mapping["col"]))
|
||
cols = [c for c in dict.fromkeys(symbol_columns) if c in matched.columns]
|
||
if not cols:
|
||
return []
|
||
coalesced = pl.coalesce(
|
||
[pl.col(c).cast(pl.String).str.strip_chars().str.split(".").list.first() for c in cols]
|
||
)
|
||
series = matched.select(coalesced.alias("_bare")).to_series()
|
||
return sorted({s for s in series.to_list() if s})
|
||
|
||
|
||
def _prev_daily_close(data_dir: Path, target_date: str) -> pl.DataFrame | None:
|
||
"""目标日前最近一个日K分区的收盘价 → (_bare, prev_close); 无则 None。"""
|
||
daily = data_dir / "kline_daily"
|
||
if not daily.exists():
|
||
return None
|
||
dates = sorted(
|
||
d.name[5:]
|
||
for d in daily.iterdir()
|
||
if d.is_dir() and d.name.startswith("date=") and (d / "part.parquet").exists()
|
||
)
|
||
prevs = [d for d in dates if d < target_date]
|
||
if not prevs:
|
||
return None
|
||
path = daily / f"date={prevs[-1]}" / "part.parquet"
|
||
if not path.exists():
|
||
return None
|
||
df = pl.read_parquet(path, columns=["symbol", "close"])
|
||
return (
|
||
df.with_columns(_bare_symbol_expr().alias("_bare"))
|
||
.select([pl.col("_bare"), pl.col("close").cast(pl.Float64).alias("prev_close")])
|
||
# 前收 <= 0 / 非有限视为缺失 (否则 close/ref 为 inf, 整个响应 JSON 渲染 500),
|
||
# 缺失时由调用方退化为当日首根有效分钟 close
|
||
.filter(pl.col("prev_close").is_finite() & (pl.col("prev_close") > 0))
|
||
.unique(subset=["_bare"], keep="last")
|
||
)
|
||
|
||
|
||
def _dimension_intraday_compute(
|
||
config: ExtConfig,
|
||
data_dir: Path,
|
||
field: str,
|
||
value: str,
|
||
snapshot_date: str | None,
|
||
) -> dict:
|
||
"""板块等权分时: 成分股当日分钟K逐分钟平均涨跌幅 + 全市场对照线。
|
||
|
||
口径: pct = 分钟close / ref − 1 (小数制, 与快照涨跌幅契约一致, 前端 ×100 显示),
|
||
ref 优先前一交易日日K收盘 (prev_close, 开盘跳空体现在曲线起点);
|
||
日K缺失的标的退化为当日首根分钟close (混合基准)。
|
||
停牌/无成交分钟按成分股 forward-fill 后再平均, 全市场线取逐时间戳可得均值。
|
||
"""
|
||
minute_dir = data_dir / "kline_minute"
|
||
partitions: list[str] = []
|
||
if minute_dir.exists():
|
||
partitions = sorted(
|
||
d.name[5:]
|
||
for d in minute_dir.iterdir()
|
||
if d.is_dir() and d.name.startswith("date=") and (d / "part.parquet").exists()
|
||
)
|
||
if snapshot_date:
|
||
target = snapshot_date if snapshot_date in partitions else None
|
||
else:
|
||
target = partitions[-1] if partitions else None
|
||
if not target:
|
||
return {"status": "no_data", "reason": "minute_missing", "date": snapshot_date, "points": []}
|
||
|
||
ext_df, _active = _read_ext_dataframe(config, data_dir)
|
||
if ext_df.is_empty() or field not in ext_df.columns:
|
||
return {"status": "empty", "reason": "no_members", "date": target, "points": []}
|
||
member_bares = _dimension_member_bares(_filter_dimension_member_rows(ext_df, field, value), config)
|
||
if not member_bares:
|
||
return {"status": "empty", "reason": "no_members", "date": target, "points": []}
|
||
|
||
try:
|
||
bars = pl.read_parquet(
|
||
minute_dir / f"date={target}" / "part.parquet",
|
||
columns=["symbol", "datetime", "close"],
|
||
)
|
||
except Exception as exc: # noqa: BLE001
|
||
logger.warning("dimension-intraday read minute partition failed: %s", exc)
|
||
return {"status": "no_data", "reason": "minute_schema", "date": target, "points": []}
|
||
# close <= 0 / 非有限的分钟行无效: 作基准时 pct 为 inf, 作分子时是 -100% 假跌幅
|
||
bars = bars.drop_nulls(subset=["datetime", "close"]).filter(
|
||
pl.col("close").cast(pl.Float64).is_finite() & (pl.col("close") > 0)
|
||
)
|
||
if bars.is_empty():
|
||
return {"status": "no_data", "reason": "minute_empty", "date": target, "points": []}
|
||
bars = bars.with_columns(_bare_symbol_expr().alias("_bare"))
|
||
|
||
prev = _prev_daily_close(data_dir, target)
|
||
joined = bars.join(prev, on="_bare", how="left") if prev is not None else bars.with_columns(
|
||
pl.lit(None, dtype=pl.Float64).alias("prev_close")
|
||
)
|
||
refs = joined.group_by("_bare").agg(
|
||
pl.col("prev_close").first().alias("_prev"),
|
||
pl.col("close").sort_by("datetime").first().alias("_first"),
|
||
).with_columns(pl.coalesce(["_prev", "_first"]).alias("_ref"))
|
||
n_prev = refs["_prev"].is_not_null().sum()
|
||
basis = "prev_close" if n_prev == refs.height else ("first_close" if n_prev == 0 else "mixed")
|
||
joined = (
|
||
joined.join(refs.select(["_bare", "_ref"]), on="_bare", how="left")
|
||
.with_columns((pl.col("close") / pl.col("_ref") - 1.0).alias("_pct"))
|
||
)
|
||
|
||
market = joined.group_by("datetime").agg(pl.col("_pct").mean().alias("_market"))
|
||
|
||
member_bars = joined.filter(pl.col("_bare").is_in(member_bares))
|
||
members_with_minute = member_bars["_bare"].n_unique() if not member_bars.is_empty() else 0
|
||
if members_with_minute == 0:
|
||
return {
|
||
"status": "empty", "reason": "no_member_bars", "date": target,
|
||
"member_count": len(member_bares), "members_with_minute": 0, "points": [],
|
||
}
|
||
if members_with_minute <= _DIMENSION_INTRADAY_FFILL_CAP:
|
||
# 网格化 (成分股 × 全时间轴) + 逐股 ffill: 停牌分钟冻结在最后价而非退出均值
|
||
grid = (
|
||
member_bars.select(pl.col("_bare").unique())
|
||
.join(joined.select(pl.col("datetime").unique()), how="cross")
|
||
)
|
||
member_bars = (
|
||
grid.join(member_bars.select(["_bare", "datetime", "_pct"]), on=["_bare", "datetime"], how="left")
|
||
.sort(["_bare", "datetime"])
|
||
.with_columns(pl.col("_pct").forward_fill().over("_bare"))
|
||
)
|
||
sector = member_bars.group_by("datetime").agg(pl.col("_pct").mean().alias("_sector"))
|
||
|
||
combined = market.join(sector, on="datetime", how="left").sort("datetime")
|
||
|
||
def _r4(v) -> float | None:
|
||
return round(float(v), 4) if v is not None and not (isinstance(v, float) and math.isnan(v)) else None
|
||
|
||
points = [
|
||
{
|
||
"time": row["datetime"].strftime("%H:%M"),
|
||
"sector": _r4(row["_sector"]),
|
||
"market": _r4(row["_market"]),
|
||
}
|
||
for row in combined.iter_rows(named=True)
|
||
]
|
||
return {
|
||
"status": "ok",
|
||
"date": target,
|
||
"basis": basis,
|
||
"member_count": len(member_bares),
|
||
"members_with_minute": members_with_minute,
|
||
"points": points,
|
||
}
|
||
|
||
|
||
@router.get("/{config_id}/dimension-intraday")
|
||
def dimension_intraday(
|
||
request: Request,
|
||
config_id: str,
|
||
field: str = Query(..., min_length=1),
|
||
value: str = Query(..., min_length=1),
|
||
snapshot_date: str | None = Query(None, alias="date"),
|
||
):
|
||
"""板块分时走势 (等权): 成分股 × 当日分钟K聚合; 60s 缓存, 点击触发不预计算。"""
|
||
config = _store(request).get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
cache_key = (config_id, field, value.strip(), snapshot_date)
|
||
now = time.monotonic()
|
||
hit = _DIMENSION_INTRADAY_CACHE.get(cache_key)
|
||
if hit is not None and now - hit[0] < _DIMENSION_INTRADAY_CACHE_TTL_S:
|
||
return hit[1]
|
||
|
||
payload = _dimension_intraday_compute(config, _data_dir(request), field, value, snapshot_date)
|
||
_DIMENSION_INTRADAY_CACHE[cache_key] = (now, payload)
|
||
return payload
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 文件上传
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# 扩展数据 CSV/Excel 上传上限(与自选截图 OCR 的 12MB 上限属同类保护, 见 watchlist.py)。
|
||
# 通过分块写入临时文件, 超限即拒绝, 避免 `await file.read()` 把整个文件读入内存。
|
||
_MAX_UPLOAD_BYTES = 50 * 1024 * 1024
|
||
_UPLOAD_CHUNK_BYTES = 1024 * 1024
|
||
|
||
|
||
async def _write_upload_capped(file: UploadFile, dest: Path, max_bytes: int) -> None:
|
||
"""分块把上传文件写入 dest, 累计超过 max_bytes 立即拒绝(413)。
|
||
|
||
避免一次性 `await file.read()` 把整个文件读入内存(大文件可能触发高内存占用、
|
||
进程 OOM 或服务不可用); 超限时停止继续读取与落盘。
|
||
"""
|
||
total = 0
|
||
with dest.open("wb") as f:
|
||
while True:
|
||
chunk = await file.read(_UPLOAD_CHUNK_BYTES)
|
||
if not chunk:
|
||
break
|
||
total += len(chunk)
|
||
if total > max_bytes:
|
||
raise HTTPException(413, f"文件过大(上限 {max_bytes // (1024 * 1024)}MB)")
|
||
f.write(chunk)
|
||
|
||
|
||
@router.post("/{config_id}/upload")
|
||
async def upload_data(
|
||
request: Request,
|
||
config_id: str,
|
||
file: UploadFile = File(...),
|
||
snapshot_date: str | None = None,
|
||
):
|
||
"""上传 CSV/Excel 文件写入扩展数据。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
# 校验文件后缀
|
||
suffix = Path(file.filename or "").suffix.lower()
|
||
if suffix not in (".csv", ".xlsx", ".xls"):
|
||
raise HTTPException(400, "仅支持 CSV / Excel 文件")
|
||
|
||
# 写到临时文件再解析
|
||
tmp_dir = Path(tempfile.mkdtemp())
|
||
tmp_path = tmp_dir / f"upload{suffix}"
|
||
try:
|
||
await _write_upload_capped(file, tmp_path, _MAX_UPLOAD_BYTES)
|
||
|
||
# 直接读取文件，不做列重命名
|
||
if suffix == ".csv":
|
||
df = pl.read_csv(ensure_utf8_csv(tmp_path), infer_schema_length=10000)
|
||
elif suffix in (".xlsx", ".xls"):
|
||
df = pl.read_excel(tmp_path)
|
||
else:
|
||
raise HTTPException(400, f"不支持的文件格式: {suffix}")
|
||
|
||
# 清洗列名：去掉括号内的时间戳等信息
|
||
df = _clean_col_names(df)
|
||
|
||
# 按映射关系自动生成 symbol 和 code 列
|
||
df = _apply_mapping(df, config, _data_dir(request))
|
||
except ValueError as e:
|
||
raise HTTPException(400, str(e)) from e
|
||
finally:
|
||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||
|
||
# 确保配置的字段列存在于上传数据中（symbol/code 由映射自动生成，不校验）
|
||
auto_fields = {"symbol", "code"}
|
||
config_cols = {f.name for f in config.fields} - auto_fields
|
||
missing = config_cols - set(df.columns)
|
||
if missing:
|
||
raise HTTPException(400, f"上传数据缺少字段: {', '.join(sorted(missing))}")
|
||
|
||
# 只保留配置中定义的列（包括自动生成的 symbol/code），忽略文件中多余的字段
|
||
all_config_cols = {f.name for f in config.fields}
|
||
keep = [c for c in df.columns if c in all_config_cols]
|
||
df = df.select(keep)
|
||
|
||
# 解析快照日期: 非法值 400 (与 /rows 同一校验); 缺省按北京日期落盘,
|
||
# 服务器时区不能决定分区归属 (UTC 容器北京 08:00 前会写进前一天)
|
||
snap = date.fromisoformat(_partition_date(snapshot_date)) if snapshot_date else cn_today()
|
||
|
||
rows = write_ext_parquet(df, config, _data_dir(request), snapshot_date=snap)
|
||
|
||
# 刷新 DuckDB 视图
|
||
_refresh_views(request)
|
||
|
||
return {"status": "ok", "rows": rows, "date": snap.isoformat()}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# JSON 接口写入
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@router.post("/{config_id}/ingest")
|
||
def ingest_data(request: Request, config_id: str, body: IngestReq):
|
||
"""通过 JSON 接口批量写入扩展数据。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
# 校验必填字段
|
||
configured = {f.name for f in config.fields}
|
||
required = configured - {"symbol"}
|
||
for i, row in enumerate(body.rows):
|
||
# 市场级表 (无标的列) 不强制 symbol; 标的表仍要求逐行携带
|
||
if not config.market_level and "symbol" not in row:
|
||
raise HTTPException(400, f"第 {i + 1} 行缺少 symbol 字段")
|
||
missing = required - set(row.keys())
|
||
if missing:
|
||
raise HTTPException(400, f"第 {i + 1} 行缺少字段: {', '.join(sorted(missing))}")
|
||
|
||
# 同 /upload: 非法日期 400, 缺省按北京日期落盘
|
||
snap = date.fromisoformat(_partition_date(body.date)) if body.date else cn_today()
|
||
|
||
rows_written = rows_to_parquet(body.rows, config, _data_dir(request), snapshot_date=snap)
|
||
|
||
_refresh_views(request)
|
||
|
||
return {"status": "ok", "rows": rows_written, "date": snap.isoformat()}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 定时拉取
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@router.put("/{config_id}/pull")
|
||
def configure_pull(request: Request, config_id: str, body: PullConfigReq):
|
||
"""配置（或更新）定时拉取。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
# 保留历史状态字段; auth 缺省时沿用现有配置 (关闭鉴权需显式传 {"type":"none"})
|
||
old_pull = config.pull
|
||
# auth.key_ref / fanout: UI 未暴露的键在保存时沿用旧值, 避免一次普通保存
|
||
# 就把出厂配方里的凭据引用与两级请求配方抹掉。
|
||
old_auth = (old_pull.auth or {}) if old_pull else {}
|
||
if body.auth is not None:
|
||
new_auth: dict | None = body.auth.model_dump()
|
||
if new_auth and old_auth.get("key_ref") and not new_auth.get("key_ref"):
|
||
new_auth["key_ref"] = old_auth["key_ref"]
|
||
else:
|
||
new_auth = old_pull.auth if old_pull else None
|
||
config.pull = PullConfig(
|
||
url=body.url,
|
||
method=body.method,
|
||
headers=body.headers,
|
||
body=body.body,
|
||
response_path=body.response_path,
|
||
field_map=body.field_map,
|
||
schedule_minutes=body.schedule_minutes,
|
||
enabled=body.enabled,
|
||
time_window_start=body.time_window_start,
|
||
time_window_end=body.time_window_end,
|
||
date_param=body.date_param,
|
||
date_format=body.date_format,
|
||
time_field=body.time_field,
|
||
auth=new_auth,
|
||
timeout_seconds=body.timeout_seconds,
|
||
fanout=body.fanout if body.fanout is not None else (old_pull.fanout if old_pull else None),
|
||
last_run=old_pull.last_run if old_pull else None,
|
||
last_status=old_pull.last_status if old_pull else None,
|
||
last_message=old_pull.last_message if old_pull else None,
|
||
last_rows=old_pull.last_rows if old_pull else None,
|
||
)
|
||
store.upsert(config)
|
||
|
||
# 刷新调度器
|
||
pull_scheduler.refresh(_data_dir(request))
|
||
|
||
# 关闭定时拉取时清理残留的 next_run, 避免前端展示一个永不执行的"下次"
|
||
if not config.pull.enabled:
|
||
cleared = store.get(config_id)
|
||
if cleared and cleared.pull and cleared.pull.next_run:
|
||
cleared.pull.next_run = None
|
||
store.upsert(cleared)
|
||
|
||
return {"status": "ok", "pull": config.pull.to_dict()}
|
||
|
||
|
||
@router.get("/{config_id}/api-key")
|
||
def get_pull_api_key(request: Request, config_id: str):
|
||
"""查询拉取接口 API Key 状态。只返回脱敏值, 不返回明文。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
from app import secrets_store
|
||
|
||
key = get_ext_api_key(config_id)
|
||
return {"key_set": bool(key), "masked_key": secrets_store.mask(key) if key else ""}
|
||
|
||
|
||
@router.put("/{config_id}/api-key")
|
||
def set_pull_api_key(request: Request, config_id: str, body: ApiKeyReq):
|
||
"""设置 (或空串清除) 拉取接口的 API Key, 存 secrets.json (权限 0600)。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
from app import secrets_store
|
||
|
||
value = body.key.strip()
|
||
if value:
|
||
secrets_store.save({ext_api_key_field(config_id): value})
|
||
else:
|
||
secrets_store.clear(ext_api_key_field(config_id))
|
||
return {"status": "ok", "key_set": bool(value), "masked_key": secrets_store.mask(value) if value else ""}
|
||
|
||
|
||
@router.post("/{config_id}/pull/test")
|
||
async def test_pull(request: Request, config_id: str):
|
||
"""测试拉取：请求外部 API 并返回预览数据，不写入。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
if not config.pull or not config.pull.url:
|
||
raise HTTPException(400, "拉取未配置或 URL 为空")
|
||
|
||
# 复用正式拉取的请求实现 (UA 标识头 + 鉴权注入同一套口径), 不带日期参数
|
||
from app.services.ext_pull import _apply_field_map, _extract_rows
|
||
|
||
pull = config.pull
|
||
try:
|
||
data = await _request_json(pull, config.id)
|
||
rows = _extract_rows(data, pull.response_path)
|
||
preview = _apply_field_map(rows[:5], pull.field_map)
|
||
return {
|
||
"status": "ok",
|
||
"total_rows": len(rows),
|
||
"preview": preview,
|
||
"has_symbol": bool(rows and "symbol" in rows[0]),
|
||
}
|
||
except Exception as e:
|
||
raise HTTPException(400, f"测试失败: {e}") from e
|
||
|
||
|
||
@router.post("/{config_id}/pull/run")
|
||
async def run_pull(request: Request, config_id: str):
|
||
"""手动触发一次拉取并写入。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
if not config.pull or not config.pull.url:
|
||
raise HTTPException(400, "拉取未配置或 URL 为空")
|
||
|
||
try:
|
||
n, d = await fetch_and_ingest(config, _data_dir(request))
|
||
_refresh_views(request)
|
||
# 写回执行状态, 让前端"上次执行"面板立即反映
|
||
updated = store.get(config_id)
|
||
if updated and updated.pull:
|
||
from datetime import datetime, timezone
|
||
updated.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||
updated.pull.last_status = "success"
|
||
updated.pull.last_message = f"{n} rows @ {d}"
|
||
updated.pull.last_rows = n
|
||
store.upsert(updated)
|
||
return {"status": "ok", "rows": n, "date": d}
|
||
except Exception as e:
|
||
# 失败也写回状态, 记录错误信息
|
||
failed = store.get(config_id)
|
||
if failed and failed.pull:
|
||
from datetime import datetime, timezone
|
||
failed.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||
failed.pull.last_status = "error"
|
||
failed.pull.last_message = str(e)[:200]
|
||
store.upsert(failed)
|
||
raise HTTPException(400, f"拉取失败: {e}") from e
|
||
|
||
|
||
@router.post("/{config_id}/backfill")
|
||
async def backfill_history_ep(
|
||
request: Request,
|
||
config_id: str,
|
||
start: str = Query(..., description="开始日期 YYYY-MM-DD"),
|
||
end: str = Query(..., description="结束日期 YYYY-MM-DD (含)"),
|
||
):
|
||
"""历史回补: 按本地交易日逐日拉取并写入 timeseries 分区。
|
||
|
||
前提: 配置为 timeseries 模式且拉取配置了 date_param (接口支持按日期
|
||
查询)。幂等 —— 已存在的分区跳过, 失败单日不中断, 结果逐项返回。
|
||
"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
try:
|
||
start_d = date.fromisoformat(start)
|
||
end_d = date.fromisoformat(end)
|
||
except ValueError as e:
|
||
raise HTTPException(422, f"日期格式错误 (应为 YYYY-MM-DD): {e}") from e
|
||
|
||
from app.services.ext_pull import backfill_history
|
||
|
||
try:
|
||
result = await backfill_history(config, _data_dir(request), start_d, end_d)
|
||
except ValueError as e:
|
||
raise HTTPException(400, str(e)) from e
|
||
_refresh_views(request)
|
||
return {"status": "ok", **result}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ---------------------------------------------------------------------------
|
||
# Symbol 格式修复
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@router.post("/{config_id}/fix-symbol")
|
||
def fix_symbol(request: Request, config_id: str):
|
||
"""扫描已有 Parquet 数据，将 symbol 列标准化为 代码.交易所 格式。"""
|
||
store = _store(request)
|
||
config = store.get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
fixed = fix_symbol_format(config, _data_dir(request))
|
||
_refresh_views(request)
|
||
return {"status": "ok", "fixed_files": fixed}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Schema 发现
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@router.post("/detect-fields")
|
||
async def detect_fields(
|
||
request: Request,
|
||
file: UploadFile = File(...),
|
||
):
|
||
"""上传 CSV/Excel 文件，自动检测列名和类型。
|
||
|
||
返回 symbol_candidates（数据匹配 000001.SZ 格式的列）和
|
||
code_candidates（数据匹配 6位纯数字的列）。
|
||
"""
|
||
suffix = Path(file.filename or "").suffix.lower()
|
||
if suffix not in (".csv", ".xlsx", ".xls"):
|
||
raise HTTPException(400, "仅支持 CSV / Excel 文件")
|
||
|
||
tmp_dir = Path(tempfile.mkdtemp())
|
||
tmp_path = tmp_dir / f"upload{suffix}"
|
||
try:
|
||
await _write_upload_capped(file, tmp_path, _MAX_UPLOAD_BYTES)
|
||
|
||
# 直接读取，不要求 symbol 列
|
||
if suffix == ".csv":
|
||
df = pl.read_csv(ensure_utf8_csv(tmp_path), infer_schema_length=10000)
|
||
elif suffix in (".xlsx", ".xls"):
|
||
df = pl.read_excel(tmp_path)
|
||
else:
|
||
raise HTTPException(400, f"不支持的文件格式: {suffix}")
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(400, str(e)) from e
|
||
finally:
|
||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||
|
||
# 清洗列名：去掉括号内的时间戳等信息
|
||
df = _clean_col_names(df)
|
||
symbol_candidates, code_candidates = detect_symbol_candidates(df)
|
||
|
||
return {
|
||
"fields": infer_fields_from_df(df),
|
||
"rows": len(df),
|
||
"symbol_candidates": symbol_candidates,
|
||
"code_candidates": code_candidates,
|
||
}
|
||
|
||
|
||
def _find_row_arrays(data, prefix: str = "", limit: int = 8) -> list[str]:
|
||
"""自动寻找 JSON 中可能的数据数组路径。"""
|
||
found: list[str] = []
|
||
|
||
def walk(value, path: str) -> None:
|
||
if len(found) >= limit:
|
||
return
|
||
if isinstance(value, list):
|
||
if value and isinstance(value[0], dict):
|
||
found.append(path)
|
||
elif value and isinstance(value[0], list):
|
||
for i, item in enumerate(value[:3]):
|
||
walk(item, f"{path}.{i}" if path else str(i))
|
||
elif isinstance(value, dict):
|
||
for key, child in value.items():
|
||
next_path = f"{path}.{key}" if path else key
|
||
walk(child, next_path)
|
||
|
||
walk(data, prefix)
|
||
return found
|
||
|
||
|
||
@router.post("/detect-url")
|
||
async def detect_url(body: DetectUrlReq):
|
||
"""请求外部 URL，自动检测 JSON 行数据的字段和标的代码列。"""
|
||
import httpx
|
||
|
||
from app.services.ext_pull import _apply_field_map, _extract_rows
|
||
|
||
method = body.method.upper()
|
||
if method not in ("GET", "POST"):
|
||
raise HTTPException(400, "仅支持 GET / POST")
|
||
|
||
try:
|
||
async with httpx.AsyncClient(timeout=body.timeout_seconds, follow_redirects=True) as client:
|
||
headers = body.headers or {}
|
||
kwargs: dict = {"headers": headers}
|
||
if method == "POST" and body.body:
|
||
kwargs["content"] = body.body
|
||
if "content-type" not in {k.lower() for k in headers}:
|
||
kwargs["headers"]["Content-Type"] = "application/json"
|
||
resp = await client.request(method, body.url, **kwargs)
|
||
resp.raise_for_status()
|
||
data = resp.json()
|
||
except Exception as e:
|
||
raise HTTPException(400, f"URL 请求失败: {e}") from e
|
||
|
||
path_candidates = _find_row_arrays(data)
|
||
response_path = body.response_path
|
||
if not response_path:
|
||
if not path_candidates:
|
||
raise HTTPException(400, "未在响应中找到对象数组，请填写响应数据路径")
|
||
response_path = path_candidates[0]
|
||
|
||
try:
|
||
rows = _extract_rows(data, response_path)
|
||
rows = _apply_field_map(rows, body.field_map or {})
|
||
except Exception as e:
|
||
raise HTTPException(400, f"响应解析失败: {e}") from e
|
||
|
||
if not rows:
|
||
raise HTTPException(400, "提取到的行数为 0")
|
||
if not all(isinstance(row, dict) for row in rows[:200]):
|
||
raise HTTPException(400, "响应数据数组中的元素必须是对象")
|
||
|
||
sample_rows = rows[: min(len(rows), 500)]
|
||
try:
|
||
df = pl.DataFrame(sample_rows)
|
||
except Exception as e:
|
||
raise HTTPException(400, f"样例数据解析失败: {e}") from e
|
||
|
||
df = _clean_col_names(df)
|
||
symbol_candidates, code_candidates = detect_symbol_candidates(df)
|
||
preview = [
|
||
{k: _safe_json_value(v) for k, v in row.items()}
|
||
for row in df.head(10).to_dicts()
|
||
]
|
||
|
||
return {
|
||
"status": "ok",
|
||
"total_rows": len(rows),
|
||
"response_path": response_path,
|
||
"response_path_candidates": path_candidates,
|
||
"fields": infer_fields_from_df(df),
|
||
"symbol_candidates": symbol_candidates,
|
||
"code_candidates": code_candidates,
|
||
"preview": preview,
|
||
}
|
||
|
||
|
||
@router.get("/schema/{config_id}")
|
||
def discover_schema(request: Request, config_id: str):
|
||
"""发现扩展数据的实际 Parquet schema（基于已有数据）。"""
|
||
config = _store(request).get(config_id)
|
||
if not config:
|
||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||
|
||
data_dir = _data_dir(request)
|
||
glob = _parquet_glob(config, data_dir)
|
||
|
||
try:
|
||
import duckdb
|
||
rows = duckdb.query(
|
||
f"SELECT column_name, data_type FROM (DESCRIBE SELECT * FROM read_parquet('{glob}', union_by_name=true))"
|
||
).fetchall()
|
||
return {"columns": [{"name": r[0], "type": r[1]} for r in rows]}
|
||
except Exception:
|
||
# 无数据时返回配置中定义的字段
|
||
return {"columns": [f.to_dict() for f in config.fields]}
|
||
|
||
|
||
@router.get("/schema-all")
|
||
def discover_all_schemas(request: Request):
|
||
"""发现所有扩展表的 schema（用于前端动态列选择）。"""
|
||
configs = _store(request).load_all()
|
||
result = []
|
||
for config in configs:
|
||
data_dir = _data_dir(request)
|
||
glob = _parquet_glob(config, data_dir)
|
||
|
||
try:
|
||
import duckdb
|
||
cols = duckdb.query(
|
||
f"SELECT column_name, data_type FROM (DESCRIBE SELECT * FROM read_parquet('{glob}', union_by_name=true))"
|
||
).fetchall()
|
||
field_labels = {f.name: f.label for f in config.fields}
|
||
columns = [{"name": r[0], "type": r[1], "label": field_labels.get(r[0], r[0])} for r in cols]
|
||
except Exception:
|
||
columns = [f.to_dict() for f in config.fields]
|
||
|
||
result.append({
|
||
"id": config.id,
|
||
"label": config.label,
|
||
"mode": config.mode,
|
||
"columns": columns,
|
||
})
|
||
return {"items": result}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 视图刷新
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _refresh_views(request: Request) -> None:
|
||
"""重新注册 DuckDB 视图以包含新的扩展数据。"""
|
||
repo = request.app.state.repo
|
||
db = repo.store.db
|
||
d = repo.store.data_dir.as_posix()
|
||
|
||
# 注册旧路径视图（兼容）
|
||
for name, subdir in [("instruments_ext", "instruments_ext"), ("kline_ext", "kline_ext")]:
|
||
old_glob = f"{d}/{subdir}/**/*.parquet"
|
||
old_dir = Path(d) / subdir
|
||
if old_dir.exists():
|
||
sql = (
|
||
f"CREATE OR REPLACE VIEW {name} AS "
|
||
f"SELECT * FROM read_parquet('{old_glob}', union_by_name=true)"
|
||
)
|
||
try:
|
||
db.execute(sql)
|
||
except Exception:
|
||
pass
|
||
|
||
# 注册新路径视图：每个扩展表一个视图 ext_{config_id}
|
||
ext_base = Path(d) / "ext_data"
|
||
if ext_base.exists():
|
||
for cfg_dir in ext_base.iterdir():
|
||
if not cfg_dir.is_dir():
|
||
continue
|
||
cp = cfg_dir / "config.json"
|
||
if not cp.exists():
|
||
continue
|
||
try:
|
||
raw = json.loads(cp.read_text(encoding="utf-8"))
|
||
cfg_id = raw["id"]
|
||
# 检查是否有数据文件（snapshot: part.parquet, timeseries: timeseries/ 目录）
|
||
has_data = (cfg_dir / "part.parquet").exists() or (cfg_dir / "timeseries").exists()
|
||
if has_data:
|
||
view_name = f"ext_{cfg_id}"
|
||
# snapshot: part.parquet 在 cfg_dir/ 根下; timeseries: 在 timeseries/ 子目录
|
||
mode = raw.get("mode", "snapshot")
|
||
if mode == "snapshot":
|
||
glob_pattern = f"{cfg_dir.as_posix()}/*.parquet"
|
||
else:
|
||
glob_pattern = f"{cfg_dir.as_posix()}/timeseries/**/*.parquet"
|
||
sql = (
|
||
f"CREATE OR REPLACE VIEW {view_name} AS "
|
||
f"SELECT * FROM read_parquet('{glob_pattern}', union_by_name=true)"
|
||
)
|
||
db.execute(sql)
|
||
except Exception:
|
||
pass
|
||
|
||
# 扩展列已接入 enriched 帧 (compute_signals/compute_enriched_today 注入):
|
||
# repo 内存 enriched 缓存 (_enriched_cache/_etf_/_index_) 持有含旧扩展列的
|
||
# 帧, 必须一并清理, 否则写入后监控/列表仍用旧值 (服务层已清扩展帧与策略缓存)。
|
||
if hasattr(repo, "clear_cache"):
|
||
repo.clear_cache()
|