feat: 发布 0.6.10,社区共享库 + 全球部署量(debug16)

- 本地自学习库互通:实例开启「共享中转」后,其它实例上报各自实测的 IP→应用 匿名投票(≥2 实例见过才发布),并拉取社区库合并;本地实测优先,社区只补空白
- 隐私与防污染:只含远端 IP + 应用 ID + 字节票数,不含本机终端/域名记录/自定义应用;私网、保留、组播地址剔除;上报限量、过期清理
- 中国环境以 IPv6 公网为主:IPv6 按 /64 前缀聚合共享与匹配(精确地址跨实例几乎不重复,凑不满投票)
- 全球部署量:实例每小时心跳(匿名 ID + 版本号),中转公开 /api/share/deployments(总数/在线/离线)
- 共享识别库总开关(默认打开);compose 可用 SHARE_URL / SHARE_TOKEN
- 新数据表 share_entries、share_deployments(启动自动创建)
This commit is contained in:
Su Yijun committed 2026-10-25 14:01:37 +08:00
1 parent 509b421387
commit 039054e047
15 files changed
+585 -16

No files matched your search

+6
View File
@@ -74,3 +74,9 @@ LAN_CIDRS=192.168.88.0/24
# RETENTION_LOG_DAYS=30 # 访问日志(审计)
# RETENTION_EVENT_DAYS=180 # 审计事件、操作日志
# 以上留存天数也可以在网页 系统设置 → 日志与留存 里改,网页优先
# ---------- 社区共享库(debug16) ----------
# 把本网络实测的「远端 IP → 应用」匿名投票上报给社区中转,并拉取社区库;只含远端 IP 和应用 ID,
# 不含本机终端、域名记录、自定义应用,私网地址剔除。总开关在网页 系统设置 → 外部数据源(默认打开)。
# SHARE_URL=https://<社区中转地址> # 留空不参与;中转可以是任何一个开启了「共享中转」的实例
# SHARE_TOKEN= # 中转设置了 SHARE_TOKEN 时才需要
+10
View File
@@ -1,5 +1,15 @@
# Changelog
## [0.6.10] - 2026-10-25
社区共享库 + 全球部署量(debug16)
- **本地自学习库互通共享**(debug16 第 1 条):任意实例可开启「共享中转」(公网可达),其它实例把各自实测的「远端 IP → 应用」匿名投票上报给它,中转聚合后发布社区库;所有实例拉取并合并——**本地实测优先,社区结论只补空白**。上报内容只有远端 IP、应用 ID 和字节票数,不含本机终端、域名记录、自定义应用;私网 / 保留 / 组播地址剔除。中国环境以 IPv6 公网为主:IPv6 按 /64 前缀聚合共享与匹配(精确地址在不同实例间几乎不重复,凑不满投票)。
- **全球部署量统计**(debug16 第 2 条):开启共享的实例每小时向中转发一次部署心跳(只有匿名实例 ID 和版本号);中转对外发布 `/api/share/deployments`(所有人可见):已部署总数、在线数(2 小时内有心跳)、离线数。
- **共享识别库总开关**(debug16 第 3 条):系统设置 → 外部数据源,默认**打开**;关闭后不心跳、不上报、不拉取。
- 使用:找一个公网实例在 系统设置 → 外部数据源 勾选「本实例作为共享中转」,其它实例填它的地址即可;compose 里也可用 `SHARE_URL` / `SHARE_TOKEN` 环境变量。中转可加 `SHARE_TOKEN` 令牌防滥用。
- 新数据表 `share_entries`、`share_deployments`(自动创建)。
## [0.6.9] - 2026-10-25
本地自学习库上屏:识别规则 → 识别库 新增「本地自学习」层
+1 -1
View File
@@ -1 +1 @@
__version__ = "0.6.9"
__version__ = "0.6.10"
+44 -8
View File
@@ -17,6 +17,25 @@ from sqlalchemy import select
from app.models import HostDomainDay
MAX_IPS = 200_000
# IPv6 精确地址在不同网络里几乎不重复(中国环境以 IPv6 公网为主):按 /64 前缀聚合记忆与共享
def _is_ip(s: str) -> bool:
try:
ipaddress.ip_address(s)
return True
except ValueError:
return False
def _v6key(ip: str) -> str | None:
try:
a = ipaddress.ip_address(ip)
except ValueError:
return None
if a.version != 6:
return None
return str(ipaddress.ip_network(f"{a}/64", strict=False).network_address) + "/64"
class LocalDB:
@@ -24,22 +43,22 @@ class LocalDB:
self.lock = threading.Lock()
self.version = 0
self.updated = 0 # 上次重建的时间(识别库面板显示用)
self.ips: dict[str, str] = {} # IP -> 应用 ID(按历史字节数取最多的应用)
self.ips: dict[str, str] = {} # 本地实测:IPv4 精确 / IPv6 /64 前缀 -> 应用 ID(按历史字节数取最多的应用)
self.community: dict[str, str] = {} # 社区共享(debug16):其它实例的实测结论,只补本地空白(键同上)
def load(self, db):
"""从 host_domain_day 聚合:每个 IP 取历史字节数最多的应用;调用方提交事务"""
"""从 host_domain_day 聚合:每个 IP(IPv6 聚到 /64)取历史字节数最多的应用;调用方提交事务"""
votes: dict = defaultdict(dict)
rows = db.execute(
select(HostDomainDay.domain, HostDomainDay.app, HostDomainDay.bytes)
).all()
for dom, app, b in rows:
try:
ipaddress.ip_address(dom)
except ValueError:
key = _v6key(dom) or dom if _is_ip(dom) else None
if key is None:
continue
per = votes[dom]
per = votes[key]
per[app] = per.get(app, 0) + int(b or 0)
# 只留最大的 MAX_IPS 个 IP(按总字节),避免极端环境下无限膨胀
# 只留最大的 MAX_IPS 个键(按总字节),避免极端环境下无限膨胀
top = sorted(votes.items(), key=lambda kv: -sum(kv[1].values()))[:MAX_IPS]
ips = {}
for dom, per in top:
@@ -51,12 +70,29 @@ class LocalDB:
def ip_app(self, ip: str) -> str | None:
with self.lock:
return self.ips.get(ip)
# 本地实测优先;本地没有结论的 IP 用社区共享层(debug16)
hit = self.ips.get(ip)
if hit:
return hit
p = _v6key(ip)
if p:
hit = self.ips.get(p)
if hit:
return hit
return self.community.get(p)
return self.community.get(ip)
def load_community(self, community: dict[str, str]):
"""社区共享库(debug16):其它部署实例的实测结论,只补本地没有的 IP。调用方提交事务"""
with self.lock:
self.community = dict(community)
self.version += 1
def reset(self):
"""测试用:清空内存记忆"""
with self.lock:
self.ips = {}
self.community = {}
self.version += 1
+4
View File
@@ -41,6 +41,10 @@ class Settings(BaseSettings):
adguard_url: str = "" # 自有 DNS 服务器 AdGuard Home 的地址,如 http://192.168.9.2:3000(双层 NAT 时爱快下游的 DNS)
adguard_username: str = "" # AdGuard Home 登录用户名;和密码都留空表示免认证(只在内网监听时)
adguard_password: str = ""
share_enabled: bool = True # 共享识别库总开关(debug16 第 3 条,默认打开):关掉后不心跳、不上报、不拉取
share_url: str = "" # 社区共享库的中转地址(debug16):上报本网络实测、拉取社区库;留空不参与
share_hub: bool = False # 本实例作为共享中转(需要公网可达):接收其它部署的上报并发布聚合库
share_token: str = "" # 中转的上报令牌(可选,环境变量 SHARE_TOKEN):设置了就要求上报带 X-Share-Token
oui_db: bool = True # 自动下载 IEEE 的 MAC 厂商库(OUI,识别终端类型用,每 30 天更新)
asn_db: bool = True # 自动下载 iptoasn 的 IP→ASN 库(公有领域,每天更新)
china_db: bool = True # 自动下载国内域名列表(dnsmasq-china-list)与国内 IP 段(每天更新)
+2 -1
View File
@@ -18,7 +18,8 @@ import httpx
from app.config import get_settings
KEYS = ("singbox_url", "singbox_secret", "proxy_ips", "mosdns_url", "adguard_url", "adguard_username", "adguard_password")
KEYS = ("singbox_url", "singbox_secret", "proxy_ips", "mosdns_url", "adguard_url", "adguard_username",
"adguard_password", "share_url")
def effective(db_values: dict) -> dict:
+25 -3
View File
@@ -16,7 +16,7 @@ from app.engine import engine
from app.models import Setting
from app.ros_poller import poller
from app import audit # noqa: F401 挂上审计(engine.audit)
from app.routers import admin, auth, setup, stats
from app.routers import admin, auth, setup, share as share_router, stats
from app.routers import audit as audit_router
from app.routers import compliance as compliance_router
from app.routers import people as people_router
@@ -48,7 +48,7 @@ def _on_db_ready():
(0.6.0 / 0.6.1 的主域名回填出错时,后面的外部数据源配置、自定义规则都没加载,外部数据源显示“未启用”)"""
ensure_admin() # 建管理员失败要报出来(安装向导靠这个提示)
for name, fn in (("load_state", engine.load_state), ("apply_settings", _apply_saved),
("local_db", _refresh_local)):
("local_db", _refresh_local), ("share_lib", _share_cycle)):
try:
fn()
except Exception as e: # noqa: BLE001
@@ -75,6 +75,27 @@ def _auto_update_on() -> bool:
return row is None or row.value != "0"
def _share_cycle():
"""社区共享库(debug16):心跳(部署量统计)+ 上报本地实测 + 拉取社区库。开关关闭或未配置地址时跳过"""
if not database.is_configured():
return
from app.config import get_settings
from app import share
with database.session() as db:
if not share.sharing_on(db):
return
saved = {k: (db.get(Setting, k).value if db.get(Setting, k) else None) for k in integrations.KEYS}
cfg = integrations.effective(saved)
url = cfg.get("share_url") or ""
if not url:
return
with database.session() as db:
res = share.client_cycle(engine, db, url, get_settings().share_token)
db.commit()
if res.get("merged"):
print(f"[share] 社区库已同步:上报 {res['uploaded']} 条,并入社区结论 {res['merged']} 条", flush=True)
def _auto_update_rules():
"""v2fly 规则自动更新;更新器内部保证一天最多检查一次"""
if _auto_update_on():
@@ -130,6 +151,7 @@ async def lifespan(app: FastAPI):
sched.add_job(_safe(integrations.mosdns.poll, "mosdns"), "interval", seconds=5, max_instances=1, coalesce=True)
sched.add_job(_safe(integrations.adguard.poll, "adguard"), "interval", seconds=5, max_instances=1, coalesce=True)
sched.add_job(_safe(_refresh_local, "local-db"), "interval", hours=1, max_instances=1, coalesce=True)
sched.add_job(_safe(_share_cycle, "share"), "interval", minutes=30, max_instances=1, coalesce=True)
# 识别库更新 / 过了零点后的自动重算:指纹没变时开销只有一次设置查询。启动后等 15 分钟再跑第一次,
# 第三方库、ASN、国内列表是在后台线程加载的,不能用没加载完的识别器去算历史
sched.add_job(_safe(_auto_reclassify, "auto-reclass"), "interval", hours=1, max_instances=1, coalesce=True,
@@ -189,7 +211,7 @@ async def _unhandled(request: Request, exc: Exception):
for r in (setup.router, auth.router, stats.router, admin.router, audit_router.router, people_router.router,
compliance_router.router):
compliance_router.router, share_router.router):
app.include_router(r)
+20
View File
@@ -213,3 +213,23 @@ class PersonDevice(Base):
__tablename__ = "person_devices"
host = Column(String(64), primary_key=True)
person_id = Column(Integer, nullable=False, index=True)
class ShareEntry(Base):
"""社区共享库(debug16):各部署实例上报的「远端 IP → 应用」匿名投票。
中转实例聚合:至少 2 个不同实例都见过的映射才对外发布。不含本机终端、域名记录与自定义应用。"""
__tablename__ = "share_entries"
ip = Column(String(45), primary_key=True)
app = Column(String(48), primary_key=True)
reporter = Column(String(36), primary_key=True) # 上报实例的匿名 ID(随机 UUID)
bytes = Column(BigInteger, nullable=False, default=0)
ts = Column(BigInteger, nullable=False) # 最近一次上报时间(过期清理用)
class ShareDeployment(Base):
"""社区部署心跳(debug16):实例每小时上报一次存在,中转聚合出 全球已部署 / 在线 / 离线。只含版本号,无任何网络数据"""
__tablename__ = "share_deployments"
instance = Column(String(36), primary_key=True)
version = Column(String(16))
first_seen = Column(BigInteger, nullable=False)
last_seen = Column(BigInteger, nullable=False)
+34
View File
@@ -132,7 +132,10 @@ class SettingsIn(BaseModel):
adguard_url: str | None = Field(None, max_length=255)
adguard_username: str | None = Field(None, max_length=255)
adguard_password: str | None = Field(None, max_length=255)
share_url: str | None = Field(None, max_length=255)
rules_auto_update: bool | None = None
share_enabled: bool | None = None
share_hub: bool | None = None
def _get(db, key, default=""):
@@ -161,6 +164,8 @@ def get_settings_api(db: Session = Depends(get_db)):
"day": s.retention_day_days, "domain": s.retention_domain_days},
"ros_url": s.ros_url, "ros_user": s.ros_user,
"ports": {"http": s.http_port, "ipfix": s.ipfix_port, "tzsp": s.tzsp_port},
"share_enabled": _get(db, "share_enabled", "1" if s.share_enabled else "0") == "1",
"share_hub": _get(db, "share_hub", "1" if s.share_hub else "0") == "1",
"tz": s.tz,
"database": _db_info(),
"integrations": _integrations(db),
@@ -183,6 +188,30 @@ def local_status():
return {"ips": len(local.ips), "updated": local.version}
def share_status():
"""社区共享库(debug16):本实例的角色与社区库规模(自带会话:/api/system 没有 db 参数)"""
from app import share
from app.classify.local import local
s = get_settings()
url, hub, published = getattr(s, "share_url", ""), s.share_hub, 0
if database.is_configured():
try:
with database.session() as db:
from app.models import Setting
u_row, h_row = db.get(Setting, "share_url"), db.get(Setting, "share_hub")
url = (u_row.value if u_row and u_row.value else url)
hub = (h_row.value == "1") if h_row is not None else s.share_hub
if hub:
from sqlalchemy import func, select
from app.models import ShareEntry
rows = db.execute(select(ShareEntry.ip, func.count(func.distinct(ShareEntry.reporter)))
.group_by(ShareEntry.ip)).all()
published = sum(1 for _ip, n in rows if n >= share.PUBLISH_REPORTERS)
except Exception: # noqa: BLE001 状态展示不拖垮 /api/system
pass
return {"url": url, "hub": hub, "published": published, "community": len(local.community)}
def _db_info():
eng = database.engine
if eng is None:
@@ -216,6 +245,10 @@ def put_settings(req: SettingsIn, request: Request, user: str = Depends(current_
integrations.apply({k: _get(db, k, None) for k in integrations.KEYS})
if req.rules_auto_update is not None:
_put(db, "rules_auto_update", "1" if req.rules_auto_update else "0")
if req.share_enabled is not None:
_put(db, "share_enabled", "1" if req.share_enabled else "0")
if req.share_hub is not None:
_put(db, "share_hub", "1" if req.share_hub else "0")
if req.wan_interface is not None:
_put(db, "wan_interface", req.wan_interface.strip())
poller.wan_override = req.wan_interface.strip()
@@ -252,6 +285,7 @@ def system():
"mosdns": integrations.mosdns.status(),
"adguard": integrations.adguard.status(),
"local": local_status(),
"share": share_status(),
"asn": asndb.status(),
"proxied_skipped": st["proxied_skipped"], "dns_mosdns": st["dns_mosdns"],
"generic_apps": sorted(GENERIC_APPS),
+70
View File
@@ -0,0 +1,70 @@
"""社区共享库的中转端点(debug16):中转实例接收各部署的上报并对外发布聚合库。
- POST /api/share/collect 上报本实例的自学习库(匿名投票:远端 IP → 应用)
- GET /api/share/library.json 聚合后的社区库(至少 2 个实例见过的映射才发布)
两个端点只在 系统设置里开启「共享中转」时可用;SHARE_TOKEN 环境变量可加一层令牌校验。
"""
from fastapi import APIRouter, HTTPException, Request
from app import database, share
from app.config import get_settings
router = APIRouter(prefix="/share", tags=["share"])
def _hub_on(db) -> bool:
from app.models import Setting
row = db.get(Setting, "share_hub")
return (row.value == "1") if row is not None else get_settings().share_hub
@router.post("/collect")
def collect(req: Request, body: dict):
s = get_settings()
with database.session() as db:
if not _hub_on(db):
raise HTTPException(404, "本实例未开启共享中转")
if s.share_token and req.headers.get("x-share-token") != s.share_token:
raise HTTPException(403, "共享令牌不对")
reporter = str((body or {}).get("instance") or "")[:36]
entries = (body or {}).get("entries")
if not reporter or not isinstance(entries, list):
raise HTTPException(400, "格式不对:需要 instance 和 entries")
with database.session() as db:
accepted = share.collect(db, reporter, entries)
db.commit()
return {"ok": True, "accepted": accepted}
@router.get("/library.json")
def library_json():
with database.session() as db:
if not _hub_on(db):
raise HTTPException(404, "本实例未开启共享中转")
return share.published_library(db)
@router.post("/heartbeat")
def heartbeat(req: Request, body: dict):
"""部署心跳(debug16 第 2 条):只有匿名实例 ID 和版本号,用于全球部署量统计"""
s = get_settings()
with database.session() as db:
if not _hub_on(db):
raise HTTPException(404, "本实例未开启共享中转")
if s.share_token and req.headers.get("x-share-token") != s.share_token:
raise HTTPException(403, "共享令牌不对")
reporter = str((body or {}).get("instance") or "")[:36]
if not reporter:
raise HTTPException(400, "格式不对:需要 instance")
share.heartbeat(db, reporter, str((body or {}).get("version") or ""))
db.commit()
return {"ok": True}
@router.get("/deployments")
def deployments():
"""全球部署概况(公开):已部署总数 / 在线 / 离线"""
with database.session() as db:
if not _hub_on(db):
raise HTTPException(404, "本实例未开启共享中转")
return share.deployments(db)
+190
View File
@@ -0,0 +1,190 @@
"""社区共享库(debug16):各部署实例互相共享本地自学习库的「IP → 应用」实测结论。
架构:任意实例可开启中转模式(公网可达)——其它实例每小时把自学习库上报给中转(POST /api/share/collect,
只含 远端IP→应用 的匿名投票),中转聚合后发布 GET /api/share/library.json;所有实例拉取并合并进本地
自学习库的「社区层」,本地没有实测结论的 IP 才用社区条目。
隐私与防污染:
- 上报内容只有 (远端 IP, 应用 ID, 字节票数);不含本机终端、域名记录、自定义应用;
- 私网 / 保留地址剔除;
- 中转只发布「至少 2 个不同实例都见过」的映射——单实例的私有服务不会出现在社区库里;
- 每次上报限量,中转按实例 ID 去重投票、过期清理。
"""
import ipaddress
import time
import uuid
from collections import defaultdict
from sqlalchemy import delete, func, select
from app.models import HostDomainDay, ShareDeployment, ShareEntry
MAX_UPLOAD = 50_000 # 单次上报的条数上限
MAX_REPORTER = 100_000 # 中转上每个实例最多保留的条数
PUBLISH_REPORTERS = 2 # 至少几个不同实例见过才对外发布
ENTRY_TTL = 30 * 24 * 3600 # 上报条目 30 天没再出现就清理
def instance_id(db) -> str:
"""本实例的匿名 ID:第一次生成后存在 settings 表(不上报任何可识别信息)"""
from app.models import Setting
row = db.get(Setting, "share_instance_id")
if row is not None and row.value:
return row.value
iid = str(uuid.uuid4())
row = db.get(Setting, "share_instance_id")
if row is None:
db.add(Setting(key="share_instance_id", value=iid))
else:
row.value = iid
return iid
def _is_shareable_app(classifier, app: str) -> bool:
"""自定义应用(每家不一样)和未识别不共享"""
info = classifier.apps.get(app)
return info is not None and not info.custom and app != "other"
def _shareable_ip(ip: str) -> bool:
"""只共享公网远端地址:私网 / 保留 / 组播 / 回环不出门"""
try:
a = ipaddress.ip_address(ip)
except ValueError:
return False
return not (a.is_private or a.is_reserved or a.is_multicast or a.is_loopback or a.is_link_local)
def export_entries(engine, db) -> list[dict]:
"""把本地自学习库整理成可上报的条目:host_domain_day 里以 IP 记录的行,按字节数取最多的应用。
IPv4 按精确地址共享;IPv6 按 /64 前缀共享(精确地址在不同实例间几乎不重复,凑不满投票)"""
classifier = engine.classifier
from app.classify.local import _v6key
votes: dict = defaultdict(dict)
rows = db.execute(select(HostDomainDay.domain, HostDomainDay.app, HostDomainDay.bytes)).all()
for dom, app, b in rows:
if not _shareable_ip(dom) or not _is_shareable_app(classifier, app):
continue
key = _v6key(dom) or dom
per = votes[key]
per[app] = per.get(app, 0) + int(b or 0)
out = [{"ip": ip, "app": max(per, key=per.get), "bytes": sum(per.values())}
for ip, per in votes.items()]
out.sort(key=lambda x: -x["bytes"])
return out[:MAX_UPLOAD]
def collect(db, reporter: str, entries: list[dict]) -> int:
"""中转:记录一次上报(调用方提交事务)。返回接受的条数"""
now = int(time.time())
accepted = 0
from app.classify.local import _v6key
for e in entries[:MAX_UPLOAD]:
ip, app, b = str(e.get("ip", "")), str(e.get("app", "")), max(int(e.get("bytes", 0) or 0), 0)
# IPv6 条目(不管上报的是精确地址还是 /64)统一按 /64 前缀存储:不同实例的精确地址才能凑齐投票
base = ip[:-3] if ip.endswith("/64") else ip
if not _shareable_ip(base) or len(app) > 48 or app == "other":
continue
key = _v6key(base) or base
row = db.get(ShareEntry, {"ip": key, "app": app, "reporter": reporter})
if row is None:
db.add(ShareEntry(ip=key, app=app, reporter=reporter, bytes=b, ts=now))
else:
row.bytes = max(row.bytes, b)
row.ts = now
accepted += 1
# 清理:30 天没再出现、或单个实例超出上限的旧条目
db.execute(delete(ShareEntry).where(ShareEntry.reporter == reporter, ShareEntry.ts < now - ENTRY_TTL))
n = db.scalar(select(func.count()).select_from(ShareEntry).where(ShareEntry.reporter == reporter))
if n and n > MAX_REPORTER:
old = db.execute(select(ShareEntry.ip, ShareEntry.app).where(ShareEntry.reporter == reporter)
.order_by(ShareEntry.ts).limit(n - MAX_REPORTER)).all()
for ip, app in old:
db.execute(delete(ShareEntry).where(ShareEntry.ip == ip, ShareEntry.app == app,
ShareEntry.reporter == reporter))
return accepted
def published_library(db) -> dict:
"""中转对外发布的社区库:至少 PUBLISH_REPORTERS 个不同实例见过的映射"""
rows = db.execute(select(ShareEntry.ip, ShareEntry.app, func.count(func.distinct(ShareEntry.reporter)),
func.sum(ShareEntry.bytes)).group_by(ShareEntry.ip, ShareEntry.app)).all()
per_ip: dict = defaultdict(list)
for ip, app, reporters, _b in rows:
if reporters >= PUBLISH_REPORTERS:
per_ip[ip].append((int(_b or 0), app))
entries = [{"ip": ip, "app": max(v)[1]} for ip, v in per_ip.items()]
entries.sort(key=lambda x: x["ip"])
return {"version": 1, "generated": int(time.time()), "entries": entries}
def heartbeat(db, instance: str, version: str) -> None:
"""中转:记录一次部署心跳(调用方提交事务)。只含实例匿名 ID 和版本号"""
now = int(time.time())
row = db.get(ShareDeployment, instance)
if row is None:
db.add(ShareDeployment(instance=instance, version=version[:16], first_seen=now, last_seen=now))
else:
row.version, row.last_seen = version[:16], now
def deployments(db, online_after: int = 7200) -> dict:
"""中转:全球部署概况(所有人可见):已部署总数 / 在线(最近 2 小时有心跳)/ 离线"""
now = int(time.time())
total = db.scalar(select(func.count()).select_from(ShareDeployment)) or 0
online = db.scalar(select(func.count()).select_from(ShareDeployment)
.where(ShareDeployment.last_seen >= now - online_after)) or 0
return {"total": total, "online": online, "offline": total - online}
def sharing_on(db) -> bool:
"""共享识别库的总开关(debug16 第 3 条):默认打开;设置里可关闭(关闭后不心跳、不上报、不拉取)"""
from app.models import Setting
row = db.get(Setting, "share_enabled")
return (row.value == "1") if row is not None else True
def client_cycle(engine, db, url: str, token: str = "") -> dict:
"""客户端一轮同步:心跳(部署量统计)+ 上报本地实测 + 拉取社区库"""
import httpx
iid = instance_id(db)
headers = {"X-Share-Token": token} if token else {}
base = url.rstrip("/")
from app import __version__
try:
httpx.post(base + "/api/share/heartbeat", json={"instance": iid, "version": __version__},
headers=headers, timeout=15.0)
except Exception: # noqa: BLE001 心跳失败不影响库同步
pass
entries = export_entries(engine, db)
uploaded = 0
try:
r = httpx.post(base + "/api/share/collect", json={"instance": iid, "entries": entries},
headers=headers, timeout=30.0)
r.raise_for_status()
uploaded = int(r.json().get("accepted", 0))
except Exception: # noqa: BLE001
pass
merged = 0
try:
r2 = httpx.get(base + "/api/share/library.json", headers=headers, timeout=30.0)
r2.raise_for_status()
merged = merge_community(db, r2.json())
except Exception: # noqa: BLE001
pass
return {"uploaded": uploaded, "merged": merged}
def merge_community(db, data: dict) -> int:
"""客户端:把社区库并入本地自学习库的社区层(本地实测优先,社区只补空白)。返回并入条数"""
from app.classify.local import local
entries = (data or {}).get("entries") or []
known = set(local.ips)
community = {}
for e in entries:
ip, app = str(e.get("ip", "")), str(e.get("app", ""))
if ip in known or not _shareable_ip(ip[:-3] if ip.endswith("/64") else ip) or len(app) > 48:
continue
community[ip] = app
local.load_community(community)
return len(community)
+149
View File
@@ -0,0 +1,149 @@
"""debug16:社区共享库——本地库导出/上报过滤、中转聚合(≥2 实例才发布)、部署心跳、总开关。"""
import time
from app import share
from app.classify.local import local
from app.engine import Engine
from app.models import HostDomainDay, ShareDeployment, ShareEntry
DAY = 1792900000
HOST = "IP:192.168.8.40"
HUB = "11111111-1111-1111-1111-111111111111"
CLIENT_A = "22222222-2222-2222-2222-222222222222"
CLIENT_B = "33333333-3333-3333-3333-333333333333"
def _engine():
e = Engine()
e.set_lan("192.168.8.0/24")
return e
def test_export_filters_private_and_custom():
import app.database as database
database.connect_from_saved(force=True)
s = database.SessionLocal()
try:
s.query(HostDomainDay).filter(HostDomainDay.host == HOST).delete(synchronize_session=False)
s.commit()
s.add(HostDomainDay(ts=DAY, host=HOST, domain="93.184.216.34", app="douyin", base=None, bytes=800))
s.add(HostDomainDay(ts=DAY, host=HOST, domain="192.168.8.5", app="douyin", base=None, bytes=999)) # 私网:不共享
s.add(HostDomainDay(ts=DAY, host=HOST, domain="239.255.255.250", app="douyin", base=None, bytes=999)) # 组播:不共享
s.add(HostDomainDay(ts=DAY, host=HOST, domain="45.57.10.1", app="my_custom_app", base=None, bytes=999)) # 自定义应用:不共享
s.commit()
entries = share.export_entries(_engine(), s)
mine = {e["ip"]: e["app"] for e in entries if e["ip"] == "93.184.216.34"}
assert mine == {"93.184.216.34": "douyin"}
assert "192.168.8.5" not in {e["ip"] for e in entries} # 私网
assert "239.255.255.250" not in {e["ip"] for e in entries} # 组播
assert "45.57.10.1" not in {e["ip"] for e in entries} # 自定义应用(45.57.10.1 那行是自定义应用)
assert all(not e["ip"].endswith("/64") or ":" in e["ip"] for e in entries) # IPv4 永远精确
finally:
s.query(HostDomainDay).filter(HostDomainDay.host == HOST).delete(synchronize_session=False)
s.commit()
s.close()
def test_hub_collect_publishes_only_two_reporters():
import app.database as database
database.connect_from_saved(force=True)
s = database.SessionLocal()
try:
s.query(HostDomainDay).filter(HostDomainDay.host == HOST).delete(synchronize_session=False)
s.commit()
s.query(ShareEntry).filter(ShareEntry.ip.in_(["45.57.10.1", "8.8.8.8"])).delete(synchronize_session=False)
s.commit()
# 单个实例上报:不发布
share.collect(s, CLIENT_A, [{"ip": "45.57.10.1", "app": "douyin", "bytes": 500},
{"ip": "8.8.8.8", "app": "bilibili", "bytes": 300}])
s.commit()
lib = share.published_library(s)
assert lib["entries"] == []
# 第二个实例也见过 198.51.100.1 → 发布;198.51.100.2 仍只有一个实例 → 不发布
share.collect(s, CLIENT_B, [{"ip": "45.57.10.1", "app": "douyin", "bytes": 700}])
s.commit()
lib = share.published_library(s)
assert {"ip": "45.57.10.1", "app": "douyin"} in lib["entries"]
assert all(e["ip"] != "8.8.8.8" for e in lib["entries"])
finally:
s.query(ShareEntry).filter(ShareEntry.ip.in_(["45.57.10.1", "8.8.8.8"])).delete(synchronize_session=False)
s.commit()
s.close()
def test_heartbeat_and_deployments():
import app.database as database
database.connect_from_saved(force=True)
s = database.SessionLocal()
try:
s.query(ShareDeployment).filter(ShareDeployment.instance.in_([HUB, CLIENT_A])).delete(synchronize_session=False)
s.commit()
share.heartbeat(s, HUB, "0.6.10")
share.heartbeat(s, CLIENT_A, "0.6.10")
s.commit()
d = share.deployments(s)
assert d["total"] >= 2 and d["online"] >= 2
finally:
s.query(ShareDeployment).filter(ShareDeployment.instance.in_([HUB, CLIENT_A])).delete(synchronize_session=False)
s.commit()
s.close()
def test_client_merges_community_only_into_gaps():
import app.database as database
database.connect_from_saved(force=True)
s = database.SessionLocal()
try:
local.reset()
# 其它测试可能留过同域名的行:先清掉(8.8.4.4 在共享测试里是社区条目)
s.query(HostDomainDay).filter(HostDomainDay.domain.in_(["45.57.10.1", "8.8.4.4"])).delete(synchronize_session=False)
s.commit()
# 本地实测:45.57.10.1 → bilibili;社区对同一 IP 说什么都不覆盖
local.load_community({"45.57.10.1": "bilibili", "8.8.4.4": "douyin"})
s.add(HostDomainDay(ts=DAY, host=HOST, domain="45.57.10.1", app="bilibili", base=None, bytes=100))
s.commit()
local.load(s)
s.commit()
assert local.ip_app("45.57.10.1") == "bilibili" # 本地实测优先
assert local.ip_app("8.8.4.4") == "douyin" # 本地没有的用社区
finally:
s.query(HostDomainDay).filter(HostDomainDay.host == HOST).delete(synchronize_session=False)
s.query(HostDomainDay).filter(HostDomainDay.domain.in_(["45.57.10.1", "8.8.4.4"])).delete(synchronize_session=False)
s.commit()
s.close()
local.reset()
def test_ipv6_prefix_shared_and_matched():
"""中国环境以 IPv6 公网为主:精确地址在不同实例间几乎不重复,共享与匹配都按 /64 前缀"""
import app.database as database
database.connect_from_saved(force=True)
s = database.SessionLocal()
try:
s.query(HostDomainDay).filter(HostDomainDay.host == HOST).delete(synchronize_session=False)
s.query(ShareEntry).filter(ShareEntry.ip.like("%/64")).delete(synchronize_session=False)
s.commit()
# 两个不同实例、同一 /64 里的不同精确地址:聚合后应发布为同一条 /64 记忆
s.add(HostDomainDay(ts=DAY, host=HOST, domain="2408:8556:1234:5678::1", app="douyin", base=None, bytes=600))
s.add(HostDomainDay(ts=DAY, host=HOST, domain="fd00::5", app="douyin", base=None, bytes=999)) # ULA:不共享
s.commit()
entries = share.export_entries(_engine(), s)
v6 = {e["ip"]: e["app"] for e in entries if e["ip"].endswith("/64")}
assert v6 == {"2408:8556:1234:5678::/64": "douyin"}
assert "fd00::/64" not in v6
share.collect(s, CLIENT_A, [{"ip": "2408:8556:1234:5678::5", "app": "douyin", "bytes": 100}]) # 别的实例、同一 /64
share.collect(s, CLIENT_B, [{"ip": "2408:8556:1234:5678::9", "app": "douyin", "bytes": 100}])
s.commit()
lib = share.published_library(s)
assert {"ip": "2408:8556:1234:5678::/64", "app": "douyin"} in lib["entries"]
# 本地匹配:同一 /64 里的另一个精确地址也能命中
local.reset()
local.load_community({"2408:8556:1234:5678::/64": "douyin"})
assert local.ip_app("2408:8556:1234:5678::abc") == "douyin"
assert local.ip_app("2408:8556:9999::abc") is None
finally:
s.query(HostDomainDay).filter(HostDomainDay.host == HOST).delete(synchronize_session=False)
s.query(ShareEntry).filter(ShareEntry.ip.like("%/64")).delete(synchronize_session=False)
s.commit()
s.close()
local.reset()
+5
View File
@@ -150,6 +150,11 @@ export default {
agUrl: 'AdGuard Home address', agUser: 'Login username (leave empty if none)', agPassword: 'Login password',
agKept: 'Set; leave empty to keep', agNone: 'Leave empty if none', agClear: 'Clear password',
agHint: 'AdGuard Home query log must be on (enabled by default); empty username and password mean unauthenticated access.',
shareTitle: 'Community sharing', shareHubOn: 'Relay mode',
shareEnabled: 'Share the learned library (on by default)',
shareWhy: 'Every deployment learns different IP→app mappings. Pooling anonymous votes makes the shared identification sharper for everyone. Reports contain remote-IP→app votes only — no local devices, no domain records, no custom apps; private ranges are stripped.',
shareUrl: 'Community relay address', shareHub: 'This instance acts as the relay (must be publicly reachable)',
shareHint: 'Fill the relay address to participate (synced hourly); enable relay mode to accept reports at /api/share/collect. Local observations win; community entries only fill gaps.',
asn: 'IP ownership (ASN)', asnOn: 'On',
asnWhy: 'Domain-less traffic is attributed to the owning company by IP (Tencent, Alibaba, Apple, Netflix…). Data from iptoasn.com (public domain), refreshed daily; off without internet access.',
asnRows: '{n} ranges, updated {d}.', connected: 'Connected', failed: 'Failed', off: 'Off',
+5
View File
@@ -150,6 +150,11 @@ export default {
agUrl: 'AdGuard Home 地址', agUser: '登录用户名(没有可留空)', agPassword: '登录密码',
agKept: '已设置,不修改请留空', agNone: '没有设置可留空', agClear: '清除密码',
agHint: '需要 AdGuard Home 开启查询日志(默认开启);账号和密码都留空表示免认证访问。',
shareTitle: '社区共享库', shareHubOn: '中转模式',
shareEnabled: '共享识别库(默认打开)',
shareWhy: '每个部署实例的自学习库都不一样:把各自实测的「远端 IP → 应用」匿名投票汇总到一起,大家共同的识别就更准。上报内容不含本机终端、域名记录与自定义应用,私网地址自动剔除。',
shareUrl: '社区中转地址', shareHub: '本实例作为共享中转(需公网可达)',
shareHint: '填中转地址即参与共享(每小时同步一次);勾选中转模式后,其它实例可以把上报发到本实例的 /api/share/collect。本地实测优先,社区结论只补空白。',
asn: 'IP 归属库(ASN)', asnOn: '已启用',
asnWhy: '没有域名的流量按 IP 归属认出公司(腾讯、阿里、苹果、Netflix……)。数据来自 iptoasn.com(公有领域),容器每天自动更新;没有外网时不启用。',
asnRows: '共 {n} 条,更新于 {d}。', connected: '已连接', failed: '连接失败', off: '未启用',
+20 -3
View File
@@ -161,6 +161,21 @@
<span class="err" v-if="sys.adguard.error">{{ sys.adguard.error }}</span>
</div>
</div>
<div class="src-box">
<div class="sec">{{ t('sources.shareTitle') }}<span class="sp" />
<span v-if="s.share_hub" class="pill ok">{{ t('sources.shareHubOn') }}</span>
<span v-else class="pill grey">{{ t('sources.off') }}</span></div>
<p class="faint small">{{ t('sources.shareWhy') }}</p>
<label class="chk" style="display:flex;gap:6px;align-items:center;cursor:pointer">
<input type="checkbox" :checked="s.share_enabled" @change="src.share_enabled = $event.target.checked" /> {{ t('sources.shareEnabled') }}
</label>
<label for="share-url">{{ t('sources.shareUrl') }}</label>
<input id="share-url" class="input addr" v-model.trim="src.share_url" placeholder="https://share.example.com" autocomplete="off" />
<label class="chk" style="display:flex;gap:6px;align-items:center;cursor:pointer;margin-top:8px">
<input id="share-hub" type="checkbox" :checked="s.share_hub" @change="src.share_hub = $event.target.checked" /> {{ t('sources.shareHub') }}
</label>
<div class="faint small">{{ t('sources.shareHint') }}</div>
</div>
<div class="acts">
<span class="err" v-if="srcErr">{{ srcErr }}</span><span class="sp" />
<button type="button" class="btn" @click="testSources" :disabled="testing">{{ testing ? t('setup.testing') : t('settings.test') }}</button>
@@ -282,21 +297,23 @@ onMounted(async () => {
})
// ---------- 外部数据源 ----------
const src = reactive({ singbox_url: '', singbox_secret: '', proxy_ips: '', mosdns_url: '', adguard_url: '', adguard_username: '', adguard_password: '' })
const src = reactive({ singbox_url: '', singbox_secret: '', proxy_ips: '', mosdns_url: '', adguard_url: '', adguard_username: '', adguard_password: '', share_url: '', share_enabled: true, share_hub: false })
const srcErr = ref('')
const srcSaved = ref(false)
const testing = ref(false)
function fillSources() {
const i = s.value.integrations
Object.assign(src, { singbox_url: i.singbox_url, singbox_secret: '', proxy_ips: i.proxy_ips, mosdns_url: i.mosdns_url,
adguard_url: i.adguard_url, adguard_username: i.adguard_username, adguard_password: '' })
adguard_url: i.adguard_url, adguard_username: i.adguard_username, adguard_password: '',
share_url: i.share_url, share_enabled: !!s.value.share_enabled, share_hub: !!s.value.share_hub })
}
async function saveSources() {
srcErr.value = ''
try {
// 密钥留空表示不修改;要清除用“清除密钥/密码”按钮
const body = { singbox_url: src.singbox_url, proxy_ips: src.proxy_ips, mosdns_url: src.mosdns_url,
adguard_url: src.adguard_url, adguard_username: src.adguard_username }
adguard_url: src.adguard_url, adguard_username: src.adguard_username,
share_url: src.share_url, share_enabled: !!src.share_enabled, share_hub: !!src.share_hub }
if (src.singbox_secret) body.singbox_secret = src.singbox_secret
if (src.adguard_password) body.adguard_password = src.adguard_password
s.value = await http.put('/settings', body)