feat(turnstile): Cookie 池预热、可配求解超时、ezsolver 求解风格与本地直连
- 求解超时统一走 SolveTimeoutValue(默认 90s,CLamp 5–600),TURNSTILE_TIMEOUT_SECONDS
- Cookie 池 warm_enabled:按站点 × 线路后台预热/续期,失败指数退避(上限 15 分钟);
warm_pool_only 仅预热池绑定线路;CookiesReady 与 slow-start 协同
- api_style = "ezsolver":兼容 {sitekey,siteurl,timeout} → {token} 的本地求解服务,
api_key 可留空;环回/内网求解地址直连(不随线路代理)
- admin: 求解器风格选择与 Cookie 池预热状态
This commit is contained in:
1 parent
6aae7b65ea
commit
c9886a81d2
15 files changed
+884
-26
No files matched your search
@@ -142,7 +142,7 @@ dsfree2api -config config.toml -check
|
||||
| `[proxy]` | `url` 主线路、`fallback_urls` 备用线路、`slow_start_seconds` 首事件超时 |
|
||||
| `[proxypool]` | 代理池总开关、健康检查间隔 / 超时 / URL、裸 `ip:port` 默认协议、Xray 路径 / 版本 / 自动下载;`[proxypool.entries.*]` 手动节点(分享链接或端点)、`[proxypool.subscriptions.*]` 订阅自动拉取 |
|
||||
| `[upstream]` | 超时、配置缓存 TTL、会话自动刷新、`cross_site_failover` 跨站切换、`continue_rounds` 工具调用截断续写轮数(0 = 不续写、直接截断补全) |
|
||||
| `[turnstile]` | **默认 `enabled = false` 且不内置任何求解服务**;`provider` 三选一:`api`(填 `api_url` / `api_key`)、`browser`(填 `browser_path` 指向本机 Chrome/Edge)、`manual`(只用控制台导入的 Cookie),另有 Cookie TTL、重试次数 |
|
||||
| `[turnstile]` | **默认 `enabled = false` 且不内置任何求解服务**;`provider` 三选一:`api`(填 `api_url` / `api_key`)、`browser`(填 `browser_path` 指向本机 Chrome/Edge)、`manual`(只用控制台导入的 Cookie),另有 Cookie TTL、重试次数、单次求解超时(`timeout_seconds`)与 Cookie 池(`warm_enabled` / `warm_ratio` / `warm_check_seconds`,后台预热,请求零等待) |
|
||||
| `[sites.*]` | 三个站点的 `base_url` / `ajax_url` / `sitekey` / `language` / `proxies`(站点级出口绑定,按顺序粘性选路) |
|
||||
| `[models.*]` | 模型到 `site` + `upstream_id` + `bot_id` / `post_id` 的映射 |
|
||||
|
||||
|
||||
@@ -86,6 +86,12 @@ func main() {
|
||||
|
||||
up := upstream.New(cfg, ts, logger, pool)
|
||||
|
||||
// Cookie pool: keep one verified session per site × route ahead of time,
|
||||
// so API requests never wait for a solve. No-op unless warm_enabled.
|
||||
warmCtx, warmCancel := context.WithCancel(context.Background())
|
||||
defer warmCancel()
|
||||
ts.StartWarmer(warmCtx, func() []turnstile.WarmTarget { return warmTargets(cfg, up, pool) })
|
||||
|
||||
apiSrv := api.New(cfg, up, met, logs, ts, logger)
|
||||
adminSrv := admin.New(cfg, up, met, logs, ts, logger, pool)
|
||||
|
||||
@@ -143,6 +149,48 @@ func main() {
|
||||
logger.Info("bye")
|
||||
}
|
||||
|
||||
// warmTargets builds the site × route matrix the cookie pool keeps fresh:
|
||||
// the sticky proxy-pool pick first, then the global primary/fallback lines.
|
||||
// A site with warm_pool_only skips the global lanes and warms only its bound
|
||||
// pool routes (request-time failover to the global lanes is unaffected).
|
||||
func warmTargets(cfg *config.Config, up *upstream.Client, pool *proxypool.Manager) []turnstile.WarmTarget {
|
||||
routes := up.Routes()
|
||||
cfg.RLock()
|
||||
sites := make([]config.Site, 0, len(cfg.Sites))
|
||||
for _, st := range cfg.Sites {
|
||||
if st != nil && st.Enabled {
|
||||
sites = append(sites, *st)
|
||||
}
|
||||
}
|
||||
cfg.RUnlock()
|
||||
out := make([]turnstile.WarmTarget, 0, len(routes)*len(sites))
|
||||
seen := map[string]bool{}
|
||||
add := func(site config.Site, proxy string) {
|
||||
key := site.Code + "|" + proxy
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, turnstile.WarmTarget{Site: site, Proxy: proxy})
|
||||
}
|
||||
for _, site := range sites {
|
||||
if pool != nil {
|
||||
if cands := pool.Candidates(site.Code); len(cands) > 0 {
|
||||
add(site, cands[0].Proxy)
|
||||
}
|
||||
}
|
||||
// warm_pool_only sites keep their cookie pool on the bound pool
|
||||
// routes; the global lanes stay unwarmed.
|
||||
if site.WarmPoolOnly {
|
||||
continue
|
||||
}
|
||||
for _, r := range routes {
|
||||
add(site, r.Proxy)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func countModels(cfg *config.Config) int {
|
||||
cfg.RLock()
|
||||
defer cfg.RUnlock()
|
||||
|
||||
+15
-2
@@ -2,7 +2,9 @@
|
||||
# 复制为 config.toml 后按需修改;运行 `dsfree2api -check -config config.toml` 可校验。
|
||||
# 环境变量优先级高于本文件:HOST PORT LOG_LEVEL API_KEYS PROXY_URL PROXY_FALLBACK_URLS
|
||||
# PROXY_SLOW_START_SECONDS TURNSTILE_ENABLED TURNSTILE_API_KEY TURNSTILE_PROVIDER
|
||||
# TURNSTILE_BROWSER_PATH ADMIN_ENABLED ADMIN_HOST ADMIN_PORT ADMIN_PASSWORD DATA_DIR
|
||||
# TURNSTILE_BROWSER_PATH TURNSTILE_TIMEOUT_SECONDS TURNSTILE_WARM_ENABLED
|
||||
# TURNSTILE_WARM_RATIO TURNSTILE_WARM_CHECK_SECONDS
|
||||
# ADMIN_ENABLED ADMIN_HOST ADMIN_PORT ADMIN_PASSWORD DATA_DIR
|
||||
|
||||
[server]
|
||||
host = "0.0.0.0"
|
||||
@@ -87,7 +89,8 @@ continue_rounds = 20
|
||||
# manual —— 只用控制台手动导入的 Cookie(方式 3)
|
||||
enabled = false
|
||||
provider = "api"
|
||||
# 方式 1:求解服务 API
|
||||
# 方式 1:求解服务 API(api_style = "sync" 为默认 CapSolver 形状;
|
||||
# api_style = "ezsolver" 适配本地 EzSolver 服务,api_key 可留空)
|
||||
# api_url = "https://your-solver.example.com/turnstile/sync"
|
||||
# api_key = "REPLACE_ME"
|
||||
# 方式 2:本地浏览器自动获取(CDP 直连,无需 Playwright / Docker)
|
||||
@@ -98,10 +101,18 @@ provider = "api"
|
||||
# browser_locale = "" # 可选:如 "ja-JP",留空取站点语言
|
||||
sitekey = "0x4AAAAAADlLZ3ljqZP6cQwq"
|
||||
action = "chat"
|
||||
# 求解超时:单次求解(浏览器 / 求解服务 API)的最长等待秒数,5–600。
|
||||
timeout_seconds = 90
|
||||
cookie_ttl_seconds = 10800
|
||||
retries = 5
|
||||
retry_backoff_seconds = 1.5
|
||||
# ── Cookie 池(后台预热)─────────────────────────────────
|
||||
# 开启后,后台按 warm_check_seconds 巡检:Cookie 缺失 / 过期 /
|
||||
# 剩余 TTL 低于 warm_ratio 时主动重解并写回缓存,请求路径不再等待求解。
|
||||
# provider = "manual" 时不预热(手动导入的 Cookie 无法自动续期)。
|
||||
warm_enabled = false
|
||||
warm_ratio = 0.3 # 剩余 TTL 低于该比例(0.05–0.95)时触发预热
|
||||
warm_check_seconds = 60 # 后台巡检间隔秒数(5–3600)
|
||||
|
||||
[runtime]
|
||||
data_dir = "./data"
|
||||
@@ -114,6 +125,8 @@ language = "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"
|
||||
# 站点级出口绑定(代理池条目名或 "sub:订阅名",按顺序粘性选路);
|
||||
# 留空 = 使用 [proxy] 主/备线路。三个站点可各自独立配置。
|
||||
# proxies = ["node-a", "sub:home"]
|
||||
# 只预热代理池线路(不预热 [proxy] 主/备;请求时仍可回落):
|
||||
# warm_pool_only = true
|
||||
|
||||
[sites.es]
|
||||
base_url = "https://deepseek.es"
|
||||
|
||||
@@ -235,12 +235,20 @@ func (s *Server) handleActions(w http.ResponseWriter, r *http.Request) {
|
||||
Challenge string `json:"challenge_action"`
|
||||
Legacy string `json:"ts_action"`
|
||||
CookieTT int `json:"cookie_ttl_seconds"`
|
||||
APIStyle *string `json:"api_style"`
|
||||
|
||||
BrowserPath *string `json:"browser_path"`
|
||||
BrowserHeadless *bool `json:"browser_headless"`
|
||||
BrowserUserDataDir *string `json:"browser_user_data_dir"`
|
||||
BrowserTimezone *string `json:"browser_timezone"`
|
||||
BrowserLocale *string `json:"browser_locale"`
|
||||
|
||||
TimeoutSeconds *int `json:"timeout_seconds"`
|
||||
Retries *int `json:"retries"`
|
||||
RetryBackoffSeconds *float64 `json:"retry_backoff_seconds"`
|
||||
WarmEnabled *bool `json:"warm_enabled"`
|
||||
WarmRatio *float64 `json:"warm_ratio"`
|
||||
WarmCheckSeconds *int `json:"warm_check_seconds"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &p)
|
||||
// Build the candidate config first: a rejected save must not leave
|
||||
@@ -277,9 +285,30 @@ func (s *Server) handleActions(w http.ResponseWriter, r *http.Request) {
|
||||
if act := firstNonEmpty(p.Challenge, p.Legacy); act != "" {
|
||||
tc.Action = act
|
||||
}
|
||||
if p.APIStyle != nil {
|
||||
tc.APIStyle = strings.ToLower(strings.TrimSpace(*p.APIStyle))
|
||||
}
|
||||
if p.CookieTT > 0 {
|
||||
tc.CookieTTLSeconds = p.CookieTT
|
||||
}
|
||||
if p.TimeoutSeconds != nil {
|
||||
tc.TimeoutSeconds = clampInt(*p.TimeoutSeconds, 5, 600)
|
||||
}
|
||||
if p.Retries != nil {
|
||||
tc.Retries = clampInt(*p.Retries, 1, 20)
|
||||
}
|
||||
if p.RetryBackoffSeconds != nil {
|
||||
tc.RetryBackoffSeconds = clampFloat(*p.RetryBackoffSeconds, 0, 60)
|
||||
}
|
||||
if p.WarmEnabled != nil {
|
||||
tc.WarmEnabled = *p.WarmEnabled
|
||||
}
|
||||
if p.WarmRatio != nil {
|
||||
tc.WarmRatio = clampFloat(*p.WarmRatio, 0.05, 0.95)
|
||||
}
|
||||
if p.WarmCheckSeconds != nil {
|
||||
tc.WarmCheckSeconds = clampInt(*p.WarmCheckSeconds, 5, 3600)
|
||||
}
|
||||
tc.Enabled = p.Enabled
|
||||
if tc.Enabled {
|
||||
// provider-specific requirements mirror config.Validate()
|
||||
|
||||
@@ -212,7 +212,7 @@ func (s *Server) handleOverview(w http.ResponseWriter, _ *http.Request) {
|
||||
"models": models,
|
||||
"sites": sites,
|
||||
"proxy": proxyCfg,
|
||||
"turnstile": map[string]any{"config": turnstileCfg, "status": s.ts.Status(siteCodes), "history": s.ts.History()},
|
||||
"turnstile": map[string]any{"config": turnstileCfg, "status": s.ts.Status(siteCodes), "history": s.ts.History(), "pool": s.ts.PoolStatus()},
|
||||
"proxypool": poolStatus,
|
||||
"keys": keys,
|
||||
"cache": map[string]any{"chat_config_entries": s.up.ConfigCacheSize(), "routes": routeList(s.up)},
|
||||
|
||||
@@ -563,6 +563,10 @@ PAGES.turnstile = async () => {
|
||||
<div class="row">
|
||||
<div class="field"><label>Sitekey</label><input id="ts-sitekey" value="${esc(t.sitekey)}"></div>
|
||||
<div class="field"><label>Action</label><input id="ts-action" value="${esc(t.action)}"></div>
|
||||
<div class="field"><label>API 风格</label><select id="ts-style">
|
||||
<option value="sync">sync(CapSolver 形状)</option>
|
||||
<option value="ezsolver">ezsolver(本地 EzSolver)</option>
|
||||
</select></div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="ts-browser" class="ts-panel" style="display:none">
|
||||
@@ -596,6 +600,20 @@ PAGES.turnstile = async () => {
|
||||
<div class="dim">Cookie 与出口 IP / UA 绑定:请在同一代理线路的浏览器中获取。导入后在 TTL 内免求解,状态见右侧「已验证 Cookie」。</div>
|
||||
<button class="btn primary mt" id="im-save">导入</button>
|
||||
</div>
|
||||
<div class="ts-panel">
|
||||
<div class="ts-panel-title">求解与 Cookie 池</div>
|
||||
<div class="row">
|
||||
<div class="field"><label>求解超时(秒)</label><input id="ts-timeout" type="number" min="5" max="600" value="${t.timeout_seconds || 90}"></div>
|
||||
<div class="field"><label>失败重试(次)</label><input id="ts-retries" type="number" min="1" max="20" value="${t.retries || 5}"></div>
|
||||
<div class="field"><label>重试退避(秒)</label><input id="ts-backoff" type="number" min="0" max="60" step="0.5" value="${t.retry_backoff_seconds == null ? 1.5 : t.retry_backoff_seconds}"></div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="field"><label>Cookie 池(后台预热)</label>${switchHTML("ts-warm", !!t.warm_enabled)}</div>
|
||||
<div class="field"><label>预热阈值(剩余 %)</label><input id="ts-ratio" type="number" min="5" max="95" value="${Math.round((t.warm_ratio == null ? 0.3 : t.warm_ratio) * 100)}"></div>
|
||||
<div class="field"><label>检查间隔(秒)</label><input id="ts-check" type="number" min="5" max="3600" value="${t.warm_check_seconds || 60}"></div>
|
||||
</div>
|
||||
<div class="dim">Cookie 池开启后:后台每隔检查间隔巡检一次,剩余 TTL 低于阈值时主动重解,请求路径不再等待求解耗时(manual 方式不预热)。</div>
|
||||
</div>
|
||||
<div class="dim" id="ts-hint" style="margin-top:10px"></div>
|
||||
<button class="btn primary mt" id="ts-save">保存配置</button>
|
||||
</div>
|
||||
@@ -605,7 +623,11 @@ PAGES.turnstile = async () => {
|
||||
<button class="btn" id="ts-refresh">强制刷新全部</button>
|
||||
<button class="btn danger" id="ts-clear">清空 Cookie</button>
|
||||
</div>
|
||||
<div class="dim mt">缓存 TTL ${t.cookie_ttl_seconds}s · 求解失败重试 ${t.retries} 次 · 退避 ${t.retry_backoff_seconds}s</div>
|
||||
<div class="dim mt">缓存 TTL ${t.cookie_ttl_seconds}s · 求解超时 ${t.timeout_seconds || 90}s · 重试 ${t.retries} 次 · 退避 ${t.retry_backoff_seconds}s</div>
|
||||
</div>
|
||||
<div class="card"><h3>Cookie 池</h3>
|
||||
<div class="dim" id="ts-pool-state"></div>
|
||||
<div id="ts-pool" class="grid mt" style="gap:8px"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card mt"><h3>求解历史</h3><div class="table-wrap"><table>
|
||||
@@ -626,13 +648,39 @@ PAGES.turnstile = async () => {
|
||||
<td><span class="pill ${h.ok ? "on" : "off"}">${h.ok ? "成功" : "失败"}</span>${h.error ? ` <span class="dim">${esc(h.error.slice(0, 70))}</span>` : ""}</td></tr>`).join("")
|
||||
: '<tr><td colspan="5" class="empty">尚无记录</td></tr>';
|
||||
|
||||
const renderPool = (o) => {
|
||||
const pool = (o.turnstile && o.turnstile.pool) || { entries: [] };
|
||||
const label = $("#ts-pool-state");
|
||||
if (label) {
|
||||
label.textContent = `状态:${pool.enabled ? (pool.running ? "运行中" : "已启用,等待下一轮巡检") : "未开启"} · 阈值 剩余 ${Math.round((pool.ratio || 0.3) * 100)}% · 每 ${pool.check_seconds || 60}s 巡检一次`;
|
||||
}
|
||||
const box = $("#ts-pool");
|
||||
if (!box) return;
|
||||
const entries = pool.entries || [];
|
||||
box.innerHTML = entries.length ? entries.map(e => {
|
||||
const state = e.warming ? '<span class="pill warn">预热中</span>'
|
||||
: e.valid ? '<span class="pill on">就绪</span>'
|
||||
: e.last_error ? '<span class="pill off">求解失败</span>' : '<span class="pill off">未建立</span>';
|
||||
const left = e.valid ? `剩余 ${Math.max(0, Math.round(e.remaining_s / 60))} 分钟` : "等待预热";
|
||||
const next = e.next_warm_at ? `下次 ${timeStr(e.next_warm_at * 1000)}` : "—";
|
||||
const last = e.last_at ? `上次 ${timeStr(e.last_at * 1000)} · ${fmtMs(e.last_ms)}` : "尚未求解";
|
||||
return `<div class="card" style="padding:10px">
|
||||
<div class="toolbar"><div><b>${esc(e.site.toUpperCase())}</b> <span class="dim">${esc(e.route)}</span></div>${state}</div>
|
||||
<div class="dim mt">${left} · ${next}<br>${last}${e.last_error ? `<br><span style="color:var(--err)">${esc(String(e.last_error).slice(0, 90))}</span>` : ""}</div>
|
||||
</div>`;
|
||||
}).join("") : '<div class="dim">暂无池内记录 — 开启预热并等待下一轮巡检</div>';
|
||||
};
|
||||
renderPool(ov);
|
||||
overviewTimer = setInterval(() => { fetchOverview().then(renderPool).catch(() => {}); }, 5000);
|
||||
|
||||
$$("[data-rf]").forEach(b => b.onclick = async () => {
|
||||
b.disabled = true;
|
||||
try { await api("/api/actions", { body: { action: "refresh_cookies", site: b.dataset.rf } }); toast("已作废,将在下次请求时重新求解", "ok"); PAGES.turnstile(); }
|
||||
try { await api("/api/actions", { body: { action: "refresh_cookies", site: b.dataset.rf } }); toast(t.warm_enabled ? "已作废,Cookie 池将在下一轮巡检时重新预热" : "已作废,将在下次请求时重新求解", "ok"); PAGES.turnstile(); }
|
||||
catch (e) { toast(e.message, "err"); }
|
||||
b.disabled = false;
|
||||
});
|
||||
$("#ts-provider").value = t.provider || "api";
|
||||
$("#ts-style").value = t.api_style === "ezsolver" ? "ezsolver" : "sync";
|
||||
const TS_HINTS = {
|
||||
api: ["已开启:调用求解服务 API 获取 Turnstile Token", "已关闭:跳过 Turnstile 求解,直接请求上游站点"],
|
||||
browser: ["已开启:用下方配置的本地浏览器自动求解 Token", "已关闭:跳过 Turnstile 求解,直接请求上游站点"],
|
||||
@@ -658,6 +706,13 @@ PAGES.turnstile = async () => {
|
||||
api_url: $("#ts-url").value.trim(),
|
||||
api_key: $("#ts-key").value.trim(), sitekey: $("#ts-sitekey").value.trim(),
|
||||
challenge_action: $("#ts-action").value.trim(), cookie_ttl_seconds: +$("#ts-ttl").value,
|
||||
api_style: $("#ts-style").value,
|
||||
timeout_seconds: Math.round(+$("#ts-timeout").value || 90),
|
||||
retries: Math.round(+$("#ts-retries").value || 5),
|
||||
retry_backoff_seconds: Math.max(0, +$("#ts-backoff").value || 0),
|
||||
warm_enabled: $("#ts-warm").checked,
|
||||
warm_ratio: Math.min(95, Math.max(5, Math.round(+$("#ts-ratio").value || 30))) / 100,
|
||||
warm_check_seconds: Math.round(+$("#ts-check").value || 60),
|
||||
browser_path: $("#ts-bpath").value.trim(),
|
||||
browser_headless: $("#ts-bhead").checked,
|
||||
browser_user_data_dir: $("#ts-bprof").value.trim(),
|
||||
|
||||
@@ -83,6 +83,7 @@ label{display:block;font-size:12px;color:var(--muted);margin-bottom:5px}
|
||||
.pill{display:inline-block;padding:2px 8px;border-radius:20px;font-size:11px;border:1px solid var(--line);color:var(--muted)}
|
||||
.pill.on{color:var(--ok);border-color:#1f6a34;background:#0f2318}
|
||||
.pill.off{color:var(--dim)}
|
||||
.pill.warn{color:var(--warn);border-color:#6b5417;background:#2a2210}
|
||||
|
||||
/* console / logs */
|
||||
.log-box{background:#0a0e14;border:1px solid var(--line);border-radius:10px;padding:10px;height:56vh;overflow:auto;font-family:var(--mono);font-size:12.5px;line-height:1.55}
|
||||
|
||||
+108
-2
@@ -14,6 +14,15 @@ import (
|
||||
|
||||
const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"
|
||||
|
||||
// Turnstile tuning defaults. DefaultSolveTimeoutSeconds is the single source
|
||||
// for how long one solve attempt may run; the warm_* values drive the cookie
|
||||
// pool (background pre-solving).
|
||||
const (
|
||||
DefaultSolveTimeoutSeconds = 90
|
||||
DefaultWarmRatio = 0.3
|
||||
DefaultWarmCheckSeconds = 60
|
||||
)
|
||||
|
||||
// Turnstile token providers: api calls a hosted solver, browser drives a
|
||||
// local Chrome/Edge over CDP, manual only ever uses imported cookies.
|
||||
const (
|
||||
@@ -29,11 +38,18 @@ type Turnstile struct {
|
||||
APIKey string `toml:"api_key" json:"api_key"`
|
||||
SiteKey string `toml:"sitekey" json:"sitekey"`
|
||||
Action string `toml:"action" json:"action"`
|
||||
APIStyle string `toml:"api_style" json:"api_style"`
|
||||
TimeoutSeconds int `toml:"timeout_seconds" json:"timeout_seconds"`
|
||||
CookieTTLSeconds int `toml:"cookie_ttl_seconds" json:"cookie_ttl_seconds"`
|
||||
Retries int `toml:"retries" json:"retries"`
|
||||
RetryBackoffSeconds float64 `toml:"retry_backoff_seconds" json:"retry_backoff_seconds"`
|
||||
|
||||
// Cookie pool: a background loop re-solves a site × route slot before its
|
||||
// cached cookie expires, so requests never pay the solve latency.
|
||||
WarmEnabled bool `toml:"warm_enabled" json:"warm_enabled"`
|
||||
WarmRatio float64 `toml:"warm_ratio" json:"warm_ratio"`
|
||||
WarmCheckSeconds int `toml:"warm_check_seconds" json:"warm_check_seconds"`
|
||||
|
||||
// provider = "browser"
|
||||
BrowserPath string `toml:"browser_path" json:"browser_path"`
|
||||
BrowserHeadless bool `toml:"browser_headless" json:"browser_headless"`
|
||||
@@ -51,6 +67,65 @@ func (t Turnstile) ProviderValue() string {
|
||||
return p
|
||||
}
|
||||
|
||||
// APIStyleValue normalizes api_style for provider "api": "sync" (CapSolver-
|
||||
// shaped single POST) or "ezsolver" (POST {sitekey,siteurl,timeout} → {token}).
|
||||
func (t Turnstile) APIStyleValue() string {
|
||||
switch strings.ToLower(strings.TrimSpace(t.APIStyle)) {
|
||||
case "ezsolver", "ez":
|
||||
return "ezsolver"
|
||||
default:
|
||||
return "sync"
|
||||
}
|
||||
}
|
||||
|
||||
// SolveTimeoutValue normalizes timeout_seconds: the longest one solve attempt
|
||||
// may run (default 90s, clamped to 5–600s).
|
||||
func (t Turnstile) SolveTimeoutValue() int {
|
||||
n := t.TimeoutSeconds
|
||||
if n <= 0 {
|
||||
return DefaultSolveTimeoutSeconds
|
||||
}
|
||||
if n < 5 {
|
||||
return 5
|
||||
}
|
||||
if n > 600 {
|
||||
return 600
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// WarmRatioValue normalizes warm_ratio: a pooled cookie is re-solved once its
|
||||
// remaining TTL drops below this fraction (default 30%, clamped 5–95%).
|
||||
func (t Turnstile) WarmRatioValue() float64 {
|
||||
r := t.WarmRatio
|
||||
if r <= 0 {
|
||||
return DefaultWarmRatio
|
||||
}
|
||||
if r < 0.05 {
|
||||
return 0.05
|
||||
}
|
||||
if r > 0.95 {
|
||||
return 0.95
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// WarmCheckValue normalizes warm_check_seconds: how often the pool is
|
||||
// inspected (default 60s, clamped 5s–1h).
|
||||
func (t Turnstile) WarmCheckValue() int {
|
||||
n := t.WarmCheckSeconds
|
||||
if n <= 0 {
|
||||
return DefaultWarmCheckSeconds
|
||||
}
|
||||
if n < 5 {
|
||||
return 5
|
||||
}
|
||||
if n > 3600 {
|
||||
return 3600
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
type Site struct {
|
||||
Code string `toml:"-"`
|
||||
BaseURL string `toml:"base_url" json:"base_url"`
|
||||
@@ -183,10 +258,12 @@ func DefaultTurnstile() Turnstile {
|
||||
APIKey: "",
|
||||
SiteKey: "0x4AAAAAADlLZ3ljqZP6cQwq",
|
||||
Action: "chat",
|
||||
TimeoutSeconds: 90,
|
||||
TimeoutSeconds: DefaultSolveTimeoutSeconds,
|
||||
CookieTTLSeconds: 10800,
|
||||
Retries: 5,
|
||||
RetryBackoffSeconds: 1.5,
|
||||
WarmRatio: DefaultWarmRatio,
|
||||
WarmCheckSeconds: DefaultWarmCheckSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -428,6 +505,34 @@ func (c *Config) applyEnv() error {
|
||||
if v := str("TURNSTILE_BROWSER_PATH"); v != nil && strings.TrimSpace(*v) != "" {
|
||||
c.Turnstile.BrowserPath = strings.TrimSpace(*v)
|
||||
}
|
||||
if v := str("TURNSTILE_TIMEOUT_SECONDS"); v != nil {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(*v))
|
||||
if err != nil || n < 1 {
|
||||
return fmt.Errorf("invalid TURNSTILE_TIMEOUT_SECONDS %q", *v)
|
||||
}
|
||||
c.Turnstile.TimeoutSeconds = n
|
||||
}
|
||||
if v := str("TURNSTILE_WARM_ENABLED"); v != nil {
|
||||
b, err := strconv.ParseBool(strings.TrimSpace(*v))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid TURNSTILE_WARM_ENABLED %q", *v)
|
||||
}
|
||||
c.Turnstile.WarmEnabled = b
|
||||
}
|
||||
if v := str("TURNSTILE_WARM_RATIO"); v != nil {
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(*v), 64)
|
||||
if err != nil || f <= 0 || f > 1 {
|
||||
return fmt.Errorf("invalid TURNSTILE_WARM_RATIO %q (expected 0–1)", *v)
|
||||
}
|
||||
c.Turnstile.WarmRatio = f
|
||||
}
|
||||
if v := str("TURNSTILE_WARM_CHECK_SECONDS"); v != nil {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(*v))
|
||||
if err != nil || n < 1 {
|
||||
return fmt.Errorf("invalid TURNSTILE_WARM_CHECK_SECONDS %q", *v)
|
||||
}
|
||||
c.Turnstile.WarmCheckSeconds = n
|
||||
}
|
||||
if v := str("TURNSTILE_ENABLED"); v != nil {
|
||||
b, err := strconv.ParseBool(strings.TrimSpace(*v))
|
||||
if err != nil {
|
||||
@@ -517,7 +622,8 @@ func (c *Config) validateTurnstile() error {
|
||||
if c.Turnstile.APIURL == "" {
|
||||
return errors.New("turnstile.api_url is empty (set config to your solver endpoint, or use provider = \"manual\")")
|
||||
}
|
||||
if c.Turnstile.APIKey == "" {
|
||||
// Local solve services (api_style = "ezsolver") usually need no key.
|
||||
if c.Turnstile.APIKey == "" && c.Turnstile.APIStyleValue() != "ezsolver" {
|
||||
return errors.New("turnstile.api_key is empty (set config, or TURNSTILE_API_KEY)")
|
||||
}
|
||||
case ProviderBrowser:
|
||||
|
||||
@@ -100,6 +100,11 @@ func TestValidateRejectsBadValues(t *testing.T) {
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("expected error for empty turnstile api_key")
|
||||
}
|
||||
cfg.Turnstile.APIStyle = "ezsolver"
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("ezsolver style without api_key: %v", err)
|
||||
}
|
||||
cfg.Turnstile.APIStyle = ""
|
||||
cfg.Turnstile.APIKey = "k"
|
||||
cfg.Turnstile.Enabled = false
|
||||
cfg.Admin.Port = cfg.Server.Port
|
||||
|
||||
@@ -467,10 +467,7 @@ const uaMetadataExpr = `(() => {
|
||||
// warms the page like a human, renders the widget and clicks until a token
|
||||
// lands or the deadline passes.
|
||||
func (s *Solver) pollForToken(ctx context.Context, conn *cdp, sid, sitekey string, cfg config.Turnstile) (string, error) {
|
||||
timeout := time.Duration(cfg.TimeoutSeconds) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = 90 * time.Second
|
||||
}
|
||||
timeout := time.Duration(cfg.SolveTimeoutValue()) * time.Second
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
var preDone, inited bool
|
||||
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -91,6 +93,11 @@ type Solver struct {
|
||||
// browserMu serializes browser solves: one Chrome window at a time.
|
||||
browserMu sync.Mutex
|
||||
|
||||
// cookie pool: background warm refresh, one slot per site × route.
|
||||
poolMu sync.Mutex
|
||||
poolOn bool
|
||||
warms map[string]*warmState
|
||||
|
||||
// refreshOverride is a test seam; nil in production.
|
||||
refreshOverride func(ctx context.Context, sess httpx.Session, site *config.Site, c *core) error
|
||||
}
|
||||
@@ -107,7 +114,7 @@ func New(cfg config.Turnstile, userAgent string) *Solver {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Solver) Enabled() bool { return s.cfg.Enabled }
|
||||
func (s *Solver) Enabled() bool { return s.Config().Enabled }
|
||||
|
||||
func (s *Solver) Config() config.Turnstile {
|
||||
s.mu.Lock()
|
||||
@@ -133,7 +140,7 @@ func (s *Solver) coreFor(proxy string) *core {
|
||||
defer s.mu.Unlock()
|
||||
c, ok := s.cores[key]
|
||||
if !ok {
|
||||
client, _ := httpx.StdClient(proxy, time.Duration(s.cfg.TimeoutSeconds+30)*time.Second)
|
||||
client, _ := httpx.StdClient(proxy, time.Duration(s.cfg.SolveTimeoutValue()+30)*time.Second)
|
||||
c = &core{
|
||||
proxy: key,
|
||||
client: client,
|
||||
@@ -287,9 +294,17 @@ func (s *Solver) ApplyValidCookies(ctx context.Context, sess httpx.Session, site
|
||||
return s.refreshLocked(ctx, sess, site, c)
|
||||
}
|
||||
|
||||
// CookiesReady reports whether an unexpired cached cookie set exists for the
|
||||
// site × route pair. Upstream uses it to keep the slow-start timer from
|
||||
// killing a first-time Turnstile solve, which can legitimately take minutes.
|
||||
func (s *Solver) CookiesReady(siteCode, proxy string) bool {
|
||||
c := s.coreFor(proxy)
|
||||
return c.load(siteCode) != nil
|
||||
}
|
||||
|
||||
// Refresh forces a fresh solve → verify cycle.
|
||||
func (s *Solver) Refresh(ctx context.Context, sess httpx.Session, site *config.Site, proxy string) error {
|
||||
if !s.cfg.Enabled {
|
||||
if !s.Enabled() {
|
||||
return nil
|
||||
}
|
||||
c := s.coreFor(proxy)
|
||||
@@ -385,32 +400,70 @@ func (s *Solver) solveToken(ctx context.Context, site *config.Site, sitekey stri
|
||||
|
||||
func (s *Solver) solveTokenAPI(ctx context.Context, baseURL, sitekey string, c *core) (string, error) {
|
||||
cfg := s.Config()
|
||||
payload, _ := json.Marshal(map[string]any{
|
||||
"url": baseURL,
|
||||
"sitekey": sitekey,
|
||||
"action": cfg.Action,
|
||||
"cdata": "",
|
||||
"timeoutSeconds": cfg.TimeoutSeconds,
|
||||
})
|
||||
style := cfg.APIStyleValue()
|
||||
var payload []byte
|
||||
if style == "ezsolver" {
|
||||
payload, _ = json.Marshal(map[string]any{
|
||||
"sitekey": sitekey,
|
||||
"siteurl": baseURL,
|
||||
"timeout": cfg.SolveTimeoutValue(),
|
||||
})
|
||||
} else {
|
||||
payload, _ = json.Marshal(map[string]any{
|
||||
"url": baseURL,
|
||||
"sitekey": sitekey,
|
||||
"action": cfg.Action,
|
||||
"cdata": "",
|
||||
"timeoutSeconds": cfg.SolveTimeoutValue(),
|
||||
})
|
||||
}
|
||||
var last error
|
||||
// A loopback solve service must be reached directly: pushing it through
|
||||
// the route proxy would resolve 127.0.0.1 on the far side of the tunnel.
|
||||
client := c.client
|
||||
if isLocalHost(cfg.APIURL) {
|
||||
client = &http.Client{Timeout: time.Duration(cfg.SolveTimeoutValue()+30) * time.Second}
|
||||
}
|
||||
for attempt := 0; attempt < cfg.Retries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.APIURL, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.APIKey)
|
||||
resp, err := c.client.Do(req)
|
||||
if cfg.APIKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.APIKey)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
last = &Error{Msg: "turnstile solver request failed: " + err.Error(), Retryable: true}
|
||||
} else {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
retry := retryableStatus(resp.StatusCode)
|
||||
if style == "ezsolver" {
|
||||
// Local solve services answer 5xx for per-attempt
|
||||
// timeouts; another attempt is reasonable.
|
||||
retry = true
|
||||
}
|
||||
last = &Error{
|
||||
Msg: fmt.Sprintf("turnstile solver http %d: %s (check [turnstile].api_key / api_url)",
|
||||
resp.StatusCode, trim(body)),
|
||||
Retryable: retryableStatus(resp.StatusCode),
|
||||
Retryable: retry,
|
||||
}
|
||||
} else if style == "ezsolver" {
|
||||
var out struct {
|
||||
Token string `json:"token"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
last = &Error{Msg: "turnstile solver bad json: " + err.Error()}
|
||||
} else if out.Error != "" {
|
||||
last = &Error{Msg: "turnstile solver error: " + trim([]byte(out.Error))}
|
||||
} else if out.Token == "" {
|
||||
last = &Error{Msg: "turnstile solver returned no token"}
|
||||
} else {
|
||||
return out.Token, nil
|
||||
}
|
||||
} else {
|
||||
var out struct {
|
||||
@@ -444,6 +497,25 @@ func (s *Solver) solveTokenAPI(ctx context.Context, baseURL, sitekey string, c *
|
||||
return "", last
|
||||
}
|
||||
|
||||
// isLocalHost reports whether rawURL points at the local machine or a
|
||||
// private-network address. Such solve services must be reached directly
|
||||
// instead of through the route proxy.
|
||||
func isLocalHost(rawURL string) bool {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
if host == "localhost" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()
|
||||
}
|
||||
|
||||
// retryableStatus decides which solver HTTP statuses are worth another attempt.
|
||||
// Solver-side rejections (bad key, exhausted quota, challenge rejected) return
|
||||
// the same answer every time, so they are not retried.
|
||||
|
||||
@@ -2,7 +2,11 @@ package turnstile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -239,3 +243,43 @@ func TestRotateGuestSwapsVisitorID(t *testing.T) {
|
||||
t.Errorf("rotating an unknown site returned %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSolveTokenAPIEzSolverStyle(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
|
||||
t.Errorf("decode request: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"token":"tok-123","elapsed":1.5}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := config.Turnstile{
|
||||
Enabled: true, Provider: config.ProviderAPI, APIStyle: "ezsolver",
|
||||
APIURL: srv.URL, TimeoutSeconds: 30, Retries: 1,
|
||||
}
|
||||
solver := New(cfg, httpx.DefaultUserAgent)
|
||||
// A broken route-proxied client proves the loopback bypass: the call must
|
||||
// still succeed because local solve services are reached directly.
|
||||
brokenURL, _ := url.Parse("socks5://127.0.0.1:1")
|
||||
c := &core{
|
||||
proxy: "direct",
|
||||
client: &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(brokenURL)}},
|
||||
cache: map[string]*cookieState{}, locks: map[string]*sync.Mutex{},
|
||||
}
|
||||
|
||||
token, err := solver.solveTokenAPI(context.Background(), "https://deepseek.de/", "0xabc", c)
|
||||
if err != nil {
|
||||
t.Fatalf("solve: %v", err)
|
||||
}
|
||||
if token != "tok-123" {
|
||||
t.Fatalf("token = %q", token)
|
||||
}
|
||||
if gotBody["sitekey"] != "0xabc" || gotBody["siteurl"] != "https://deepseek.de/" {
|
||||
t.Fatalf("request body = %v", gotBody)
|
||||
}
|
||||
if gotBody["timeout"] != float64(30) {
|
||||
t.Fatalf("timeout = %v", gotBody["timeout"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package turnstile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nyoungo/dsfree2api/internal/config"
|
||||
"github.com/nyoungo/dsfree2api/internal/httpx"
|
||||
)
|
||||
|
||||
// WarmTarget is one cookie-pool slot: a site reached through one proxy route.
|
||||
// Verified cookies are bound to the egress IP that solved them, so the pool
|
||||
// keeps one slot per site × route.
|
||||
type WarmTarget struct {
|
||||
Site config.Site
|
||||
Proxy string // "" means direct
|
||||
}
|
||||
|
||||
// WarmEntry describes the pool state of one site × route slot.
|
||||
type WarmEntry struct {
|
||||
Site string `json:"site"`
|
||||
Route string `json:"route"`
|
||||
Valid bool `json:"valid"`
|
||||
Remaining int64 `json:"remaining_s"`
|
||||
Warming bool `json:"warming"`
|
||||
LastAt int64 `json:"last_at"`
|
||||
LastMs int64 `json:"last_ms"`
|
||||
LastErr string `json:"last_error,omitempty"`
|
||||
NextWarm int64 `json:"next_warm_at"`
|
||||
}
|
||||
|
||||
// PoolStatus is the console view of the cookie pool.
|
||||
type PoolStatus struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Running bool `json:"running"`
|
||||
Ratio float64 `json:"ratio"`
|
||||
CheckS int `json:"check_seconds"`
|
||||
Entries []WarmEntry `json:"entries"`
|
||||
}
|
||||
|
||||
type warmState struct {
|
||||
site string
|
||||
route string
|
||||
warming bool
|
||||
fails int
|
||||
lastAt time.Time
|
||||
lastMs int64
|
||||
lastErr string
|
||||
}
|
||||
|
||||
// StartWarmer launches the cookie-pool loop. targets is consulted on every
|
||||
// sweep, so console edits (sites, routes, toggles) are picked up live. A slot
|
||||
// is warmed when its cookie is missing, expired, or has less than warm_ratio
|
||||
// of its TTL left — requests then never wait for a solve to finish.
|
||||
func (s *Solver) StartWarmer(ctx context.Context, targets func() []WarmTarget) {
|
||||
if targets == nil {
|
||||
return
|
||||
}
|
||||
s.poolMu.Lock()
|
||||
if s.poolOn {
|
||||
s.poolMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.poolOn = true
|
||||
if s.warms == nil {
|
||||
s.warms = map[string]*warmState{}
|
||||
}
|
||||
s.poolMu.Unlock()
|
||||
go s.warmLoop(ctx, targets)
|
||||
}
|
||||
|
||||
func (s *Solver) warmLoop(ctx context.Context, targets func() []WarmTarget) {
|
||||
defer func() {
|
||||
s.poolMu.Lock()
|
||||
s.poolOn = false
|
||||
s.poolMu.Unlock()
|
||||
}()
|
||||
for {
|
||||
s.sweep(ctx, targets)
|
||||
wait := time.Duration(s.Config().WarmCheckValue()) * time.Second
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(wait):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweep warms every due slot. Solves run in the background so one slow
|
||||
// browser window cannot stall the inspection; a slot already being warmed is
|
||||
// skipped until it finishes, and a slot whose last attempt failed waits out an
|
||||
// exponential backoff before the next try.
|
||||
func (s *Solver) sweep(ctx context.Context, targets func() []WarmTarget) {
|
||||
cfg := s.Config()
|
||||
if !cfg.Enabled || !cfg.WarmEnabled || cfg.ProviderValue() == config.ProviderManual {
|
||||
return
|
||||
}
|
||||
ttl := time.Duration(cfg.CookieTTLSeconds) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 3 * time.Hour
|
||||
}
|
||||
for _, tgt := range targets() {
|
||||
key := poolKey(tgt.Proxy, tgt.Site.Code)
|
||||
if !s.warmDue(key, tgt, ttl, cfg.WarmRatioValue(), cfg.WarmCheckValue()) {
|
||||
continue
|
||||
}
|
||||
s.markWarm(key, tgt, true, time.Time{}, 0, "")
|
||||
go s.warmOne(ctx, key, tgt, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Solver) warmDue(key string, tgt WarmTarget, ttl time.Duration, ratio float64, checkS int) bool {
|
||||
s.poolMu.Lock()
|
||||
st := s.warms[key]
|
||||
if st != nil && st.warming {
|
||||
s.poolMu.Unlock()
|
||||
return false
|
||||
}
|
||||
var lastAt time.Time
|
||||
fails := 0
|
||||
if st != nil {
|
||||
lastAt, fails = st.lastAt, st.fails
|
||||
}
|
||||
s.poolMu.Unlock()
|
||||
|
||||
// A failed slot backs off exponentially (×2 per consecutive failure, up
|
||||
// to 15 minutes) so a hopeless route cannot retry-storm the browser pool.
|
||||
if fails > 0 && !lastAt.IsZero() {
|
||||
backoff := time.Duration(checkS) * time.Second
|
||||
for i := 1; i < fails && backoff < 15*time.Minute; i++ {
|
||||
backoff *= 2
|
||||
}
|
||||
if backoff > 15*time.Minute || backoff <= 0 {
|
||||
backoff = 15 * time.Minute
|
||||
}
|
||||
if time.Since(lastAt) < backoff {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
c := s.coreFor(tgt.Proxy)
|
||||
c.mu.Lock()
|
||||
cs := c.cache[tgt.Site.Code]
|
||||
c.mu.Unlock()
|
||||
if cs == nil {
|
||||
return true // cold slot — fill it
|
||||
}
|
||||
return time.Until(cs.expiresAt) <= time.Duration(ratio*float64(ttl))
|
||||
}
|
||||
|
||||
func (s *Solver) warmOne(ctx context.Context, key string, tgt WarmTarget, cfg config.Turnstile) {
|
||||
site := tgt.Site
|
||||
start := time.Now()
|
||||
sess, err := httpx.NewSession(httpx.Options{
|
||||
ProxyURL: tgt.Proxy,
|
||||
Timeout: time.Duration(cfg.SolveTimeoutValue()+30) * time.Second,
|
||||
})
|
||||
if err == nil {
|
||||
err = s.Refresh(ctx, sess, &site, tgt.Proxy)
|
||||
sess.Close()
|
||||
}
|
||||
lastErr := ""
|
||||
if err != nil {
|
||||
lastErr = err.Error()
|
||||
}
|
||||
s.markWarm(key, tgt, false, start, time.Since(start).Milliseconds(), lastErr)
|
||||
}
|
||||
|
||||
func (s *Solver) markWarm(key string, tgt WarmTarget, warming bool, at time.Time, ms int64, lastErr string) {
|
||||
s.poolMu.Lock()
|
||||
defer s.poolMu.Unlock()
|
||||
if s.warms == nil {
|
||||
s.warms = map[string]*warmState{}
|
||||
}
|
||||
st, ok := s.warms[key]
|
||||
if !ok {
|
||||
st = &warmState{site: tgt.Site.Code, route: routeLabel(tgt.Proxy)}
|
||||
s.warms[key] = st
|
||||
}
|
||||
st.warming = warming
|
||||
if !at.IsZero() {
|
||||
st.lastAt = at
|
||||
st.lastMs = ms
|
||||
st.lastErr = lastErr
|
||||
if lastErr == "" {
|
||||
st.fails = 0
|
||||
} else {
|
||||
st.fails++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PoolStatus reports every known slot: cached lifetime, warm progress, last
|
||||
// solve duration and the next planned refresh.
|
||||
func (s *Solver) PoolStatus() PoolStatus {
|
||||
cfg := s.Config()
|
||||
pooling := cfg.Enabled && cfg.WarmEnabled && cfg.ProviderValue() != config.ProviderManual
|
||||
ps := PoolStatus{
|
||||
Enabled: pooling,
|
||||
Ratio: cfg.WarmRatioValue(),
|
||||
CheckS: cfg.WarmCheckValue(),
|
||||
Entries: make([]WarmEntry, 0, 4),
|
||||
}
|
||||
s.poolMu.Lock()
|
||||
ps.Running = s.poolOn
|
||||
states := make(map[string]*warmState, len(s.warms))
|
||||
for k, v := range s.warms {
|
||||
cp := *v
|
||||
states[k] = &cp
|
||||
}
|
||||
s.poolMu.Unlock()
|
||||
|
||||
s.mu.Lock()
|
||||
cores := make([]*core, 0, len(s.cores))
|
||||
for _, c := range s.cores {
|
||||
cores = append(cores, c)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
ttl := time.Duration(cfg.CookieTTLSeconds) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 3 * time.Hour
|
||||
}
|
||||
|
||||
entries := map[string]*WarmEntry{}
|
||||
for _, c := range cores {
|
||||
c.mu.Lock()
|
||||
for code, cs := range c.cache {
|
||||
e := &WarmEntry{Site: code, Route: c.proxy, Valid: now.Before(cs.expiresAt)}
|
||||
if e.Valid {
|
||||
e.Remaining = int64(cs.expiresAt.Sub(now).Seconds())
|
||||
}
|
||||
entries[poolKey(c.proxy, code)] = e
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
for k, st := range states {
|
||||
e, ok := entries[k]
|
||||
if !ok {
|
||||
e = &WarmEntry{Site: st.site, Route: st.route}
|
||||
entries[k] = e
|
||||
}
|
||||
e.Warming = st.warming
|
||||
if !st.lastAt.IsZero() {
|
||||
e.LastAt = st.lastAt.Unix()
|
||||
}
|
||||
e.LastMs = st.lastMs
|
||||
e.LastErr = st.lastErr
|
||||
}
|
||||
for _, e := range entries {
|
||||
if pooling && e.Valid {
|
||||
until := time.Duration(e.Remaining)*time.Second - time.Duration(ps.Ratio*float64(ttl))
|
||||
if until < 0 {
|
||||
until = 0
|
||||
}
|
||||
e.NextWarm = now.Add(until).Unix()
|
||||
}
|
||||
ps.Entries = append(ps.Entries, *e)
|
||||
}
|
||||
sort.Slice(ps.Entries, func(i, j int) bool {
|
||||
if ps.Entries[i].Site != ps.Entries[j].Site {
|
||||
return ps.Entries[i].Site < ps.Entries[j].Site
|
||||
}
|
||||
return ps.Entries[i].Route < ps.Entries[j].Route
|
||||
})
|
||||
return ps
|
||||
}
|
||||
|
||||
func poolKey(proxy, site string) string { return routeLabel(proxy) + "|" + site }
|
||||
|
||||
func routeLabel(proxy string) string {
|
||||
if proxy == "" {
|
||||
return "direct"
|
||||
}
|
||||
return proxy
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package turnstile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nyoungo/dsfree2api/internal/config"
|
||||
"github.com/nyoungo/dsfree2api/internal/httpx"
|
||||
)
|
||||
|
||||
func warmerSolver(t *testing.T, mutate func(*config.Turnstile)) (*Solver, *config.Config) {
|
||||
t.Helper()
|
||||
cfg, err := config.Load(examplePath)
|
||||
if err != nil {
|
||||
t.Fatalf("load config: %v", err)
|
||||
}
|
||||
cfg.Turnstile.Enabled = true
|
||||
cfg.Turnstile.WarmEnabled = true
|
||||
cfg.Turnstile.CookieTTLSeconds = 3600
|
||||
cfg.Turnstile.WarmRatio = 0.3
|
||||
if mutate != nil {
|
||||
mutate(&cfg.Turnstile)
|
||||
}
|
||||
return New(cfg.Turnstile, httpx.DefaultUserAgent), cfg
|
||||
}
|
||||
|
||||
func waitCalls(t *testing.T, calls *int32, want int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if atomic.LoadInt32(calls) >= want {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("refresh calls = %d, want >= %d", atomic.LoadInt32(calls), want)
|
||||
}
|
||||
|
||||
func waitSettled(t *testing.T, solver *Solver) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
ps := solver.PoolStatus()
|
||||
busy := false
|
||||
for _, e := range ps.Entries {
|
||||
if e.Warming {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
if !busy {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("cookie pool did not settle")
|
||||
}
|
||||
|
||||
// rewindLastAt backdates the last attempt of a slot so backoff windows can be
|
||||
// tested without sleeping through them.
|
||||
func rewindLastAt(solver *Solver, key string, d time.Duration) {
|
||||
solver.poolMu.Lock()
|
||||
if st := solver.warms[key]; st != nil {
|
||||
st.lastAt = time.Now().Add(-d)
|
||||
}
|
||||
solver.poolMu.Unlock()
|
||||
}
|
||||
|
||||
func TestWarmDueBacksOffAfterFailure(t *testing.T) {
|
||||
solver, cfg := warmerSolver(t, nil)
|
||||
tgt := WarmTarget{Site: *cfg.Sites["de"], Proxy: ""}
|
||||
key := poolKey("", "de")
|
||||
ttl := time.Hour
|
||||
|
||||
if !solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("cold slot should be due")
|
||||
}
|
||||
|
||||
// First failure: the 5s backoff must hold.
|
||||
solver.markWarm(key, tgt, false, time.Now(), 100, "boom")
|
||||
if solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("first backoff was ignored")
|
||||
}
|
||||
|
||||
// Second failure doubles it: 6s elapsed is still inside the 10s window.
|
||||
solver.markWarm(key, tgt, false, time.Now(), 100, "boom")
|
||||
rewindLastAt(solver, key, 6*time.Second)
|
||||
if solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("second failure did not double the backoff")
|
||||
}
|
||||
|
||||
// 11s elapsed clears the 10s window.
|
||||
rewindLastAt(solver, key, 11*time.Second)
|
||||
if !solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("slot should be due once the backoff elapsed")
|
||||
}
|
||||
|
||||
// A success resets the failure counter.
|
||||
solver.markWarm(key, tgt, false, time.Now(), 50, "")
|
||||
if !solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("success should clear the backoff")
|
||||
}
|
||||
|
||||
// The backoff is capped at 15 minutes.
|
||||
for i := 0; i < 12; i++ {
|
||||
solver.markWarm(key, tgt, false, time.Now(), 100, "boom")
|
||||
}
|
||||
rewindLastAt(solver, key, 11*time.Second)
|
||||
if solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("capped backoff should still hold at 11s")
|
||||
}
|
||||
rewindLastAt(solver, key, 16*time.Minute)
|
||||
if !solver.warmDue(key, tgt, ttl, 0.3, 5) {
|
||||
t.Fatal("slot should be due after the capped backoff")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarmSweepFillsColdSlotAndSkipsFresh(t *testing.T) {
|
||||
solver, cfg := warmerSolver(t, nil)
|
||||
var calls int32
|
||||
solver.refreshOverride = func(_ context.Context, _ httpx.Session, s *config.Site, c *core) error {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
c.mu.Lock()
|
||||
c.cache[s.Code] = &cookieState{cookies: map[string]string{"dsts": "ok"}, expiresAt: time.Now().Add(time.Hour)}
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
targets := func() []WarmTarget {
|
||||
return []WarmTarget{{Site: *cfg.Sites["de"], Proxy: ""}}
|
||||
}
|
||||
|
||||
// Cold slot: the first sweep must start a solve.
|
||||
solver.sweep(context.Background(), targets)
|
||||
waitCalls(t, &calls, 1)
|
||||
waitSettled(t, solver)
|
||||
|
||||
// Fresh cookie (1h left, threshold 0.3×1h): a second sweep must not solve.
|
||||
solver.sweep(context.Background(), targets)
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if got := atomic.LoadInt32(&calls); got != 1 {
|
||||
t.Fatalf("fresh slot was re-warmed: calls = %d", got)
|
||||
}
|
||||
|
||||
// Remaining TTL below the threshold: due again.
|
||||
c := solver.coreFor("")
|
||||
c.mu.Lock()
|
||||
c.cache["de"] = &cookieState{cookies: map[string]string{"dsts": "ok"}, expiresAt: time.Now().Add(30 * time.Second)}
|
||||
c.mu.Unlock()
|
||||
solver.sweep(context.Background(), targets)
|
||||
waitCalls(t, &calls, 2)
|
||||
waitSettled(t, solver)
|
||||
|
||||
ps := solver.PoolStatus()
|
||||
if !ps.Enabled {
|
||||
t.Error("pool should report enabled")
|
||||
}
|
||||
if len(ps.Entries) != 1 {
|
||||
t.Fatalf("pool entries = %d, want 1", len(ps.Entries))
|
||||
}
|
||||
e := ps.Entries[0]
|
||||
if !e.Valid || e.LastAt == 0 || e.LastErr != "" || e.Warming {
|
||||
t.Errorf("pool entry = %+v", e)
|
||||
}
|
||||
if e.NextWarm == 0 {
|
||||
t.Error("next_warm_at should be scheduled for a valid pooled cookie")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarmSweepSkipsManualAndDisabled(t *testing.T) {
|
||||
var calls int32
|
||||
override := func(context.Context, httpx.Session, *config.Site, *core) error {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
solver, cfg := warmerSolver(t, func(tc *config.Turnstile) { tc.Provider = config.ProviderManual })
|
||||
solver.refreshOverride = override
|
||||
solver.sweep(context.Background(), func() []WarmTarget {
|
||||
return []WarmTarget{{Site: *cfg.Sites["de"]}}
|
||||
})
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if calls != 0 {
|
||||
t.Fatalf("manual provider must not warm, calls = %d", calls)
|
||||
}
|
||||
|
||||
solver2, cfg2 := warmerSolver(t, func(tc *config.Turnstile) { tc.Enabled = false })
|
||||
solver2.refreshOverride = override
|
||||
solver2.sweep(context.Background(), func() []WarmTarget {
|
||||
return []WarmTarget{{Site: *cfg2.Sites["de"]}}
|
||||
})
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if calls != 0 {
|
||||
t.Fatalf("disabled solver must not warm, calls = %d", calls)
|
||||
}
|
||||
|
||||
solver3, cfg3 := warmerSolver(t, func(tc *config.Turnstile) { tc.WarmEnabled = false })
|
||||
solver3.refreshOverride = override
|
||||
solver3.sweep(context.Background(), func() []WarmTarget {
|
||||
return []WarmTarget{{Site: *cfg3.Sites["de"]}}
|
||||
})
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if calls != 0 {
|
||||
t.Fatalf("pool switch off must not warm, calls = %d", calls)
|
||||
}
|
||||
}
|
||||
@@ -411,7 +411,11 @@ func (c *Client) chatWithRetries(ctx context.Context, modelID string, model conf
|
||||
for attempt := 0; ; attempt++ {
|
||||
var failed []Route
|
||||
for i, route := range routes {
|
||||
allowSlow := i == 0 && len(routes) > 1
|
||||
// A first-time Turnstile solve can far exceed the slow-start
|
||||
// budget; only arm the timer when cookies are already cached
|
||||
// (or no solver is configured at all).
|
||||
cookiesReady := c.ts == nil || c.ts.CookiesReady(site.Code, route.Proxy)
|
||||
allowSlow := i == 0 && len(routes) > 1 && cookiesReady
|
||||
err := c.chatOnceSlowStart(ctx, site, modelID, model, prompt, route, allowSlow, info, yield)
|
||||
if err == nil {
|
||||
if sawQuota {
|
||||
|
||||
Reference in new issue
Block a user