perf: 并行获取所有账号邮件(ThreadPoolExecutor)
串行 5 账号耗时 ~145s → 并行后预计 ~30s 提取 _fetch_account_emails 独立函数用于并行 游标逻辑不变:成功推进、失败保留 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
1 parent
f2f3c62b57
commit
9c838da1f3
1 file changed
+49
-31
@@ -263,9 +263,32 @@ def _fetch_new_emails_graph(account: dict, since: str) -> List[dict]:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_account_emails(account: dict) -> tuple:
|
||||||
|
"""并行获取单个账号新邮件,返回 (account, emails_list, error)。"""
|
||||||
|
last_checked = account.get("telegram_last_checked_at")
|
||||||
|
if last_checked is None:
|
||||||
|
return (account, None, None) # 首次运行,仅设置游标
|
||||||
|
|
||||||
|
try:
|
||||||
|
if account.get("provider") == "outlook":
|
||||||
|
emails = _fetch_new_emails_graph(account, last_checked)
|
||||||
|
else:
|
||||||
|
emails = _fetch_new_emails_imap(account, last_checked)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"[telegram_push] account=%s provider=%s since=%s found=%d",
|
||||||
|
account.get("email"), account.get("provider"), last_checked, len(emails),
|
||||||
|
)
|
||||||
|
return (account, emails, None)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("[telegram_push] account=%s error: %s", account.get("email"), e)
|
||||||
|
return (account, None, e)
|
||||||
|
|
||||||
|
|
||||||
def run_telegram_push_job(app) -> None:
|
def run_telegram_push_job(app) -> None:
|
||||||
"""主入口:轮询 → 推送 → 更新游标。由调度器调用。"""
|
"""主入口:并行轮询 → 推送 → 更新游标。由调度器调用。"""
|
||||||
import time
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
|
||||||
t0 = time.monotonic()
|
t0 = time.monotonic()
|
||||||
logger.info("[telegram_push] job started")
|
logger.info("[telegram_push] job started")
|
||||||
@@ -294,41 +317,36 @@ def run_telegram_push_job(app) -> None:
|
|||||||
job_start_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
job_start_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
||||||
sent_count = 0
|
sent_count = 0
|
||||||
|
|
||||||
for account in accounts:
|
# 并行获取所有账号邮件
|
||||||
|
fetch_results = []
|
||||||
|
with ThreadPoolExecutor(max_workers=min(len(accounts), 10)) as executor:
|
||||||
|
futures = {executor.submit(_fetch_account_emails, acc): acc for acc in accounts}
|
||||||
|
for future in as_completed(futures):
|
||||||
|
fetch_results.append(future.result())
|
||||||
|
|
||||||
|
# 顺序推送 + 更新游标
|
||||||
|
for account, emails, error in fetch_results:
|
||||||
|
if emails is None and error is None:
|
||||||
|
# 首次运行,仅设置游标
|
||||||
|
update_telegram_cursor(account["id"], job_start_time)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if error is not None:
|
||||||
|
# fetch 失败,不推进游标
|
||||||
|
continue
|
||||||
|
|
||||||
if sent_count >= MAX_SENT_PER_JOB:
|
if sent_count >= MAX_SENT_PER_JOB:
|
||||||
update_telegram_cursor(account["id"], job_start_time)
|
update_telegram_cursor(account["id"], job_start_time)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
last_checked = account.get("telegram_last_checked_at")
|
for em in sorted(emails, key=lambda e: e.get("received_at", "")):
|
||||||
|
if sent_count >= MAX_SENT_PER_JOB:
|
||||||
|
break
|
||||||
|
msg = _build_telegram_message(account["email"], em)
|
||||||
|
if _send_telegram_message(bot_token, chat_id, msg):
|
||||||
|
sent_count += 1
|
||||||
|
|
||||||
if last_checked is None:
|
update_telegram_cursor(account["id"], job_start_time)
|
||||||
update_telegram_cursor(account["id"], job_start_time)
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
if account.get("provider") == "outlook":
|
|
||||||
emails = _fetch_new_emails_graph(account, last_checked)
|
|
||||||
else:
|
|
||||||
emails = _fetch_new_emails_imap(account, last_checked)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"[telegram_push] account=%s provider=%s since=%s found=%d",
|
|
||||||
account.get("email"), account.get("provider"), last_checked, len(emails),
|
|
||||||
)
|
|
||||||
|
|
||||||
for em in sorted(emails, key=lambda e: e.get("received_at", "")):
|
|
||||||
if sent_count >= MAX_SENT_PER_JOB:
|
|
||||||
break
|
|
||||||
msg = _build_telegram_message(account["email"], em)
|
|
||||||
if _send_telegram_message(bot_token, chat_id, msg):
|
|
||||||
sent_count += 1
|
|
||||||
|
|
||||||
# 仅在 fetch 成功时推进游标(失败时保留旧游标,避免漏推)
|
|
||||||
update_telegram_cursor(account["id"], job_start_time)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("[telegram_push] account=%s error: %s", account.get("email"), e)
|
|
||||||
# 不推进游标——下次重试时会重新拉取
|
|
||||||
|
|
||||||
elapsed = time.monotonic() - t0
|
elapsed = time.monotonic() - t0
|
||||||
logger.info("[telegram_push] job finished: sent=%d elapsed=%.1fs", sent_count, elapsed)
|
logger.info("[telegram_push] job finished: sent=%d elapsed=%.1fs", sent_count, elapsed)
|
||||||
Reference in new issue
Block a user