from __future__ import annotations import json import os from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import urlparse from urllib.request import ProxyHandler, Request, build_opener from .utils import DATA_DIR, now_iso, read_json, sha1_hex, write_json CACHE_DIR = DATA_DIR / "cache" / "http" class HttpRequestError(RuntimeError): pass def _cache_path(namespace: str, url: str) -> Path: return CACHE_DIR / namespace / f"{sha1_hex(url)}.json" def _cache_payload(path: Path, payload: Any, ttl_seconds: int, max_stale_seconds: int) -> None: now_ms = int(__import__("time").time() * 1000) write_json( path, { "fetchedAt": now_iso(), "fetchedAtMs": now_ms, "expiresAtMs": now_ms + max(1, ttl_seconds) * 1000, "staleUntilMs": now_ms + max(ttl_seconds, max_stale_seconds) * 1000, "payload": payload, }, ) def _cache_is_fresh(cache: dict[str, Any] | None) -> bool: if not cache: return False return int(cache.get("expiresAtMs") or 0) > int(__import__("time").time() * 1000) def _cache_is_usable(cache: dict[str, Any] | None) -> bool: if not cache: return False return int(cache.get("staleUntilMs") or 0) > int(__import__("time").time() * 1000) def _should_bypass_proxy(hostname: str, network_settings: dict[str, Any]) -> bool: no_proxy = [item.lower() for item in network_settings.get("noProxy", [])] host = (hostname or "").lower() return any(host == item or host.endswith(f".{item}") for item in no_proxy) def _build_opener(url: str, network_settings: dict[str, Any] | None): if not network_settings: return build_opener() parsed = urlparse(url) if not network_settings.get("proxyEnabled") or not network_settings.get("proxyUrl") or _should_bypass_proxy(parsed.hostname or "", network_settings): return build_opener() proxy_url = str(network_settings.get("proxyUrl") or "").strip() scheme = parsed.scheme.lower() proxies = { "http": proxy_url, "https": proxy_url, } if scheme in {"http", "https"}: return build_opener(ProxyHandler(proxies)) return build_opener() def request_text( method: str, url: str, *, headers: dict[str, str] | None = None, payload: Any = None, timeout_seconds: int = 45, network_settings: dict[str, Any] | None = None, ) -> str: body: bytes | None if payload is None: body = None elif isinstance(payload, (bytes, bytearray)): body = bytes(payload) elif isinstance(payload, str): body = payload.encode("utf-8") else: body = json.dumps(payload).encode("utf-8") request = Request(url=url, method=method.upper(), data=body) merged_headers = { "accept": "application/json", "user-agent": "python-trading-agent/1.0", } if body is not None and "content-type" not in {key.lower() for key in (headers or {})}: merged_headers["content-type"] = "application/json" merged_headers.update(headers or {}) for key, value in merged_headers.items(): request.add_header(key, value) opener = _build_opener(url, network_settings or {}) try: with opener.open(request, timeout=timeout_seconds) as response: return response.read().decode("utf-8") except HTTPError as error: detail = error.read().decode("utf-8", errors="replace") raise HttpRequestError(f"{error.code} {error.reason}: {detail}") from error except URLError as error: raise HttpRequestError(str(error.reason)) from error def request_json( method: str, url: str, *, headers: dict[str, str] | None = None, payload: Any = None, timeout_seconds: int = 45, network_settings: dict[str, Any] | None = None, ) -> Any: text = request_text( method, url, headers=headers, payload=payload, timeout_seconds=timeout_seconds, network_settings=network_settings, ) try: return json.loads(text) except json.JSONDecodeError as error: snippet = text[:220].replace("\n", " ").strip() if len(text) > 220: snippet += "..." message = f"invalid JSON response from {url}: {error}" if snippet: message += f" | response starts with: {snippet}" raise HttpRequestError(message) from error def cached_get_json( url: str, *, namespace: str = "generic", ttl_seconds: int = 60, max_stale_seconds: int = 3600, timeout_seconds: int = 45, headers: dict[str, str] | None = None, network_settings: dict[str, Any] | None = None, ) -> Any: path = _cache_path(namespace, url) cache = read_json(path, {}) if _cache_is_fresh(cache): return cache.get("payload") try: payload = request_json( "GET", url, headers=headers, timeout_seconds=timeout_seconds, network_settings=network_settings, ) _cache_payload(path, payload, ttl_seconds, max_stale_seconds) return payload except Exception: if _cache_is_usable(cache): return cache.get("payload") raise