diff --git a/app/api/v1/public_api/voice.py b/app/api/v1/public_api/voice.py index 12612f0..16f81df 100644 --- a/app/api/v1/public_api/voice.py +++ b/app/api/v1/public_api/voice.py @@ -1,19 +1,248 @@ +import asyncio +from typing import Any +from urllib.parse import urlencode, urlparse, urlunparse + +import aiohttp from fastapi import APIRouter, Depends +from fastapi import Request, WebSocket, WebSocketDisconnect from pydantic import BaseModel from app.core.auth import verify_public_key +from app.core.config import get_config from app.core.exceptions import AppException +from app.core.logger import logger from app.services.grok.services.voice import VoiceService +from app.services.reverse.utils.headers import build_ws_headers +from app.services.reverse.utils.websocket import WebSocketClient from app.services.token.manager import get_token_manager router = APIRouter() +DEFAULT_LIVEKIT_URL = "wss://livekit.grok.com" class VoiceTokenResponse(BaseModel): token: str url: str + urls: list[str] | None = None participant_name: str = "" room_name: str = "" + ice_servers: list[dict[str, Any]] | None = None + signal_proxy_url: str | None = None + + +def _deep_get(data: dict[str, Any], path: tuple[str, ...]) -> Any: + """按路径安全读取嵌套字段。""" + node: Any = data + for key in path: + if not isinstance(node, dict): + return None + node = node.get(key) + return node + + +def _first_string(data: dict[str, Any], paths: list[tuple[str, ...]]) -> str: + """按优先级获取第一个非空字符串字段。""" + for path in paths: + value = _deep_get(data, path) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _normalize_ice_servers(raw: Any) -> list[dict[str, Any]]: + """归一化 ICE servers,兼容 url/urls 多种结构。""" + if not isinstance(raw, list): + return [] + + result: list[dict[str, Any]] = [] + for item in raw: + if not isinstance(item, dict): + continue + + urls_value = item.get("urls") + if urls_value is None: + urls_value = item.get("url") + + urls: list[str] = [] + if isinstance(urls_value, str) and urls_value.strip(): + urls = [urls_value.strip()] + elif isinstance(urls_value, list): + urls = [u.strip() for u in urls_value if isinstance(u, str) and u.strip()] + + if not urls: + continue + + normalized: dict[str, Any] = {"urls": urls} + username = item.get("username") + credential = item.get("credential") + if isinstance(username, str) and username.strip(): + normalized["username"] = username.strip() + if credential is not None: + normalized["credential"] = credential + result.append(normalized) + + return result + + +def _normalize_ws_url(raw: Any) -> str: + """规范化 ws/wss URL,保留协议、主机与路径。""" + if not isinstance(raw, str): + return "" + value = raw.strip() + if not value: + return "" + if "://" not in value: + value = f"wss://{value}" + try: + parsed = urlparse(value) + if parsed.scheme not in ("ws", "wss") or not parsed.netloc: + return "" + path = (parsed.path or "").rstrip("/") + return urlunparse((parsed.scheme, parsed.netloc, path, "", "", "")) + except Exception: + return "" + + +def _normalize_ws_url_list(raw: Any) -> list[str]: + """归一化 URL 列表,兼容字符串、列表与逗号分隔形式。""" + values: list[str] = [] + if isinstance(raw, str): + values = [v.strip() for v in raw.replace("\n", ",").split(",")] + elif isinstance(raw, list): + values = [v for v in raw if isinstance(v, str)] + else: + return [] + + result: list[str] = [] + for value in values: + normalized = _normalize_ws_url(value) + if normalized and normalized not in result: + result.append(normalized) + return result + + +def _extract_connection_info( + data: dict[str, Any], +) -> tuple[str, list[str], list[dict[str, Any]]]: + """提取连接 URL 列表与 ICE 配置,缺失时使用配置和默认值。""" + url_paths = [ + ("url",), + ("livekitUrl",), + ("livekit_url",), + ("livekitServerUrl",), + ("ws_url",), + ("serverUrl",), + ("livekit", "url"), + ("livekit", "ws_url"), + ("connection", "url"), + ("connectionDetails", "url"), + ("connection_details", "url"), + ] + upstream_primary = _normalize_ws_url(_first_string(data, url_paths)) + + url_list_paths = [ + ("urls",), + ("livekitUrls",), + ("livekit_urls",), + ("connection", "urls"), + ("connectionDetails", "urls"), + ("connection_details", "urls"), + ] + upstream_urls: list[str] = [] + for path in url_list_paths: + upstream_urls = _normalize_ws_url_list(_deep_get(data, path)) + if upstream_urls: + break + + configured_primary = _normalize_ws_url(get_config("voice.livekit_url", "")) + configured_urls = _normalize_ws_url_list(get_config("voice.livekit_urls", [])) + urls: list[str] = [] + for candidate in ( + [configured_primary] + configured_urls + [upstream_primary] + upstream_urls + [DEFAULT_LIVEKIT_URL] + ): + normalized = _normalize_ws_url(candidate) + if normalized and normalized not in urls: + urls.append(normalized) + + ice_paths = [ + ("iceServers",), + ("ice_servers",), + ("rtcConfig", "iceServers"), + ("rtcConfig", "ice_servers"), + ("rtc_config", "iceServers"), + ("rtc_config", "ice_servers"), + ("connectionDetails", "rtcConfig", "iceServers"), + ("connectionDetails", "rtc_config", "ice_servers"), + ("connection_details", "rtcConfig", "iceServers"), + ] + ice_servers: list[dict[str, Any]] = [] + for path in ice_paths: + ice_servers = _normalize_ice_servers(_deep_get(data, path)) + if ice_servers: + break + + return urls[0], urls, ice_servers + + +def _build_signal_proxy_url(request: Request) -> str: + """生成前端可直连的同域信令代理地址。""" + if not bool(get_config("voice.signal_proxy_enabled", True)): + return "" + + configured = _normalize_ws_url(get_config("voice.signal_proxy_url", "")) + if configured: + return configured + + forwarded_host = request.headers.get("x-forwarded-host", "") + host = (forwarded_host.split(",")[0].strip() if forwarded_host else "") or request.headers.get("host", "") + if not host: + return "" + + forwarded_proto = request.headers.get("x-forwarded-proto", "") + proto = (forwarded_proto.split(",")[0].strip().lower() if forwarded_proto else "") or request.url.scheme.lower() + ws_scheme = "wss" if proto == "https" else "ws" + return f"{ws_scheme}://{host}/v1/public/voice/signal" + + +def _build_upstream_signal_url(websocket: WebSocket, tail: str) -> str: + """把浏览器请求映射到上游 LiveKit 信令 URL。""" + upstream_override = _normalize_ws_url(websocket.query_params.get("upstream")) + configured_primary = _normalize_ws_url(get_config("voice.livekit_url", "")) + configured_urls = _normalize_ws_url_list(get_config("voice.livekit_urls", [])) + base_url = upstream_override or configured_primary or ( + configured_urls[0] if configured_urls else DEFAULT_LIVEKIT_URL + ) + + parsed = urlparse(base_url) + base_path = (parsed.path or "").rstrip("/") + tail_clean = (tail or "").strip("/") + if tail_clean: + tail_path = f"/{tail_clean}" + if base_path.endswith(tail_path): + target_path = base_path + elif base_path: + target_path = f"{base_path}{tail_path}" + else: + target_path = tail_path + else: + target_path = base_path + + query_items = [(k, v) for k, v in websocket.query_params.multi_items() if k != "upstream"] + query = urlencode(query_items, doseq=True) + return urlunparse((parsed.scheme, parsed.netloc, target_path, "", query, "")) + + +def _mask_url(url: str) -> str: + """仅保留协议与主机,避免记录敏感参数。""" + try: + from urllib.parse import urlparse + + parsed = urlparse(url) + if parsed.scheme and parsed.netloc: + return f"{parsed.scheme}://{parsed.netloc}{parsed.path or ''}" + except Exception: + pass + return url @router.get( @@ -22,6 +251,7 @@ class VoiceTokenResponse(BaseModel): response_model=VoiceTokenResponse, ) async def public_voice_token( + request: Request, voice: str = "ara", personality: str = "assistant", speed: float = 1.0, @@ -57,11 +287,41 @@ async def public_voice_token( status_code=502, ) + url, urls, ice_servers = _extract_connection_info(data) + signal_proxy_url = _build_signal_proxy_url(request) + participant_name = _first_string( + data, + [ + ("participant_name",), + ("participantName",), + ("identity",), + ], + ) + room_name = _first_string( + data, + [ + ("room_name",), + ("roomName",), + ("room",), + ], + ) + + logger.info( + "Voice token prepared: " + f"url={_mask_url(url)}, urls={len(urls)}, ice_servers={len(ice_servers)}, " + f"signal_proxy={'yes' if signal_proxy_url else 'no'}, " + f"participant_name={'yes' if participant_name else 'no'}, " + f"room_name={'yes' if room_name else 'no'}, token_len={len(token)}" + ) + return VoiceTokenResponse( token=token, - url="wss://livekit.grok.com", - participant_name="", - room_name="", + url=url, + urls=urls, + participant_name=participant_name, + room_name=room_name, + ice_servers=ice_servers or None, + signal_proxy_url=signal_proxy_url or None, ) except Exception as e: @@ -74,6 +334,88 @@ async def public_voice_token( ) +@router.websocket("/voice/signal") +@router.websocket("/voice/signal/{tail:path}") +async def public_voice_signal_proxy(websocket: WebSocket, tail: str = ""): + """同域信令代理:用于移动端无法直连 livekit.grok.com 的场景。""" + upstream_conn = None + await websocket.accept() + try: + upstream_url = _build_upstream_signal_url(websocket, tail) + ws_client = WebSocketClient() + upstream_conn = await ws_client.connect( + upstream_url, + headers=build_ws_headers(origin="https://grok.com"), + timeout=get_config("voice.timeout"), + ) + upstream_ws = upstream_conn.ws + logger.info(f"Voice signal proxy connected: {_mask_url(upstream_url)}") + + async def _client_to_upstream(): + while True: + message = await websocket.receive() + msg_type = message.get("type") + if msg_type == "websocket.disconnect": + break + if msg_type != "websocket.receive": + continue + text_data = message.get("text") + if text_data is not None: + await upstream_ws.send_str(text_data) + continue + bytes_data = message.get("bytes") + if bytes_data is not None: + await upstream_ws.send_bytes(bytes_data) + + async def _upstream_to_client(): + async for msg in upstream_ws: + if msg.type == aiohttp.WSMsgType.TEXT: + await websocket.send_text(msg.data) + elif msg.type == aiohttp.WSMsgType.BINARY: + await websocket.send_bytes(msg.data) + elif msg.type in ( + aiohttp.WSMsgType.CLOSE, + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.ERROR, + ): + break + + relay_tasks = [ + asyncio.create_task(_client_to_upstream()), + asyncio.create_task(_upstream_to_client()), + ] + done, pending = await asyncio.wait( + relay_tasks, return_when=asyncio.FIRST_COMPLETED + ) + + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + + for task in done: + exc = task.exception() + if exc: + raise exc + + except WebSocketDisconnect: + logger.debug("Voice signal proxy: client disconnected") + except Exception as e: + logger.warning(f"Voice signal proxy error: {e}") + finally: + if upstream_conn: + try: + await upstream_conn.close() + except Exception: + pass + try: + from starlette.websockets import WebSocketState + + if websocket.client_state == WebSocketState.CONNECTED: + await websocket.close(code=1000) + except Exception: + pass + + @router.get("/verify", dependencies=[Depends(verify_public_key)]) async def public_verify_api(): """验证 Public Key""" diff --git a/app/static/admin/js/config.js b/app/static/admin/js/config.js index e2978a6..490a894 100644 --- a/app/static/admin/js/config.js +++ b/app/static/admin/js/config.js @@ -116,7 +116,11 @@ const LOCALE_MAP = { "voice": { "label": "语音配置", - "timeout": { title: "请求超时", desc: "Voice 请求超时时间(秒)。" } + "timeout": { title: "请求超时", desc: "Voice 请求超时时间(秒)。" }, + "livekit_url": { title: "LiveKit 地址", desc: "优先使用的 LiveKit 信令地址,留空则使用上游返回值。" }, + "livekit_urls": { title: "LiveKit 地址列表", desc: "可配置多个信令地址,连接时会按顺序回退。" }, + "signal_proxy_enabled": { title: "Signal 代理", desc: "开启后返回同域 WebSocket 代理地址,移动端可优先尝试。" }, + "signal_proxy_url": { title: "代理地址", desc: "同域 signal 代理的自定义地址,留空将自动按当前域名生成。" } }, diff --git a/app/static/public/js/voice.js b/app/static/public/js/voice.js index 3da0a2a..b4017a9 100644 --- a/app/static/public/js/voice.js +++ b/app/static/public/js/voice.js @@ -5,6 +5,8 @@ let Track; let room = null; let visualizerTimer = null; + let isConnecting = false; + let suppressDisconnectLog = false; const startBtn = document.getElementById('startBtn'); const stopBtn = document.getElementById('stopBtn'); @@ -122,12 +124,269 @@ throw new Error(`当前环境不支持麦克风权限,${secureHint}`); } + function getLivekitHost(rawUrl) { + if (typeof rawUrl !== 'string' || !rawUrl.trim()) { + return '-'; + } + try { + return new URL(rawUrl).host || rawUrl; + } catch (err) { + return rawUrl; + } + } + + function normalizeLivekitUrl(rawUrl, fallback = '') { + if (typeof rawUrl !== 'string' || !rawUrl.trim()) { + return fallback; + } + const value = rawUrl.trim(); + const withProtocol = value.includes('://') ? value : `wss://${value}`; + try { + const u = new URL(withProtocol); + if (!u.protocol.startsWith('ws')) { + return fallback; + } + const path = (u.pathname || '').replace(/\/+$/, ''); + return `${u.protocol}//${u.host}${path}`; + } catch (err) { + return fallback; + } + } + + function normalizeLivekitUrls(rawUrls, fallbackUrl = 'wss://livekit.grok.com') { + const result = []; + const push = (value) => { + const normalized = normalizeLivekitUrl(value, ''); + if (!normalized || result.includes(normalized)) { + return; + } + result.push(normalized); + }; + + if (Array.isArray(rawUrls)) { + for (const value of rawUrls) { + push(value); + } + } else { + push(rawUrls); + } + + push(fallbackUrl); + if (!result.length) { + result.push('wss://livekit.grok.com'); + } + return result; + } + + function isSignalProxyUrl(rawUrl) { + try { + const parsed = new URL(rawUrl); + const path = (parsed.pathname || '').replace(/\/+$/, ''); + return path.endsWith('/voice/signal') || path.endsWith('/voice/signal/rtc'); + } catch (err) { + return false; + } + } + + function normalizeIceServers(raw) { + if (!Array.isArray(raw)) { + return []; + } + const normalized = []; + for (const item of raw) { + if (!item || typeof item !== 'object') { + continue; + } + const urlsRaw = item.urls || item.url; + let urls = []; + if (typeof urlsRaw === 'string') { + const value = urlsRaw.trim(); + if (value) { + urls = [value]; + } + } else if (Array.isArray(urlsRaw)) { + urls = urlsRaw + .filter((u) => typeof u === 'string') + .map((u) => u.trim()) + .filter((u) => u.length > 0); + } + if (!urls.length) { + continue; + } + + const entry = { urls }; + if (typeof item.username === 'string' && item.username.trim()) { + entry.username = item.username.trim(); + } + if (item.credential !== undefined && item.credential !== null) { + entry.credential = item.credential; + } + normalized.push(entry); + } + return normalized; + } + + function buildConnectOptions(iceServers, forceRelay = false) { + const options = { + autoSubscribe: true, + maxRetries: 1, + websocketTimeout: 30000, + peerConnectionTimeout: 20000 + }; + const rtcConfig = {}; + if (Array.isArray(iceServers) && iceServers.length > 0) { + rtcConfig.iceServers = iceServers; + } + if (forceRelay) { + rtcConfig.iceTransportPolicy = 'relay'; + } + if (Object.keys(rtcConfig).length > 0) { + options.rtcConfig = rtcConfig; + } + return options; + } + + function buildCandidateUrls(rawUrls) { + const seeds = Array.isArray(rawUrls) ? rawUrls : [rawUrls]; + const candidates = []; + const push = (u) => { + if (!u || candidates.includes(u)) { + return; + } + candidates.push(u); + }; + + for (const rawUrl of seeds) { + const main = normalizeLivekitUrl(rawUrl, '').replace(/\/+$/, ''); + if (!main) { + continue; + } + + push(main); + if (main.endsWith('/rtc')) { + push(main.slice(0, -4)); + } else { + push(`${main}/rtc`); + } + + // 某些移动网络对默认端口推断不稳定,追加 :443 变体做兜底。 + try { + const parsed = new URL(main); + if (parsed.protocol === 'wss:' && !parsed.port) { + const base443 = `${parsed.protocol}//${parsed.hostname}:443`; + const path = parsed.pathname && parsed.pathname !== '/' ? parsed.pathname : ''; + push(`${base443}${path}`); + if (path.endsWith('/rtc')) { + push(base443); + } else { + push(`${base443}/rtc`); + } + } + } catch (err) { + // ignore + } + } + if (!candidates.length) { + push('wss://livekit.grok.com'); + push('wss://livekit.grok.com/rtc'); + } + return candidates; + } + + function bindRoomEvents(targetRoom) { + targetRoom.on(RoomEvent.ParticipantConnected, (p) => log(`参与者已连接: ${p.identity}`)); + targetRoom.on(RoomEvent.ParticipantDisconnected, (p) => log(`参与者已断开: ${p.identity}`)); + targetRoom.on(RoomEvent.TrackSubscribed, (track) => { + log(`订阅音轨: ${track.kind}`); + if (track.kind === Track.Kind.Audio) { + const element = track.attach(); + element.autoplay = true; + element.playsInline = true; + if (audioRoot) { + audioRoot.appendChild(element); + } else { + document.body.appendChild(element); + } + } + }); + + targetRoom.on(RoomEvent.Disconnected, () => { + if (!suppressDisconnectLog) { + log('已断开连接'); + } + if (!isConnecting) { + resetUI(); + } + }); + } + + function createRoomInstance() { + const targetRoom = new Room({ + adaptiveStream: true, + dynacast: true + }); + bindRoomEvents(targetRoom); + return targetRoom; + } + + async function connectWithFallbacks(rawUrls, token, iceServers) { + const urlCandidates = buildCandidateUrls(rawUrls); + const strategies = [{ forceRelay: false, label: '默认' }]; + if (Array.isArray(iceServers) && iceServers.length > 0) { + strategies.push({ forceRelay: true, label: 'relay' }); + } + + let lastError = null; + let attempt = 0; + + suppressDisconnectLog = true; + try { + for (const candidateUrl of urlCandidates) { + for (const strategy of strategies) { + attempt += 1; + const connectOptions = buildConnectOptions(iceServers, strategy.forceRelay); + log(`尝试连接 #${attempt} (${strategy.label}): ${candidateUrl}`); + + room = createRoomInstance(); + try { + await room.connect(candidateUrl, token, connectOptions); + return { + usedUrl: candidateUrl, + usedProxy: isSignalProxyUrl(candidateUrl), + usedRelay: strategy.forceRelay, + attempts: attempt + }; + } catch (err) { + lastError = err; + const message = err && err.message ? err.message : String(err || ''); + log(`连接失败 #${attempt} (${strategy.label}): ${message}`, 'warn'); + try { + await room.disconnect(); + } catch (disconnectErr) { + // ignore + } + room = null; + } + } + } + } finally { + suppressDisconnectLog = false; + } + + if (lastError) { + throw lastError; + } + throw new Error('连接失败:未找到可用连接策略'); + } + async function startSession() { if (!ensureLiveKit()) { return; } + let localTracks = []; try { + isConnecting = true; const authHeader = await ensurePublicKey(); if (authHeader === null) { toast('请先配置 Public Key', 'error'); @@ -156,53 +415,73 @@ throw new Error(`获取 Token 失败: ${response.status}`); } - const { token, url } = await response.json(); + const payload = await response.json(); + const token = typeof payload.token === 'string' ? payload.token.trim() : ''; + const url = typeof payload.url === 'string' && payload.url.trim() + ? payload.url.trim() + : 'wss://livekit.grok.com'; + const urls = normalizeLivekitUrls(payload.urls, url); + const signalProxyUrl = normalizeLivekitUrl(payload.signal_proxy_url || '', ''); + const isMobile = /Android|webOS|iPhone|iPad|iPod|Mobile/i.test(navigator.userAgent || ''); + const connectUrlSeeds = []; + if (signalProxyUrl && isMobile) { + connectUrlSeeds.push(signalProxyUrl); + } + connectUrlSeeds.push(...urls); + if (signalProxyUrl && !isMobile) { + connectUrlSeeds.push(signalProxyUrl); + } + const iceServers = normalizeIceServers(payload.ice_servers); + const livekitHost = getLivekitHost(connectUrlSeeds[0] || url); + if (!token) { + throw new Error('服务端未返回有效 Token'); + } log(`获取 Token 成功 (${voiceSelect.value}, ${personalitySelect.value}, ${speedRange.value}x)`); + log(`连接服务器: ${livekitHost}${iceServers.length ? `, ICE=${iceServers.length}` : ''}`); + if (signalProxyUrl) { + log(`signal 代理已就绪: ${signalProxyUrl}${isMobile ? ' (移动端优先)' : ' (回退使用)'}`); + } - room = new Room({ - adaptiveStream: true, - dynacast: true - }); + log('正在请求麦克风权限...'); + ensureMicSupport(); + localTracks = await createLocalTracks({ audio: true, video: false }); + log('麦克风权限已授权'); - room.on(RoomEvent.ParticipantConnected, (p) => log(`参与者已连接: ${p.identity}`)); - room.on(RoomEvent.ParticipantDisconnected, (p) => log(`参与者已断开: ${p.identity}`)); - room.on(RoomEvent.TrackSubscribed, (track) => { - log(`订阅音轨: ${track.kind}`); - if (track.kind === Track.Kind.Audio) { - const element = track.attach(); - if (audioRoot) { - audioRoot.appendChild(element); - } else { - document.body.appendChild(element); - } - } - }); - - room.on(RoomEvent.Disconnected, () => { - log('已断开连接'); - resetUI(); - }); - - await room.connect(url, token); - log('已连接到 LiveKit 服务器'); + const connectResult = await connectWithFallbacks(connectUrlSeeds, token, iceServers); + if (connectResult.usedRelay) { + log('relay 模式连接成功', 'warn'); + } + if (connectResult.usedProxy) { + log('通过同域 signal 代理连接成功', 'warn'); + } + log(`已连接到 LiveKit 服务器(尝试 ${connectResult.attempts} 次)`); setStatus('connected', '通话中'); setButtons(true); - log('正在开启麦克风...'); - ensureMicSupport(); - const tracks = await createLocalTracks({ audio: true, video: false }); - for (const track of tracks) { + for (const track of localTracks) { await room.localParticipant.publishTrack(track); } log('语音已开启'); toast('语音连接成功', 'success'); } catch (err) { + for (const track of localTracks) { + try { + track.stop(); + } catch (stopErr) { + // ignore + } + } const message = err && err.message ? err.message : '连接失败'; log(`错误: ${message}`, 'error'); + if (/could not establish pc connection/i.test(message)) { + log('提示: 该错误通常与网络环境或 ICE/STUN/TURN 配置相关', 'warn'); + } toast(message, 'error'); setStatus('error', '连接错误'); startBtn.disabled = false; + } finally { + isConnecting = false; } } diff --git a/app/static/public/pages/voice.html b/app/static/public/pages/voice.html index e9fe5cc..38e3fdf 100644 --- a/app/static/public/pages/voice.html +++ b/app/static/public/pages/voice.html @@ -3,24 +3,32 @@
- +