From 37108828c3fd76dabd2d668cb64c3fb4420669c3 Mon Sep 17 00:00:00 2001 From: XuanRan_Nyan <31968665+xxxxuanran@users.noreply.github.com> Date: Wed, 10 Apr 2024 23:04:45 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=8A=9F=E8=83=BD=20(#860)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 调整功能 * 修正错误的健康检查 * 文件列表黑名单 * "/bili/space/myinfo"接口错误信息优化 * 在 302 之前替换 * 日志编码指定utf-8 * 在 Tags 处支持 {streamer} --- app/history/page.tsx | 2 +- app/job/page.tsx | 20 +++++++++++++++----- app/ui/UserList.tsx | 3 ++- app/ui/plugins/bilibili.tsx | 28 +++++++++++++++++++++++----- app/ui/plugins/global.tsx | 4 +++- biliup/__init__.py | 3 ++- biliup/engine/download.py | 24 ++++++++++++++++-------- biliup/engine/event.py | 2 +- biliup/handler.py | 9 ++++----- biliup/plugins/bili_webup.py | 2 ++ biliup/plugins/bilibili.py | 34 +++++++++++++++++++++------------- biliup/plugins/biliuprs.py | 2 ++ biliup/plugins/egame.py | 34 ---------------------------------- biliup/web/__init__.py | 8 ++++++-- 14 files changed, 98 insertions(+), 77 deletions(-) delete mode 100644 biliup/plugins/egame.py diff --git a/app/history/page.tsx b/app/history/page.tsx index 8cdbf32..a2a5f8d 100644 --- a/app/history/page.tsx +++ b/app/history/page.tsx @@ -84,7 +84,7 @@ export default function Home() { display: 'flex', // justifyContent: 'center', padding: '6px' - }}>

录播管理

} + }}>

历史记录

} mode="horizontal" > diff --git a/app/job/page.tsx b/app/job/page.tsx index deeb280..2761288 100644 --- a/app/job/page.tsx +++ b/app/job/page.tsx @@ -1,6 +1,7 @@ 'use client' -import {AutoComplete, Layout, Nav, Spin, Table, Typography} from "@douyinfe/semi-ui"; +import { AutoComplete, Layout, Nav, Spin, Table, Typography } from "@douyinfe/semi-ui"; +import { SortOrder } from "@douyinfe/semi-ui/lib/es/table/interface"; import useSWR from "swr"; import {fetcher, FileList} from "@/app/lib/api-streamer"; import { @@ -17,7 +18,8 @@ export default function Home() { if (isLoading) { return (); } - const {Text} = Typography; + const { Text } = Typography; + const ascendSort: SortOrder = "ascend"; const columns = [ { title: '名称', @@ -47,6 +49,7 @@ export default function Home() { { title: '更新日期', dataIndex: 'date', + defaultSortOrder: ascendSort, render: (text: any, record: any, index: any) => { return (<>{humDate(text)}); }, @@ -54,9 +57,16 @@ export default function Home() { }, ]; const expandRowRender = (record: any, index: number | undefined) => { - return <>文件列表：{record.files.map((it: { id: Key | null | undefined; file: string | number | boolean | ReactElement> | Iterable | ReactPortal | null | undefined; }) => { - return
  文件名：{it.file}
; - })}; + return <>文件列表：{record.files.map( + (it: + { + id: Key | null | undefined; + file: string | number | boolean | ReactElement> | Iterable | ReactPortal | null | undefined; + } + ) => { + return
  文件名：{it.file}
; + } + )}; }; return (<>
diff --git a/app/ui/UserList.tsx b/app/ui/UserList.tsx index fdae0ca..afcd4c4 100644 --- a/app/ui/UserList.tsx +++ b/app/ui/UserList.tsx @@ -42,9 +42,10 @@ const UserList: React.FC = ({onCancel, visible}) => { setVisible(false); Toast.success('创建成功'); } catch (e: any) { + const messageObj = JSON.parse(e.message); return Notification.error({ title: '创建失败', - content: {e.message}, + content: {messageObj.error}, // theme: 'light', // duration: 0, style: {width: 'min-content'} diff --git a/app/ui/plugins/bilibili.tsx b/app/ui/plugins/bilibili.tsx index 59bc105..6e3a58a 100644 --- a/app/ui/plugins/bilibili.tsx +++ b/app/ui/plugins/bilibili.tsx @@ -112,7 +112,7 @@ const Bilibili: React.FC = (props) => { field="bili_fallback_api" extraText={
- 上方的主要API不可用时，回退使用的 API。默认使用官方 API。 + 上方的主要 API 不可用或受到区域限制时，回退使用的 API。默认使用官方 API。
海外机器玩法：哔哩哔哩直播API（bili_liveapi）设置为能获取大陆直播流的 API，并将哔哩哔哩直播回退API（bili_fallback_api）设置为官方 @@ -130,10 +130,10 @@ const Bilibili: React.FC = (props) => { padding: 0, }} /> - = (props) => { padding: 0, }} /> + + ); diff --git a/app/ui/plugins/global.tsx b/app/ui/plugins/global.tsx index c2d7d51..224fa57 100644 --- a/app/ui/plugins/global.tsx +++ b/app/ui/plugins/global.tsx @@ -225,13 +225,15 @@ const Global: React.FC = () => { + AUTO（自动，默认） bda2 kodo ws diff --git a/biliup/__init__.py b/biliup/__init__.py index 020ce07..785fd45 100644 --- a/biliup/__init__.py +++ b/biliup/__init__.py @@ -30,7 +30,8 @@ LOG_CONF = { 'interval': 1, 'backupCount': 1, 'filename': 'ds_update.log', - 'formatter': 'verbose' + 'formatter': 'verbose', + 'encoding': 'utf-8' } }, 'root': { diff --git a/biliup/engine/download.py b/biliup/engine/download.py index af255c1..a5cc1ae 100644 --- a/biliup/engine/download.py +++ b/biliup/engine/download.py @@ -4,6 +4,7 @@ import re import subprocess import sys import time +import shutil from typing import Generator, List from urllib.parse import urlparse @@ -108,13 +109,17 @@ class DownloadBase: self.danmaku_download_start(fmtname) parsed_url_path = urlparse(self.raw_stream_url).path - if self.downloader == 'streamlink': - if '.flv' in parsed_url_path: # streamlink无法处理flv,所以回退到ffmpeg + if shutil.which("ffmpeg"): + if self.downloader == 'streamlink': + if '.flv' in parsed_url_path: # streamlink无法处理flv,所以回退到ffmpeg + return self.ffmpeg_download(fmtname) + else: + return self.streamlink_download(fmtname) + elif self.downloader == 'ffmpeg': return self.ffmpeg_download(fmtname) - else: - return self.streamlink_download(fmtname) - elif self.downloader == 'ffmpeg': - return self.ffmpeg_download(fmtname) + else: + logger.error("未安装 FFMpeg 或不存在于 PATH 内，本次下载使用 stream-gears") + logger.debug("Current user's PATH is:" + os.getenv("PATH")) if not self.suffix in ['flv', 'ts']: self.suffix = 'flv' if '.flv' in parsed_url_path else 'ts' @@ -338,8 +343,11 @@ class DownloadBase: logger.exception(f'封面下载失败：{self.__class__.__name__} - {self.fname}') def check_url_healthy(self, http_session, url): + timeout = 5 try: - r = http_session.get(url, stream=True, timeout=5, allow_redirects=False) + if 'flv' in url: + timeout = 60 + r = http_session.get(url, stream=True, timeout=timeout, allow_redirects=False) if 'm3u8' in url: import m3u8 m3u8_obj = m3u8.loads(r.text) @@ -350,7 +358,7 @@ class DownloadBase: elif r.headers.get('Location', False): url = r.headers['Location'] logger.info(f'stream url: {url}') - r = http_session.get(url, stream=True, timeout=5) + r = http_session.get(url, stream=True, timeout=timeout) if r.status_code == 200: return True, url except Exception as e: diff --git a/biliup/engine/event.py b/biliup/engine/event.py index bfeaabc..50766d0 100644 --- a/biliup/engine/event.py +++ b/biliup/engine/event.py @@ -167,4 +167,4 @@ class Event: """事件对象""" type_: str # 事件类型 args: tuple = () - dict: dict = field(default_factory=dict) # 字典用于保存具体的事件数据 + dict: dict = field(default_factory=dict) # type: ignore # 字典用于保存具体的事件数据 diff --git a/biliup/handler.py b/biliup/handler.py index 4f86573..6fe25e3 100644 --- a/biliup/handler.py +++ b/biliup/handler.py @@ -39,12 +39,12 @@ def singleton_check(platform, name, url): yield Event(PRE_DOWNLOAD, args=(name, turl,)) return if context['PluginInfo'].url_status[url] == 1: - logger.debug(f'{url}正在下载中，跳过检测') + logger.debug(f'{url} 正在下载中，跳过检测') return # 可能对同一个url同时发送两次上传事件 with NamedLock(f"upload_count_{url}"): if context['url_upload_count'][url] > 0: - logger.debug(f'{url}正在上传中，跳过') + logger.debug(f'{url} 正在上传中，跳过') return # from .handler import event_manager, UPLOAD # += 不是原子操作 @@ -59,9 +59,9 @@ def singleton_check(platform, name, url): @event_manager.register(PRE_DOWNLOAD, block='Asynchronous1') def pre_processor(name, url): if context['PluginInfo'].url_status[url] == 1: - logger.debug(f'{name}-正在下载中，跳过下载') + logger.debug(f'{name} 正在下载中，跳过下载') return - logger.info(f'{name}-{url}-开播了准备下载') + logger.info(f'{name} - {url} 开播了准备下载') preprocessor = config['streamers'].get(name, {}).get('preprocessor') if preprocessor: processor(preprocessor, json.dumps({ @@ -94,7 +94,6 @@ def process(name, url): finally: # 下载结束 url_status[url] = 0 - @event_manager.register(DOWNLOADED, block='Asynchronous1') def processed(stream_info): diff --git a/biliup/plugins/bili_webup.py b/biliup/plugins/bili_webup.py index f1ae2ee..6d2db54 100644 --- a/biliup/plugins/bili_webup.py +++ b/biliup/plugins/bili_webup.py @@ -35,6 +35,8 @@ class BiliWeb(UploadBase): super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor) if tags is None: tags = [] + else: + tags = [str(tag).format(streamer=self.data['name']) for tag in tags] self.user = user self.lines = lines self.submit_api = submit_api diff --git a/biliup/plugins/bilibili.py b/biliup/plugins/bilibili.py index 960bc99..ffd5d1a 100644 --- a/biliup/plugins/bilibili.py +++ b/biliup/plugins/bilibili.py @@ -1,6 +1,7 @@ import time import requests import json +import re from biliup.config import config from . import match1, logger @@ -54,11 +55,10 @@ class Bililive(DownloadBase): _res = do_login(s) user_data = _res.get('data', {}) is_login = user_data.get('isLogin', False) - if not is_login: - # logger.warning(f"{plugin_msg}: 登录态校验失败") - logger.warning("登录态校验失败: " + str(_res)) - else: + if is_login: logger.info(f"用户名：{user_data['uname']}, mid：{user_data['mid']}, isLogin：{is_login}") + else: + logger.warning("登录态校验失败: " + str(_res)) return True protocol = config.get('bili_protocol', 'stream') @@ -68,6 +68,8 @@ class Bililive(DownloadBase): ov05_ip = config.get('bili_force_ov05_ip') main_api = config.get('bili_liveapi', OFFICIAL_API).rstrip('/') fallback_api = config.get('bili_fallback_api', OFFICIAL_API).rstrip('/') + cn01_sids = config.get('bili_replace_cn01_sid', '').split(',') + normalize_cn204 = config.get('bili_normalize_cn204', False) params = { 'room_id': room_id, @@ -86,13 +88,14 @@ class Bililive(DownloadBase): if self.raw_stream_url is not None \ and qualityNumber >= 10000 \ and not is_new_live: - # 同一个 streamName 即可复用，与其他参数无关，目前没有报告说链接会超时失效。 + # 同一个 streamName 即可复用，除非被超管切断 # 前面拿不到 streamName，目前使用开播时间判断 health, url = super().check_url_healthy(s, self.raw_stream_url) if health: logger.debug(f"{plugin_msg}: 复用 {url}") return True else: + logger.debug(f"{plugin_msg}: health-{health}; url-{self.raw_stream_url}") self.raw_stream_url = None try: @@ -146,16 +149,19 @@ class Bililive(DownloadBase): streamName = match1(stream_url['base_url'], streamname_regexp) if streamName is not None and force_source and qualityNumber >= 10000: new_base_url = stream_url['base_url'].replace(f"_{streamName.split('_')[-1]}", '') - if super().check_url_healthy(s, f"{stream_url['host']}{new_base_url}{stream_url['extra']}"): + if super().check_url_healthy(s, f"{stream_url['host']}{new_base_url}{stream_url['extra']}")[0]: stream_url['base_url'] = new_base_url logger.debug(stream_url['base_url']) self.raw_stream_url = f"{stream_url['host']}{stream_url['base_url']}{stream_url['extra']}" + if normalize_cn204: + self.raw_stream_url = re.sub(r"(?<=cn-gotcha204)-[1-4]", "", self.raw_stream_url, 1) + if ov05_ip and "ov-gotcha05" in stream_url['host']: self.raw_stream_url = oversea_expand(s, self.raw_stream_url, ov05_ip) - url_health, url = super().check_url_healthy(s, self.raw_stream_url) + url_health, _url = super().check_url_healthy(s, self.raw_stream_url) if not url_health: if cdn_fallback: i = len(stream_info['url_info']) @@ -163,12 +169,15 @@ class Bililive(DownloadBase): i -= 1 try: self.raw_stream_url = "{}{}{}".format(stream_info['url_info'][i]['host'], stream_url['base_url'], stream_info['url_info'][i]['extra']) - url_health, url = super().check_url_healthy(s, self.raw_stream_url) + url_health, _url = super().check_url_healthy(s, self.raw_stream_url) if url_health: - self.raw_stream_url = url + self.raw_stream_url = _url break - except Exception as e: - logger.debug(e) + except TimeoutError as e: + logger.debug(f"Timeout Error: {e} at {stream_info['url_info'][i]['host']}") + continue + except Exception: + logger.exception("Uncaught exception:") continue finally: logger.debug(f"{i} - {self.raw_stream_url}") @@ -177,7 +186,7 @@ class Bililive(DownloadBase): self.raw_stream_url = None return False else: - self.raw_stream_url = url + self.raw_stream_url = _url return True @@ -219,7 +228,6 @@ def do_login(s): def oversea_expand(s, url, ov05_ip): # 强制替换ov05 302redirect之后的真实地址为指定的域名或ip达到自选ov05节点的目的 - import re r = s.get(url, stream=True) logger.debug(f'将ov-gotcha05的节点ip替换为了{ov05_ip}') return re.sub(r".*(?=/d1--ov-gotcha05)", f"http://{ov05_ip}", r.url, 1) diff --git a/biliup/plugins/biliuprs.py b/biliup/plugins/biliuprs.py index 37a0032..c17ec3d 100644 --- a/biliup/plugins/biliuprs.py +++ b/biliup/plugins/biliuprs.py @@ -19,6 +19,8 @@ class BiliWeb(UploadBase): super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor) if tags is None: tags = [] + else: + tags = [str(tag).format(streamer=self.data['name']) for tag in tags] self.lines = lines self.submit_api = submit_api self.threads = threads diff --git a/biliup/plugins/egame.py b/biliup/plugins/egame.py deleted file mode 100644 index 3349417..0000000 --- a/biliup/plugins/egame.py +++ /dev/null @@ -1,34 +0,0 @@ -import requests -from . import logger -from ..engine.decorators import Plugin -from ..engine.download import DownloadBase - - -@Plugin.download(regexp=r'(?:https?://)?(?:egame\.)?qq\.com') -class egame(DownloadBase): - def __init__(self, fname, url, suffix='flv'): - super().__init__(fname, url, suffix) - - def check_stream(self, is_check=False): - if len(self.url.split("egame.qq.com/")) < 2: - logger.debug("直播间格式错误") - return False - rid = self.url.split("egame.qq.com/")[1] - url = 'https://share.egame.qq.com/cgi-bin/pgg_async_fcgi' - data = {'param': '''{"0":{"module":"pgg_live_read_svr","method":"get_live_and_profile_info","param":{ - "anchor_id":''' + str(rid) + ''',"layout_id":"hot","index":1,"other_uid":0}}}'''} - r = requests.post(url=url, data=data, timeout=5).json() - if r['ecode'] != 0: - logger.debug("直播间地址错误") - return False - pid = r['data']["0"]["retBody"]["data"]["video_info"]["pid"] - if pid == "": - logger.debug("直播间不存在") - return False - is_live = r['data']["0"]["retBody"]["data"]["profile_info"]["is_live"] - if is_live != 1: - logger.debug("主播未开播") - return False - self.raw_stream_url = r['data']["0"]["retBody"]["data"]["video_info"]["stream_infos"][0]["play_url"] - self.room_title = r['data']["0"]["retBody"]['data']['video_info']['title'] - return True diff --git a/biliup/web/__init__.py b/biliup/web/__init__.py index 13fdfd1..cf27ef8 100644 --- a/biliup/web/__init__.py +++ b/biliup/web/__init__.py @@ -171,10 +171,13 @@ async def tag_check(request): @routes.get('/v1/videos') async def streamers(request): media_extensions = ['.mp4', '.flv', '.3gp', '.webm', '.mkv', '.ts'] + _blacklist = ['next-env.d.ts'] # 获取文件列表 file_list = [] i = 1 for file_name in os.listdir('.'): + if file_name in _blacklist: + continue name, ext = os.path.splitext(file_name) if ext in media_extensions: file_list.append({'key': i, 'name': file_name, 'updateTime': os.path.getmtime(file_name), @@ -445,7 +448,8 @@ async def app_status(request): from biliup.app import context from biliup.config import Config from biliup.app import PluginInfo - res = {} + from biliup import __version__ + res = {'version': __version__,} for key, value in context.items(): # 遍历删除不能被 json 序列化的键值对 if isinstance(value, Config): continue @@ -481,7 +485,7 @@ async def myinfo(request): try: config.load_cookies(file) except FileNotFoundError: - return web.json_response({"status": 500, 'error': f"找不到 {file} ！！！"}, status=500) + return web.json_response({"status": 500, 'error': f"{file} 文件不存在"}, status=500) cookies = config.data['user']['cookies'] return web.json_response(BiliBili.myinfo(cookies))