okooooooo/biliup
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整功能 (#860)

Browse Source 
* 调整功能

* 修正错误的健康检查

* 文件列表黑名单

* "/bili/space/myinfo"接口错误信息优化

* 在 302 之前替换

* 日志编码指定utf-8

* 在 Tags 处支持 {streamer}
This commit is contained in:
XuanRan_Nyan authored and GitHub committed 2024-04-10 15:04:45 +00:00
1 parent 0e9eeea8d1
commit 37108828c3
14 files changed
+98 -77
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/history/page.tsx
+1 -1
View File
@@ -84,7 +84,7 @@ export default function Home() {
display: 'flex',
// justifyContent: 'center',
padding: '6px'
}}><IconVideoListStroked size='large' /></div><h4 style={{ marginLeft: '12px' }}>录播管理</h4></>}
}}><IconVideoListStroked size='large' /></div><h4 style={{ marginLeft: '12px' }}>历史记录</h4></>}
mode="horizontal"
></Nav>
</Header>
app/job/page.tsx
+15 -5
View File
@@ -1,6 +1,7 @@
'use client'
import {AutoComplete, Layout, Nav, Spin, Table, Typography} from "@douyinfe/semi-ui";
import { AutoComplete, Layout, Nav, Spin, Table, Typography } from "@douyinfe/semi-ui";
import { SortOrder } from "@douyinfe/semi-ui/lib/es/table/interface";
import useSWR from "swr";
import {fetcher, FileList} from "@/app/lib/api-streamer";
import {
@@ -17,7 +18,8 @@ export default function Home() {
if (isLoading) {
return (<Spin size="large" />);
}
const {Text} = Typography;
const { Text } = Typography;
const ascendSort: SortOrder = "ascend";
const columns = [
{
title: '名称',
@@ -47,6 +49,7 @@ export default function Home() {
{
title: '更新日期',
dataIndex: 'date',
defaultSortOrder: ascendSort,
render: (text: any, record: any, index: any) => {
return (<>{humDate(text)}</>);
},
@@ -54,9 +57,16 @@ export default function Home() {
},
];
const expandRowRender = (record: any, index: number | undefined) => {
return <>文件列表：{record.files.map((it: { id: Key | null | undefined; file: string | number | boolean | ReactElement<any, string | JSXElementConstructor<any>> | Iterable<ReactNode> | ReactPortal | null | undefined; }) => {
return <div key={it.id}>&nbsp;&nbsp;文件名：{it.file}</div>;
})}</>;
return <>文件列表：{record.files.map(
(it:
{
id: Key | null | undefined;
file: string | number | boolean | ReactElement<any, string | JSXElementConstructor<any>> | Iterable<ReactNode> | ReactPortal | null | undefined;
}
) => {
return <div key={it.id}>&nbsp;&nbsp;文件名：{it.file}</div>;
}
)}</>;
};
return (<>
<Header style={{backgroundColor: 'var(--semi-color-bg-1)'}}>
app/ui/UserList.tsx
+2 -1
View File
@@ -42,9 +42,10 @@ const UserList: React.FC<UserListProps> = ({onCancel, visible}) => {
setVisible(false);
Toast.success('创建成功');
} catch (e: any) {
const messageObj = JSON.parse(e.message);
return Notification.error({
title: '创建失败',
content: <Typography.Paragraph style={{maxWidth: 450}}>{e.message}</Typography.Paragraph>,
content: <Typography.Paragraph style={{ maxWidth: 450 }}>{messageObj.error}</Typography.Paragraph>,
// theme: 'light',
// duration: 0,
style: {width: 'min-content'}
app/ui/plugins/bilibili.tsx
+23 -5
View File
@@ -112,7 +112,7 @@ const Bilibili: React.FC<Props> = (props) => {
field="bili_fallback_api"
extraText={
<div style={{ fontSize: "14px" }}>
上方的主要API不可用时，回退使用的 API。默认使用官方 API。
上方的主要 API 不可用或受到区域限制时，回退使用的 API。默认使用官方 API。
<br />
海外机器玩法：哔哩哔哩直播API（bili_liveapi）设置为能获取大陆直播流的
API，并将哔哩哔哩直播回退API（bili_fallback_api）设置为官方
@@ -130,10 +130,10 @@ const Bilibili: React.FC<Props> = (props) => {
padding: 0,
}}
/>
<Form.Switch
field="bili_force_source"
extraText="哔哩哔哩强制真原画（仅限非 FLV 流，且画质等级 bili_qn >= 10000），默认为关闭，不保证可用性。"
label="强制获取真原画（bili_force_source）"
<Form.Input
field="bili_replace_cn01_sid"
label="替换 CN01 sid (bili_replace_cn01_sid)"
style={{ width: "100%" }}
fieldStyle={{
alignSelf: "stretch",
padding: 0,
@@ -148,6 +148,24 @@ const Bilibili: React.FC<Props> = (props) => {
padding: 0,
}}
/>
<Form.Switch
field="bili_force_source"
extraText="哔哩哔哩强制真原画（仅限非 FLV 流，且画质等级 bili_qn >= 10000），默认为关闭，不保证可用性。"
label="强制获取真原画（bili_force_source）"
fieldStyle={{
alignSelf: "stretch",
padding: 0,
}}
/>
<Form.Switch
field="bili_normalize_cn204"
extraText="去除 cn-gotcha204 后面的小尾巴（-[1-4]）"
label="标准化 CN204（bili_normalize_cn204）"
fieldStyle={{
alignSelf: "stretch",
padding: 0,
}}
/>
</Collapse.Panel>
</>
);
app/ui/plugins/global.tsx
+3 -1
View File
@@ -225,13 +225,15 @@ const Global: React.FC = () => {
<Form.Select
field="lines"
label="上传线路（lines）"
extraText="b站上传线路选择，默认为自动模式，目前可手动切换为bda2, kodo, ws, qn"
extraText="b站上传线路选择，默认为自动模式，可手动切换为bda2, kodo, ws, qn"
placeholder="AUTO（自动，默认）"
style={{ width: "100%" }}
fieldStyle={{
alignSelf: "stretch",
padding: 0,
}}
>
<Form.Select.Option value="AUTO">AUTO（自动，默认）</Form.Select.Option>
<Form.Select.Option value="bda2">bda2</Form.Select.Option>
<Form.Select.Option value="kodo">kodo</Form.Select.Option>
<Form.Select.Option value="ws">ws</Form.Select.Option>
biliup/__init__.py
+2 -1
View File
@@ -30,7 +30,8 @@ LOG_CONF = {
'interval': 1,
'backupCount': 1,
'filename': 'ds_update.log',
'formatter': 'verbose'
'formatter': 'verbose',
'encoding': 'utf-8'
}
},
'root': {
biliup/engine/download.py
+16 -8
View File
@@ -4,6 +4,7 @@ import re
import subprocess
import sys
import time
import shutil
from typing import Generator, List
from urllib.parse import urlparse
@@ -108,13 +109,17 @@ class DownloadBase:
self.danmaku_download_start(fmtname)
parsed_url_path = urlparse(self.raw_stream_url).path
if self.downloader == 'streamlink':
if '.flv' in parsed_url_path: # streamlink无法处理flv,所以回退到ffmpeg
if shutil.which("ffmpeg"):
if self.downloader == 'streamlink':
if '.flv' in parsed_url_path: # streamlink无法处理flv,所以回退到ffmpeg
return self.ffmpeg_download(fmtname)
else:
return self.streamlink_download(fmtname)
elif self.downloader == 'ffmpeg':
return self.ffmpeg_download(fmtname)
else:
return self.streamlink_download(fmtname)
elif self.downloader == 'ffmpeg':
return self.ffmpeg_download(fmtname)
else:
logger.error("未安装 FFMpeg 或不存在于 PATH 内，本次下载使用 stream-gears")
logger.debug("Current user's PATH is:" + os.getenv("PATH"))
if not self.suffix in ['flv', 'ts']:
self.suffix = 'flv' if '.flv' in parsed_url_path else 'ts'
@@ -338,8 +343,11 @@ class DownloadBase:
logger.exception(f'封面下载失败：{self.__class__.__name__} - {self.fname}')
def check_url_healthy(self, http_session, url):
timeout = 5
try:
r = http_session.get(url, stream=True, timeout=5, allow_redirects=False)
if 'flv' in url:
timeout = 60
r = http_session.get(url, stream=True, timeout=timeout, allow_redirects=False)
if 'm3u8' in url:
import m3u8
m3u8_obj = m3u8.loads(r.text)
@@ -350,7 +358,7 @@ class DownloadBase:
elif r.headers.get('Location', False):
url = r.headers['Location']
logger.info(f'stream url: {url}')
r = http_session.get(url, stream=True, timeout=5)
r = http_session.get(url, stream=True, timeout=timeout)
if r.status_code == 200:
return True, url
except Exception as e:
biliup/engine/event.py
+1 -1
View File
@@ -167,4 +167,4 @@ class Event:
"""事件对象"""
type_: str # 事件类型
args: tuple = ()
dict: dict = field(default_factory=dict) # 字典用于保存具体的事件数据
dict: dict = field(default_factory=dict) # type: ignore # 字典用于保存具体的事件数据
biliup/handler.py
+4 -5
View File
@@ -39,12 +39,12 @@ def singleton_check(platform, name, url):
yield Event(PRE_DOWNLOAD, args=(name, turl,))
return
if context['PluginInfo'].url_status[url] == 1:
logger.debug(f'{url}正在下载中，跳过检测')
logger.debug(f'{url} 正在下载中，跳过检测')
return
# 可能对同一个url同时发送两次上传事件
with NamedLock(f"upload_count_{url}"):
if context['url_upload_count'][url] > 0:
logger.debug(f'{url}正在上传中，跳过')
logger.debug(f'{url} 正在上传中，跳过')
return
# from .handler import event_manager, UPLOAD
# += 不是原子操作
@@ -59,9 +59,9 @@ def singleton_check(platform, name, url):
@event_manager.register(PRE_DOWNLOAD, block='Asynchronous1')
def pre_processor(name, url):
if context['PluginInfo'].url_status[url] == 1:
logger.debug(f'{name}-正在下载中，跳过下载')
logger.debug(f'{name} 正在下载中，跳过下载')
return
logger.info(f'{name}-{url}-开播了准备下载')
logger.info(f'{name} - {url} 开播了准备下载')
preprocessor = config['streamers'].get(name, {}).get('preprocessor')
if preprocessor:
processor(preprocessor, json.dumps({
@@ -94,7 +94,6 @@ def process(name, url):
finally:
# 下载结束
url_status[url] = 0
@event_manager.register(DOWNLOADED, block='Asynchronous1')
def processed(stream_info):
biliup/plugins/bili_webup.py
+2
View File
@@ -35,6 +35,8 @@ class BiliWeb(UploadBase):
super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor)
if tags is None:
tags = []
else:
tags = [str(tag).format(streamer=self.data['name']) for tag in tags]
self.user = user
self.lines = lines
self.submit_api = submit_api
biliup/plugins/bilibili.py
+21 -13
View File
@@ -1,6 +1,7 @@
import time
import requests
import json
import re
from biliup.config import config
from . import match1, logger
@@ -54,11 +55,10 @@ class Bililive(DownloadBase):
_res = do_login(s)
user_data = _res.get('data', {})
is_login = user_data.get('isLogin', False)
if not is_login:
# logger.warning(f"{plugin_msg}: 登录态校验失败")
logger.warning("登录态校验失败: " + str(_res))
else:
if is_login:
logger.info(f"用户名：{user_data['uname']}, mid：{user_data['mid']}, isLogin：{is_login}")
else:
logger.warning("登录态校验失败: " + str(_res))
return True
protocol = config.get('bili_protocol', 'stream')
@@ -68,6 +68,8 @@ class Bililive(DownloadBase):
ov05_ip = config.get('bili_force_ov05_ip')
main_api = config.get('bili_liveapi', OFFICIAL_API).rstrip('/')
fallback_api = config.get('bili_fallback_api', OFFICIAL_API).rstrip('/')
cn01_sids = config.get('bili_replace_cn01_sid', '').split(',')
normalize_cn204 = config.get('bili_normalize_cn204', False)
params = {
'room_id': room_id,
@@ -86,13 +88,14 @@ class Bililive(DownloadBase):
if self.raw_stream_url is not None \
and qualityNumber >= 10000 \
and not is_new_live:
# 同一个 streamName 即可复用，与其他参数无关，目前没有报告说链接会超时失效。
# 同一个 streamName 即可复用，除非被超管切断
# 前面拿不到 streamName，目前使用开播时间判断
health, url = super().check_url_healthy(s, self.raw_stream_url)
if health:
logger.debug(f"{plugin_msg}: 复用 {url}")
return True
else:
logger.debug(f"{plugin_msg}: health-{health}; url-{self.raw_stream_url}")
self.raw_stream_url = None
try:
@@ -146,16 +149,19 @@ class Bililive(DownloadBase):
streamName = match1(stream_url['base_url'], streamname_regexp)
if streamName is not None and force_source and qualityNumber >= 10000:
new_base_url = stream_url['base_url'].replace(f"_{streamName.split('_')[-1]}", '')
if super().check_url_healthy(s, f"{stream_url['host']}{new_base_url}{stream_url['extra']}"):
if super().check_url_healthy(s, f"{stream_url['host']}{new_base_url}{stream_url['extra']}")[0]:
stream_url['base_url'] = new_base_url
logger.debug(stream_url['base_url'])
self.raw_stream_url = f"{stream_url['host']}{stream_url['base_url']}{stream_url['extra']}"
if normalize_cn204:
self.raw_stream_url = re.sub(r"(?<=cn-gotcha204)-[1-4]", "", self.raw_stream_url, 1)
if ov05_ip and "ov-gotcha05" in stream_url['host']:
self.raw_stream_url = oversea_expand(s, self.raw_stream_url, ov05_ip)
url_health, url = super().check_url_healthy(s, self.raw_stream_url)
url_health, _url = super().check_url_healthy(s, self.raw_stream_url)
if not url_health:
if cdn_fallback:
i = len(stream_info['url_info'])
@@ -163,12 +169,15 @@ class Bililive(DownloadBase):
i -= 1
try:
self.raw_stream_url = "{}{}{}".format(stream_info['url_info'][i]['host'], stream_url['base_url'], stream_info['url_info'][i]['extra'])
url_health, url = super().check_url_healthy(s, self.raw_stream_url)
url_health, _url = super().check_url_healthy(s, self.raw_stream_url)
if url_health:
self.raw_stream_url = url
self.raw_stream_url = _url
break
except Exception as e:
logger.debug(e)
except TimeoutError as e:
logger.debug(f"Timeout Error: {e} at {stream_info['url_info'][i]['host']}")
continue
except Exception:
logger.exception("Uncaught exception:")
continue
finally:
logger.debug(f"{i} - {self.raw_stream_url}")
@@ -177,7 +186,7 @@ class Bililive(DownloadBase):
self.raw_stream_url = None
return False
else:
self.raw_stream_url = url
self.raw_stream_url = _url
return True
@@ -219,7 +228,6 @@ def do_login(s):
def oversea_expand(s, url, ov05_ip):
# 强制替换ov05 302redirect之后的真实地址为指定的域名或ip达到自选ov05节点的目的
import re
r = s.get(url, stream=True)
logger.debug(f'将ov-gotcha05的节点ip替换为了{ov05_ip}')
return re.sub(r".*(?=/d1--ov-gotcha05)", f"http://{ov05_ip}", r.url, 1)
biliup/plugins/biliuprs.py
+2
View File
@@ -19,6 +19,8 @@ class BiliWeb(UploadBase):
super().__init__(principal, data, persistence_path='bili.cookie', postprocessor=postprocessor)
if tags is None:
tags = []
else:
tags = [str(tag).format(streamer=self.data['name']) for tag in tags]
self.lines = lines
self.submit_api = submit_api
self.threads = threads
biliup/plugins/egame.py
-34
View File
@@ -1,34 +0,0 @@
import requests
from . import logger
from ..engine.decorators import Plugin
from ..engine.download import DownloadBase
@Plugin.download(regexp=r'(?:https?://)?(?:egame\.)?qq\.com')
class egame(DownloadBase):
def __init__(self, fname, url, suffix='flv'):
super().__init__(fname, url, suffix)
def check_stream(self, is_check=False):
if len(self.url.split("egame.qq.com/")) < 2:
logger.debug("直播间格式错误")
return False
rid = self.url.split("egame.qq.com/")[1]
url = 'https://share.egame.qq.com/cgi-bin/pgg_async_fcgi'
data = {'param': '''{"0":{"module":"pgg_live_read_svr","method":"get_live_and_profile_info","param":{
"anchor_id":''' + str(rid) + ''',"layout_id":"hot","index":1,"other_uid":0}}}'''}
r = requests.post(url=url, data=data, timeout=5).json()
if r['ecode'] != 0:
logger.debug("直播间地址错误")
return False
pid = r['data']["0"]["retBody"]["data"]["video_info"]["pid"]
if pid == "":
logger.debug("直播间不存在")
return False
is_live = r['data']["0"]["retBody"]["data"]["profile_info"]["is_live"]
if is_live != 1:
logger.debug("主播未开播")
return False
self.raw_stream_url = r['data']["0"]["retBody"]["data"]["video_info"]["stream_infos"][0]["play_url"]
self.room_title = r['data']["0"]["retBody"]['data']['video_info']['title']
return True
biliup/web/__init__.py
+6 -2
View File
@@ -171,10 +171,13 @@ async def tag_check(request):
@routes.get('/v1/videos')
async def streamers(request):
media_extensions = ['.mp4', '.flv', '.3gp', '.webm', '.mkv', '.ts']
_blacklist = ['next-env.d.ts']
# 获取文件列表
file_list = []
i = 1
for file_name in os.listdir('.'):
if file_name in _blacklist:
continue
name, ext = os.path.splitext(file_name)
if ext in media_extensions:
file_list.append({'key': i, 'name': file_name, 'updateTime': os.path.getmtime(file_name),
@@ -445,7 +448,8 @@ async def app_status(request):
from biliup.app import context
from biliup.config import Config
from biliup.app import PluginInfo
res = {}
from biliup import __version__
res = {'version': __version__,}
for key, value in context.items(): # 遍历删除不能被 json 序列化的键值对
if isinstance(value, Config):
continue
@@ -481,7 +485,7 @@ async def myinfo(request):
try:
config.load_cookies(file)
except FileNotFoundError:
return web.json_response({"status": 500, 'error': f"找不到 {file} ！！！"}, status=500)
return web.json_response({"status": 500, 'error': f"{file} 文件不存在"}, status=500)
cookies = config.data['user']['cookies']
return web.json_response(BiliBili.myinfo(cookies))