新增NICO录制,新增Youtube的StreamLink下载模式,配置文件文案修正 (#460)

* 增加Twitch的去广告开关(解决广告分段问题)
修改当请求流错误时等待delay的值的合理性,写死为30秒(防止有人等待的太久)
增加、修正toml,yaml文件的配置和注释

* 新增NICO的录制,登录信息选项在配置文件的user中
新增Youtube的StreamLink下载模式,由配置文件里use_new_ytb_downloader,默认关闭
配置文件文案修正

---------

Co-authored-by: KNaiFen <knaifen@foxmail.com>
This commit is contained in:
奶粉andKNaiFen authored and GitHub committed 2023-06-06 02:31:56 +00:00
1 parent e77b11995a
commit fe5b9c986d
10 files changed
+489 -69

No files matched your search

+5 -1
View File
@@ -41,7 +41,11 @@ class DanmakuClient:
self.__url = url
else:
self.__url = 'http://' + url
for u, s in {'douyu.com': Douyu, 'huya.com': Huya, 'live.bilibili.com': Bilibili, 'twitch.tv': Twitch}.items():
for u, s in {'douyu.com': Douyu,
'huya.com': Huya,
'live.bilibili.com': Bilibili,
'twitch.tv': Twitch
}.items():
if re.match(r'^(?:http[s]?://)?.*?%s/(.+?)$' % u, url):
self.__site = s
self.__u = u
@@ -0,0 +1 @@
import os
@@ -0,0 +1,44 @@
from . import enc
from base64 import urlsafe_b64encode as b64enc
from urllib.parse import quote
def _header(video_id, channel_id) -> str:
S1_3 = enc.rs(1, video_id)
S1_5 = enc.rs(1, channel_id) + enc.rs(2, video_id)
S1 = enc.rs(3, S1_3) + enc.rs(5, S1_5)
S3 = enc.rs(48687757, enc.rs(1, video_id))
header_replay = enc.rs(1, S1) + enc.rs(3, S3) + enc.nm(4, 1)
return b64enc(header_replay)
def _build(video_id, seektime, topchat_only, channel_id) -> str:
chattype = 4 if topchat_only else 1
if seektime < 0:
seektime = 0
timestamp = int(seektime * 1000000)
header = enc.rs(3, _header(video_id, channel_id))
timestamp = enc.nm(5, timestamp)
s6 = enc.nm(6, 0)
s7 = enc.nm(7, 0)
s8 = enc.nm(8, 0)
s9 = enc.nm(9, 4)
s10 = enc.rs(10, enc.nm(4, 0))
chattype = enc.rs(14, enc.nm(1, 4))
s15 = enc.nm(15, 0)
entity = b"".join((header, timestamp, s6, s7, s8, s9, s10, chattype, s15))
continuation = enc.rs(156074452, entity)
return quote(b64enc(continuation).decode())
def getparam(video_id, seektime=0, topchat_only=False, channel_id="") -> str:
"""
Parameter
---------
seektime : int
unit:seconds
start position of fetching chat data.
topchat_only : bool
if True, fetch only 'top chat'
"""
return _build(video_id, seektime, topchat_only, channel_id)
+24
View File
@@ -0,0 +1,24 @@
def vn(val):
if val < 0:
raise ValueError
buf = b""
while val >> 7:
m = val & 0xFF | 0x80
buf += m.to_bytes(1, "big")
val >>= 7
buf += val.to_bytes(1, "big")
return buf
def tp(a, b, ary):
return vn((b << 3) | a) + ary
def rs(a, ary):
if isinstance(ary, str):
ary = ary.encode()
return tp(2, a, vn(len(ary)) + ary)
def nm(a, ary):
return tp(0, a, vn(ary))
@@ -0,0 +1,84 @@
import random
import time
from . import enc
from base64 import urlsafe_b64encode as b64enc
from urllib.parse import quote
def _header(video_id, channel_id) -> str:
S1_3 = enc.rs(1, video_id)
S1_5 = enc.rs(1, channel_id) + enc.rs(2, video_id)
S1 = enc.rs(3, S1_3) + enc.rs(5, S1_5)
S3 = enc.rs(48687757, enc.rs(1, video_id))
header_replay = enc.rs(1, S1) + enc.rs(3, S3) + enc.nm(4, 1)
return b64enc(header_replay)
def _build(video_id, channel_id, ts1, ts2, ts3, ts4, ts5, topchat_only) -> str:
chattype = 4 if topchat_only else 1
b1 = enc.nm(1, 0)
b2 = enc.nm(2, 0)
b3 = enc.nm(3, 0)
b4 = enc.nm(4, 0)
b7 = enc.rs(7, "")
b8 = enc.nm(8, 0)
b9 = enc.rs(9, "")
timestamp2 = enc.nm(10, ts2)
b11 = enc.nm(11, 3)
b15 = enc.nm(15, 0)
header = enc.rs(3, _header(video_id, channel_id))
timestamp1 = enc.nm(5, ts1)
s6 = enc.nm(6, 0)
s7 = enc.nm(7, 0)
s8 = enc.nm(8, 1)
body = enc.rs(9, b"".join((b1, b2, b3, b4, b7, b8, b9, timestamp2, b11, b15)))
timestamp3 = enc.nm(10, ts3)
timestamp4 = enc.nm(11, ts4)
s13 = enc.nm(13, chattype)
chattype = enc.rs(16, enc.nm(1, chattype))
s17 = enc.nm(17, 0)
str19 = enc.rs(19, enc.nm(1, 0))
timestamp5 = enc.nm(20, ts5)
entity = b"".join(
(
header,
timestamp1,
s6,
s7,
s8,
body,
timestamp3,
timestamp4,
s13,
chattype,
s17,
str19,
timestamp5,
)
)
continuation = enc.rs(119693434, entity)
return quote(b64enc(continuation).decode())
def _times(past_sec):
n = int(time.time())
_ts1 = n - random.uniform(0, 1 * 3)
_ts2 = n - random.uniform(0.01, 0.99)
_ts3 = n - past_sec + random.uniform(0, 1)
_ts4 = n - random.uniform(10 * 60, 60 * 60)
_ts5 = n - random.uniform(0.01, 0.99)
return list(map(lambda x: int(x * 1000000), [_ts1, _ts2, _ts3, _ts4, _ts5]))
def getparam(video_id, channel_id, past_sec=0, topchat_only=False) -> str:
"""
Parameter
---------
past_sec : int
seconds to load past chat data
topchat_only : bool
if True, fetch only 'top chat'
"""
return _build(video_id, channel_id, *_times(past_sec), topchat_only)
+137
View File
@@ -0,0 +1,137 @@
import json, re, select, random, traceback, urllib, datetime, base64
import asyncio, aiohttp
# The core codes for YouTube support are basically from taizan-hokuto/pytchat
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36",
}
class Youtube:
q = None
url = ""
vid = ""
ctn = ""
client = None
stop = False
@classmethod
async def run(cls, url, q, client, **kargs):
from .paramgen import liveparam
cls.q = q
cls.url = url
cls.client = client
cls.stop = False
cls.key = "eW91dHViZWkvdjEvbGl2ZV9jaGF0L2dldF9saXZlX2NoYXQ/a2V5PUFJemFTeUFPX0ZKMlNscVU4UTRTVEVITEdDaWx3X1k5XzExcWNXOA=="
await cls.get_url()
while cls.stop == False:
try:
await cls.get_room_info()
cls.ctn = liveparam.getparam(cls.vid, cls.cid, 1)
await cls.get_chat()
except:
traceback.print_exc()
await asyncio.sleep(1)
@classmethod
async def stop(cls):
cls.stop = True
@classmethod
async def get_url(cls):
a = re.search(r"youtube.com/channel/([^/?]+)", cls.url)
try:
cid = a.group(1)
cls.cid = cid
cls.url = f"https://www.youtube.com/channel/{cid}/videos"
except:
a = re.search(r"youtube.com/watch\?v=([^/?]+)", cls.url)
async with cls.client.request(
"get", f"https://www.youtube.com/embed/{a.group(1)}"
) as resp:
b = re.search(r'\\"channelId\\":\\"(.{24})\\"', await resp.text())
cls.cid = b.group(1)
cls.url = f"https://www.youtube.com/channel/{cls.cid}/videos"
@classmethod
async def get_room_info(cls):
async with cls.client.request("get", cls.url) as resp:
t = re.search(
r'"gridVideoRenderer"((.(?!"gridVideoRenderer"))(?!"style":"UPCOMING"))+"label":"(LIVE|LIVE NOW|PREMIERING NOW)"([\s\S](?!"style":"UPCOMING"))+?("gridVideoRenderer"|</script>)',
await resp.text(),
).group(0)
cls.vid = re.search(r'"gridVideoRenderer".+?"videoId":"(.+?)"', t).group(1)
# print(cls.vid)
@classmethod
async def get_chat_single(cls):
msgs = []
data = {
"context": {
"client": {
"visitorData": "",
"userAgent": headers["user-agent"],
"clientName": "WEB",
"clientVersion": "".join(
(
"2.",
(datetime.datetime.today() - datetime.timedelta(days=1)).strftime(
"%Y%m%d"
),
".01.00",
)
),
},
},
"continuation": cls.ctn,
}
u = f'https://www.youtube.com/{base64.b64decode(cls.key).decode("utf-8")}'
async with cls.client.request("post", u, headers=headers, json=data) as resp:
# print(await resp.text())
j = await resp.json()
j = j["continuationContents"]
cont = j["liveChatContinuation"]["continuations"][0]
if cont is None:
raise Exception("No Continuation")
metadata = (
cont.get("invalidationContinuationData")
or cont.get("timedContinuationData")
or cont.get("reloadContinuationData")
or cont.get("liveChatReplayContinuationData")
)
cls.ctn = metadata["continuation"]
# print(j['liveChatContinuation'].get('actions'))
for action in j["liveChatContinuation"].get("actions", []):
try:
renderer = action["addChatItemAction"]["item"]["liveChatTextMessageRenderer"]
msg = {}
msg["name"] = renderer["authorName"]["simpleText"]
message = ""
runs = renderer["message"].get("runs")
for r in runs:
if r.get("emoji"):
message += r["emoji"].get("shortcuts", [""])[0]
else:
message += r.get("text", "")
msg["content"] = message
msg["msg_type"] = "danmaku"
msg["color"] = "16777215"
msgs.append(msg)
except:
pass
return msgs
@classmethod
async def get_chat(cls):
while cls.stop == False:
ms = await cls.get_chat_single()
if len(ms) != 0:
interval = 1 / len(ms)
else:
await asyncio.sleep(1)
for m in ms:
await cls.q.put(m)
await asyncio.sleep(interval)
+68
View File
@@ -0,0 +1,68 @@
import random
import string
import json
import subprocess
import time
import requests
from . import logger
import requests
import json
from biliup.config import config
from ..engine.decorators import Plugin
from ..engine.download import DownloadBase
from bs4 import BeautifulSoup
from ..plugins import logger
@Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m|live)\.)?nicovideo\.jp')
class Nico(DownloadBase):
def __init__(self, fname, url, suffix='flv'):
super().__init__(fname, url, suffix)
def check_stream(self):
try:
response = requests.get(self.url)
soup = BeautifulSoup(response.text, 'html.parser')
script_tag = soup.find('script', {'type': 'application/ld+json'})
if script_tag:
data = script_tag.string
json_sc = json.loads(data)
self.room_title = json_sc['name']
except:
logger.info("获取标题失败")
port = random.randint(1025, 65535)
stream_shell = [
"streamlink",
"--player-external-http", # 为外部程序提供流媒体数据
"--player-external-http-port", str(port), # 对外部输出流的端口
self.url, "best" # 流链接
]
if config.get('user', {}).get('niconico-email') is not None:
niconico_email = "--niconico-email " + config.get('user', {}).get('niconico-email')
stream_shell.insert(1, niconico_email)
if config.get('user', {}).get('niconico-password') is not None:
niconico_password = "--niconico-password " + config.get('user', {}).get('niconico-password')
stream_shell.insert(1, niconico_password)
if config.get('user', {}).get('niconico-user-session') is not None:
niconico_user_session = "--niconico-user-session " + config.get('user', {}).get('niconico-user-session')
stream_shell.insert(1, niconico_user_session)
if config.get('user', {}).get('niconico-purge-credentials') is not None:
niconico_purge_credentials = "--niconico-purge-credentials " + config.get('user', {}).get('niconico-purge-credentials')
stream_shell.insert(1, niconico_purge_credentials)
self.proc = subprocess.Popen(stream_shell)
self.raw_stream_url = f"http://localhost:{port}"
i = 0
while i < 5:
if not (self.proc.poll() is None):
return False
time.sleep(1)
i += 1
return True
def close(self):
try:
if self.proc is not None:
self.proc.terminate()
except:
logger.exception(f'terminate {self.fname} failed')
+119 -68
View File
@@ -1,4 +1,11 @@
import asyncio
import random
import subprocess
import threading
from urllib.parse import urlparse
import yt_dlp
import streamlink
import os
import time
from PIL import Image
@@ -6,8 +13,9 @@ from yt_dlp.utils import MaxDownloadsReached
from yt_dlp.utils import DateRange
from biliup.config import config
from biliup.plugins.Danmaku import DanmakuClient
from ..engine.decorators import Plugin
from ..engine.download import DownloadBase
from ..engine.download import DownloadBase, get_valid_filename, stream_gears_download
from . import logger
VALID_URL_BASE = r'https?://(?:(?:www|m)\.)?youtube\.com/(?P<id>.*?)\??(.*?)'
@@ -16,6 +24,8 @@ VALID_URL_BASE = r'https?://(?:(?:www|m)\.)?youtube\.com/(?P<id>.*?)\??(.*?)'
class Youtube(DownloadBase):
def __init__(self, fname, url, suffix='webm'):
DownloadBase.__init__(self, fname, url, suffix=suffix)
self.use_new_ytb_downloader = config.get('use_new_ytb_downloader', False)
self.ytb_danmaku = config.get('ytb_danmaku', False)
self.cookiejarFile = config.get('user', {}).get('youtube_cookie')
self.vcodec = config.get('youtube_prefer_vcodec','av01|vp9|avc')
self.acodec = config.get('youtube_prefer_acodec','opus|mp4a')
@@ -26,73 +36,114 @@ class Youtube(DownloadBase):
self.use_youtube_cover = config.get('use_youtube_cover', False)
def check_stream(self):
with yt_dlp.YoutubeDL({'download_archive': 'archive.txt', 'ignoreerrors': True, 'extract_flat': True,
'cookiefile': self.cookiejarFile}) as ydl:
info = ydl.extract_info(self.url, download=False, process=False)
if info is None:
logger.warning(self.cookiejarFile)
logger.warning(self.url)
return False
if '_type' in info:
# /live 形式链接取标题
if info['_type'] in 'url' and info['webpage_url_basename'] in 'live':
live = ydl.extract_info(info['url'], download=False, process=False)
self.room_title = live['title']
# Playlist 暂时返回列表名
if info['_type'] in 'playlist':
self.room_title = info['title']
else:
# 视频取标题
self.room_title = info['title']
if info.get('entries') is None:
if ydl.in_download_archive(info):
if self.use_new_ytb_downloader and self.downloader == 'ffmpeg':
_fname, self.room_title = self.get_stream_info(self.url)
port = random.randint(1025, 65535)
stream_shell = [
"streamlink",
"--player-external-http", # 为外部程序提供流媒体数据
"--player-external-http-port", str(port), # 对外部输出流的端口
self.url, "best" # 流链接
]
self.proc = subprocess.Popen(stream_shell)
self.raw_stream_url = f"http://localhost:{port}"
i = 0
while i < 5:
if not (self.proc.poll() is None):
return
time.sleep(1)
i += 1
return True
else:
with yt_dlp.YoutubeDL({'download_archive': 'archive.txt', 'ignoreerrors': True, 'extract_flat': True,
'cookiefile': self.cookiejarFile}) as ydl:
info = ydl.extract_info(self.url, download=False, process=False)
if info is None:
logger.warning(self.cookiejarFile)
logger.warning(self.url)
return False
return True
for entry in info['entries']:
# 取 Playlist 内视频标题
self.room_title = entry['title']
if ydl.in_download_archive(entry):
continue
# ydl.record_download_archive()
return True
if '_type' in info:
# /live 形式链接取标题
if info['_type'] in 'url' and info['webpage_url_basename'] in 'live':
live = ydl.extract_info(info['url'], download=False, process=False)
self.room_title = live['title']
# Playlist 暂时返回列表名
if info['_type'] in 'playlist':
self.room_title = info['title']
else:
# 视频取标题
self.room_title = info['title']
if info.get('entries') is None:
if ydl.in_download_archive(info):
return False
return True
for entry in info['entries']:
# 取 Playlist 内视频标题
self.room_title = entry['title']
if ydl.in_download_archive(entry):
continue
# ydl.record_download_archive()
return True
def download(self, filename):
try:
ydl_opts = {
'outtmpl': filename+ '.%(ext)s',
'format': f"bestvideo[vcodec~='^({self.vcodec})'][height<={self.resolution}][filesize<{self.filesize}]+bestaudio[acodec~='^({self.acodec})']/best[height<={self.resolution}]/best",
'cookiefile': self.cookiejarFile,
# 'proxy': proxyUrl,
'ignoreerrors': True,
'max_downloads': 1,
'daterange' : DateRange(self.afterdate, self.beforedate),
'break_on_reject': True,
'download_archive': 'archive.txt',
}
if self.use_youtube_cover is True:
ydl_opts['writethumbnail'] = True
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.download([self.url])
except MaxDownloadsReached:
save_dir = f'cover/youtube/'
webp_cover_path = f'{filename}.webp'
jpg_cover_path = f'{filename}.jpg'
if os.path.exists(webp_cover_path):
with Image.open(webp_cover_path) as img:
img = img.convert('RGB')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
img.save(f'{save_dir}{filename}.jpg', format='JPEG')
os.remove(webp_cover_path)
self.live_cover_path = f'{save_dir}{filename}.jpg'
elif os.path.exists(jpg_cover_path):
os.rename(jpg_cover_path, f'{save_dir}{filename}.jpg')
self.live_cover_path = f'{save_dir}{filename}.jpg'
return False
except yt_dlp.utils.RejectedVideoReached:
logger.info("已下载完毕指定日期内的视频,结束本次任务")
return False
except yt_dlp.utils.DownloadError:
logger.exception(self.fname)
return 1
return 0
if self.use_new_ytb_downloader and self.downloader == 'ffmpeg':
if self.filename_prefix: # 判断是否存在自定义录播命名设置
filename = (self.filename_prefix.format(streamer=self.fname, title=self.room_title).encode(
'unicode-escape').decode()).encode().decode("unicode-escape")
else:
filename = f'{self.fname}%Y-%m-%dT%H_%M_%S'
filename = get_valid_filename(filename)
fmtname = time.strftime(filename.encode("unicode-escape").decode()).encode().decode("unicode-escape")
loop = asyncio.new_event_loop()
threading.Thread(target=loop.run_until_complete, args=(self.danmaku_download_start(fmtname),)).start()
self.ffmpeg_download(fmtname)
else:
try:
ydl_opts = {
'outtmpl': filename+ '.%(ext)s',
'format': f"bestvideo[vcodec~='^({self.vcodec})'][height<={self.resolution}][filesize<{self.filesize}]+bestaudio[acodec~='^({self.acodec})']/best[height<={self.resolution}]/best",
'cookiefile': self.cookiejarFile,
# 'proxy': proxyUrl,
'ignoreerrors': True,
'max_downloads': 1,
'daterange' : DateRange(self.afterdate, self.beforedate),
'break_on_reject': True,
'download_archive': 'archive.txt',
}
if self.use_youtube_cover is True:
ydl_opts['writethumbnail'] = True
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.download([self.url])
except MaxDownloadsReached:
save_dir = f'cover/youtube/'
webp_cover_path = f'{filename}.webp'
jpg_cover_path = f'{filename}.jpg'
if os.path.exists(webp_cover_path):
with Image.open(webp_cover_path) as img:
img = img.convert('RGB')
if not os.path.exists(save_dir):
os.makedirs(save_dir)
img.save(f'{save_dir}{filename}.jpg', format='JPEG')
os.remove(webp_cover_path)
self.live_cover_path = f'{save_dir}{filename}.jpg'
elif os.path.exists(jpg_cover_path):
os.rename(jpg_cover_path, f'{save_dir}{filename}.jpg')
self.live_cover_path = f'{save_dir}{filename}.jpg'
return False
except yt_dlp.utils.RejectedVideoReached:
logger.info("已下载完毕指定日期内的视频,结束本次任务")
return False
except yt_dlp.utils.DownloadError:
logger.exception(self.fname)
return 1
return 0
def get_stream_info(self, url):
session = streamlink.Streamlink()
# Streamlink 插件系统的内部运行机制,通过 URL 匹配到 YouTube 插件
plugin = session.resolve_url(url)[1]
streams = plugin(session=session, url=url)
streams._get_streams()
author = streams.get_author()
title = streams.get_title()
return author, title
+4
View File
@@ -118,6 +118,10 @@ check_sourcecode = 15
#youtube_after_date = '20220201'
### 仅下载该日期之前的视频(可与上面的youtube_after_date配合使用,构成指定下载范围区间)
#youtube_before_date = '20230501'
### YouTube的Streamlink下载器模式,开启之后会调用Streamlink获取直播流并且使用FFMPEG下载,默认关闭【仅当下载器是FFMPEG时才有效】
### 开启之后上面关于YOUTUBE的选项全部无效【我也不知道这个功能有什么屌用】
#use_new_ytb_downloader = false
#------Twitch------#
### 录制Twitch弹幕,默认关闭【只有下载器为FFMPEG时才有效】
+3
View File
@@ -116,6 +116,9 @@ check_sourcecode: 15
#youtube_after_date: '20220201'
### 仅下载该日期之前的视频(可与上面的youtube_after_date配合使用,构成指定下载范围区间)
#youtube_before_date: '20230501'
### YouTube的Streamlink下载器模式,开启之后会调用Streamlink获取直播流并且使用FFMPEG下载,默认关闭【仅当下载器是FFMPEG时才有效】
### 开启之后上面关于YOUTUBE的选项全部无效【我也不知道这个功能有什么屌用】
use_new_ytb_downloader: false
#------Twitch------#
### 录制Twitch弹幕,默认关闭【只有下载器为FFMPEG时才有效】