Merge branch 'setup'

.gitignore

@@ -18,6 +18,6 @@ test.py
 /upload.py
 /work.py
 *.zip
-dist
 *.egg-info
-build
+dist/
+build/

biliup/__init__.py

@@ -1,16 +1,11 @@
-#!/usr/bin/python3
-# coding:utf8
 import asyncio
 import platform
 import sys
-import common
-from engine import main
-from common.Daemon import Daemon
+from .engine import main
+from .common.Daemon import Daemon
 
-if __name__ == '__main__':
-
-    sys.excepthook = common.new_hook
 
+def _main():
     daemon = Daemon('watch_process.pid')
     if platform.system() != 'Windows' and len(sys.argv) == 2:
         if 'start' == sys.argv[1]:

biliup/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python3
+# coding:utf8
+from . import _main
+
+
+if __name__ == '__main__':
+    _main()

biliup/common/Daemon.py

@@ -5,7 +5,7 @@ import sys
 import os
 import time
 import atexit
-from engine import main
+from ..engine import main
 
 logger = logging.getLogger('log01')
 

biliup/common/__init__.py

@@ -1,6 +1,8 @@
 import os
 from datetime import datetime, timezone, timedelta
+from .log import SafeRotatingFileHandler
 import logging.config
+import sys
 
 
 def time_now():
@@ -15,6 +17,8 @@ def new_hook(t, v, tb):
     logger.error("Uncaught exception:", exc_info=(t, v, tb))
 
 
+logging.SafeRotatingFileHandler = SafeRotatingFileHandler
+sys.excepthook = new_hook
 log_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configlog.ini')
 logging.config.fileConfig(log_file_path)
 logger = logging.getLogger('log01')

biliup/common/configlog.ini

@@ -21,7 +21,7 @@ formatter=form02
 args=(sys.stdout,)
 
 [handler_handler02]
-class=common.log.SafeRotatingFileHandler
+class=SafeRotatingFileHandler
 level=INFO
 formatter=form01
 args=("ds_update.log", "W0", 1, 1)

biliup/common/decorators.py

@@ -40,7 +40,7 @@ class Plugin:
 
     @classmethod
     def sorted_checker(cls, urls):
-        from engine.plugins import general
+        from ..engine.plugins import general
         curls = urls.copy()
         checker_plugins = {}
         for plugin in cls.download_plugins:

biliup/common/reload.py

@@ -1,11 +1,10 @@
 import asyncio
 import sys
 import os
-import time
 import subprocess
 import logging
 
-from common.timer import Timer
+from .timer import Timer
 
 logger = logging.getLogger('log01')
 
@@ -73,11 +72,12 @@ class AutoReload(Timer):
                 for watched in self.watched:
                     watched.stop()
                 self.stop()
-                parent_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))  # 获得所在的目录的父级目
-                path = os.path.join(parent_path, 'Bilibili.py')
-                if sys.platform == 'win32':
-                    args = ["python", path]
-                else:
-                    args = [path, 'start']
+                # parent_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))  # 获得所在的目录的父级目
+                # path = os.path.join(parent_path, '__main__.py')
+                # if sys.platform == 'win32':
+                #     args = ["python", path]
+                # else:
+                #     args = [path, 'start']
+                args = ['biliup', 'start']
                 subprocess.Popen(args)
                 return logger.info('重启')

biliup/engine/__init__.py

@@ -2,10 +2,10 @@ import asyncio
 
 import yaml
 
-from common.decorators import Plugin
-from common.event import Event, event_manager
-from common.reload import AutoReload
-from common.timer import Timer
+from ..common.decorators import Plugin
+from ..common.event import Event, event_manager
+from ..common.reload import AutoReload
+from ..common.timer import Timer
 
 with open(r'config.yaml', encoding='utf-8') as stream:
     config = yaml.load(stream, Loader=yaml.FullLoader)
@@ -20,17 +20,17 @@ def invert_dict(d: dict):
 
 
 async def main():
-    import engine.plugins
+    from . import plugins
     streamers = config['streamers']
     streamer_url = {k: v['url'] for k, v in streamers.items()}
     inverted_index = invert_dict(streamer_url)
     urls = list(inverted_index.keys())
     url_status = dict.fromkeys(inverted_index, 0)
-    checker = Plugin(engine.plugins).sorted_checker(urls)
+    checker = Plugin(plugins).sorted_checker(urls)
     # 初始化事件管理器
     event_manager.context = {**config, 'urls': urls, 'url_status': url_status,
                              'checker': checker, 'inverted_index': inverted_index, 'streamer_url': streamer_url}
-    from engine.handler import CHECK_UPLOAD, CHECK
+    from ..engine.handler import CHECK_UPLOAD, CHECK
     event_manager.start()
 
     async def check_timer():

biliup/engine/downloader.py

@@ -3,8 +3,8 @@ import re
 import time
 from urllib.error import HTTPError
 
-from common.decorators import Plugin
-from engine.plugins import general, BatchCheckBase
+from ..common.decorators import Plugin
+from .plugins import general, BatchCheckBase
 
 logger = logging.getLogger('log01')
 

biliup/engine/handler.py

@@ -1,9 +1,10 @@
-import common
-from common import logger
-from common.event import Event, event_manager
-from engine.downloader import download, check_url
-from engine.plugins.upload import UploadBase
-from engine.uploader import upload
+# import common
+from .. import common
+from ..common import logger
+from ..common.event import Event, event_manager
+from .downloader import download, check_url
+from .plugins.upload import UploadBase
+from .uploader import upload
 
 CHECK = 'check'
 CHECK_UPLOAD = 'check_upload'

biliup/engine/plugins/afreecaTV.py

@@ -1,8 +1,8 @@
 import requests
 
-from common.decorators import Plugin
-from engine.plugins import match1, logger
-from engine.plugins.general import DownloadBase
+from ...common.decorators import Plugin
+from ..plugins import match1, logger
+from .general import DownloadBase
 
 VALID_URL_BASE = r"https?://play\.afreecatv\.com/(?P<username>\w+)(?:/\d+)?"
 

biliup/engine/plugins/base_adapter.py

@@ -0,0 +1,176 @@
+import os
+import subprocess
+import sys
+import time
+from threading import Event
+
+import streamlink
+import youtube_dl
+
+from .. import config
+from ..plugins import logger
+
+fake_headers = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.2.1'
+}
+
+
+class DownloadBase:
+    def __init__(self, fname, url, suffix=None):
+        self.fname = fname
+        self.url = url
+        self.suffix = suffix
+
+    def check_stream(self):
+        logger.debug(self.fname)
+        raise NotImplementedError()
+
+    def download(self, filename):
+        raise NotImplementedError()
+
+    def run(self):
+        if not self.check_stream():
+            return False
+        file_name = f'{self.file_name}.{self.suffix}'
+        retval = self.download(file_name)
+        logger.info(f'{retval}part: {file_name}')
+        self.rename(file_name)
+        return retval
+
+    def start(self):
+        i = 0
+        try:
+            logger.info('开始下载%s：%s' % (self.__class__.__name__, self.fname))
+            while i < 30:
+                ret = self.run()
+                if ret is False:
+                    return
+                elif ret == 1:
+                    time.sleep(45)
+                i += 1
+        except:
+            logger.exception("Uncaught exception:")
+        finally:
+            logger.info(f'退出下载{i}: {self.fname}')
+
+    @staticmethod
+    def rename(file_name):
+        try:
+            os.rename(file_name + '.part', file_name)
+            logger.debug('更名{0}为{1}'.format(file_name + '.part', file_name))
+        except FileNotFoundError:
+            logger.info('FileNotFoundError:' + file_name)
+        except FileExistsError:
+            os.rename(file_name + '.part', file_name)
+            logger.info('FileExistsError:更名{0}为{1}'.format(file_name + '.part', file_name))
+
+    @property
+    def file_name(self):
+        file_name = '%s%s' % (self.fname, str(time.time())[:10])
+        return file_name
+
+
+class YDownload(DownloadBase):
+    def __init__(self, fname, url, suffix='flv'):
+        super().__init__(fname, url, suffix)
+        self.ydl_opts = {}
+
+    def check_stream(self):
+        try:
+            self.get_sinfo()
+            return True
+        except youtube_dl.utils.DownloadError:
+            logger.debug('%s未开播或读取下载信息失败' % self.fname)
+            return False
+
+    def get_sinfo(self):
+        info_list = []
+        with youtube_dl.YoutubeDL() as ydl:
+            if self.url:
+                info = ydl.extract_info(self.url, download=False)
+            else:
+                logger.debug('%s不存在' % self.__class__.__name__)
+                return
+            for i in info['formats']:
+                info_list.append(i['format_id'])
+            logger.debug(info_list)
+        return info_list
+
+    def download(self, filename):
+        try:
+            self.ydl_opts = {'outtmpl': filename}
+            self.dl()
+        except youtube_dl.utils.DownloadError:
+            return 1
+        return 0
+
+    def dl(self):
+        with youtube_dl.YoutubeDL(self.ydl_opts) as ydl:
+            ydl.download([self.url])
+
+
+class SDownload(DownloadBase):
+    def __init__(self, fname, url, suffix='mp4'):
+        super().__init__(fname, url, suffix)
+        self.stream = None
+        self.flag = Event()
+
+    def check_stream(self):
+        logger.debug(self.fname)
+        try:
+            streams = streamlink.streams(self.url)
+            if streams:
+                self.stream = streams["best"]
+                fd = self.stream.open()
+                fd.close()
+                # streams.close()
+                return True
+        except streamlink.StreamlinkError:
+            return
+
+    def download(self, filename):
+
+        # fd = stream.open()
+        try:
+            with self.stream.open() as fd:
+                with open(filename + '.part', 'wb') as file:
+                    for f in fd:
+                        file.write(f)
+                        if self.flag.is_set():
+                            # self.flag.clear()
+                            return 1
+                    return 0
+        except OSError:
+            self.rename(filename)
+            raise
+
+
+# ffmpeg.exe -i  http://vfile1.grtn.cn/2018/1542/0254/3368/154202543368.ssm/154202543368.m3u8
+# -c copy -bsf:a aac_adtstoasc -movflags +faststart output.mp4
+class FFmpegdl(DownloadBase):
+    def __init__(self, fname, url, suffix=None):
+        super().__init__(fname, url, suffix)
+        self.raw_stream_url = None
+        self.opt_args = []
+        self.default_output_args = [
+            '-bsf:a', 'aac_adtstoasc',
+            '-fs', f"{config.get('file_size') if config.get('file_size') else '2621440000'}"
+        ]
+        self.default_input_args = ['-headers', ''.join('%s: %s\r\n' % x for x in fake_headers.items()),
+                                   '-reconnect_streamed', '1', '-reconnect_delay_max', '20', '-rw_timeout', '20000000']
+
+    def download(self, filename):
+        args = ['ffmpeg', '-y', *self.default_input_args,
+                '-i', self.raw_stream_url, *self.default_output_args, *self.opt_args,
+                '-c', 'copy', '-f', self.suffix, f'{filename}.part']
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE)
+        try:
+            retval = proc.wait()
+        except KeyboardInterrupt:
+            if sys.platform != 'win32':
+                proc.communicate(b'q')
+            raise
+        return retval

biliup/engine/plugins/bilibili.py

@@ -1,7 +1,7 @@
 from ykdl.common import url_to_module
 
-from common.decorators import Plugin
-from engine.plugins.general import DownloadBase
+from ...common.decorators import Plugin
+from .general import DownloadBase
 
 
 @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m|live)\.)?bilibili\.com')

biliup/engine/plugins/douyu.py

@@ -3,9 +3,9 @@ import platform
 from ykdl.common import url_to_module
 from ykdl.util.jsengine import chakra_available, quickjs_available, external_interpreter
 
-from common.decorators import Plugin
-from engine.plugins import logger
-from engine.plugins.general import DownloadBase
+from ...common.decorators import Plugin
+from ..plugins import logger
+from .general import DownloadBase
 
 
 @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m)\.)?douyu\.com')

biliup/engine/plugins/general.py

@@ -9,7 +9,7 @@ import youtube_dl
 from ykdl.common import url_to_module
 
 import engine
-from engine.plugins import logger, fake_headers
+from ..plugins import logger, fake_headers
 
 
 class DownloadBase:

biliup/engine/plugins/huya.py

@@ -4,9 +4,9 @@ import json
 
 import requests
 
-from common.decorators import Plugin
-from engine.plugins import match1, logger, fake_headers
-from engine.plugins.general import DownloadBase
+from ...common.decorators import Plugin
+from ..plugins import match1, logger, fake_headers
+from .general import DownloadBase
 
 
 @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m)\.)?huya\.com')

biliup/engine/plugins/twitch.py

@@ -3,9 +3,9 @@ from urllib.parse import urlencode
 
 import requests
 
-from common.decorators import Plugin
-from engine.plugins import BatchCheckBase, match1
-from engine.plugins.general import DownloadBase
+from ...common.decorators import Plugin
+from ..plugins import BatchCheckBase, match1
+from .general import DownloadBase
 
 VALID_URL_BASE = r'(?:https?://)?(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[0-9_a-zA-Z]+)'
 _OPERATION_HASHES = {

biliup/engine/plugins/upload/bili_chrome.py

@@ -9,10 +9,10 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
 from selenium.webdriver.support.ui import WebDriverWait
 
-import engine
-from common.decorators import Plugin
-from engine.plugins.upload import UploadBase, logger
-from engine.plugins.upload.slider import slider_cracker
+from ... import config
+from ....common.decorators import Plugin
+from ..upload import UploadBase, logger
+from .slider import slider_cracker
 
 
 @Plugin.upload("bilibili")
@@ -54,7 +54,7 @@ class BiliChrome(UploadBase):
         options = webdriver.ChromeOptions()
 
         options.add_argument('headless')
-        self.driver = webdriver.Chrome(executable_path=engine.config.get('chromedriver_path'), chrome_options=options)
+        self.driver = webdriver.Chrome(executable_path=config.get('chromedriver_path'), chrome_options=options)
         # service_log_path=service_log_path)
         try:
             self.driver.get("https://www.bilibili.com")
@@ -112,9 +112,9 @@ class BiliChrome(UploadBase):
         WebDriverWait(self.driver, 10).until(
             ec.presence_of_element_located((By.XPATH, r'//*[@id="login-username"]')))
         username = self.driver.find_element_by_xpath(r'//*[@id="login-username"]')
-        username.send_keys(engine.config['user']['account']['username'])
+        username.send_keys(config['user']['account']['username'])
         password = self.driver.find_element_by_xpath('//*[@id="login-passwd"]')
-        password.send_keys(engine.config['user']['account']['password'])
+        password.send_keys(config['user']['account']['password'])
         self.driver.find_element_by_class_name("btn-login").click()
         # logger.info('第四步')
         # try:

biliup/engine/plugins/upload/bili_web.py

@@ -19,15 +19,15 @@ import rsa
 from requests import utils
 from requests.adapters import HTTPAdapter, Retry
 
-import engine
-from common.decorators import Plugin
-from engine.plugins.upload import UploadBase, logger
+from ... import config
+from ....common.decorators import Plugin
+from ..upload import UploadBase, logger
 
 
 @Plugin.upload(platform="bili_web")
 class BiliWeb(UploadBase):
     def __init__(self, principal, data, user, lines='AUTO', threads=3, tid=174, tags=None, cover_path=None):
-        super().__init__(principal, data, persistence_path='engine/bili.cookie')
+        super().__init__(principal, data, persistence_path='bili.cookie')
         if tags is None:
             tags = ['星际争霸2', '电子竞技']
         self.user = user
@@ -135,13 +135,16 @@ class BiliBili:
         r = response.json()
         if r['code'] != 0 or r.get('data') is None or r['data'].get('cookie_info') is None:
             raise RuntimeError(r)
-        for cookie in r['data']['cookie_info']['cookies']:
-            self.__session.cookies.set(cookie['name'], cookie['value'])
-            if 'bili_jct' == cookie['name']:
-                self.__bili_jct = cookie['value']
-        self.cookies = self.__session.cookies.get_dict()
-        self.access_token = r['data']['token_info']['access_token']
-        self.refresh_token = r['data']['token_info']['refresh_token']
+        try:
+            for cookie in r['data']['cookie_info']['cookies']:
+                self.__session.cookies.set(cookie['name'], cookie['value'])
+                if 'bili_jct' == cookie['name']:
+                    self.__bili_jct = cookie['value']
+            self.cookies = self.__session.cookies.get_dict()
+            self.access_token = r['data']['token_info']['access_token']
+            self.refresh_token = r['data']['token_info']['refresh_token']
+        except:
+            raise RuntimeError(r)
         return r
 
     def login_by_cookies(self, cookie):
@@ -376,14 +379,14 @@ class BiliBili:
 
         logger.info(f'用户权重: {user_weight} => 网页端分p数量受到限制使用客户端api端提交')
         if not self.access_token:
-            self.login_by_password(**engine.config['user']['account'])
+            self.login_by_password(**config['user']['account'])
             self.store()
         while True:
             ret = self.__session.post(f'http://member.bilibili.com/x/vu/client/add?access_key={self.access_token}',
                                       timeout=5, json=asdict(self.video)).json()
             if ret['code'] == -101:
                 logger.info(f'刷新token{ret}')
-                self.login_by_password(**engine.config['user']['account'])
+                self.login_by_password(**config['user']['account'])
                 self.store()
                 continue
             break

biliup/engine/uploader.py

@@ -1,8 +1,8 @@
 import inspect
 
-from common import logger
-from common.decorators import Plugin
-import engine
+from .. import engine
+from ..common import logger
+from ..common.decorators import Plugin
 
 
 def upload(platform, index, data):

config(demo).yaml

@@ -6,8 +6,6 @@ user: # 在填了cookies的情况下优先使用cookies上传，如需使用用
     account:
         username: your usrname
         password: your password
-# 默认通过网页接口上传,可选通过操作chrome上传,此时需要填写chromedriver路径
-#chromedriver_path: /usr/local/bin/chromedriver
 
 # b站上传线路选择，默认为自动模式，目前可手动切换为bda2, kodo, ws, qn。
 lines: AUTO
@@ -70,3 +68,6 @@ streamers:
     星际2Classic经典神皇神族第一视角:
         url:
             - https://www.twitch.tv/kimclassic
+
+# 默认通过网页接口上传,可选通过操作chrome上传,此时需要填写chromedriver路径
+#chromedriver_path: /usr/local/bin/chromedriver

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

setup.cfg

@@ -0,0 +1,35 @@
+[metadata]
+name = biliup
+version = 0.0.1
+author = ForgQi
+url = https://github.com/ForgQi/bilibiliupload
+description = stream download and upload
+long_description = file: README.md
+long_description_content_type = text/markdown
+license_files = LICENSE
+keywords = bilibili douyu huya
+
+[options]
+packages = find:
+python_requires = >= 3.7
+install_requires =
+  selenium >= 3.8.1
+  psutil >= 5.4.6
+  youtube_dl >= 2018.11.18
+  Pillow >= 5.2.0
+  aiohttp[speedups]
+  Requests >= 2.20.0
+  PyYAML >= 4.2b1
+  streamlink >= 0.14.2
+  ykdl @ git+https://github.com/zhangn1985/ykdl.git
+  rsa >= 4.6
+
+[options.package_data]
+biliup = common/*.ini
+
+[options.packages.find]
+include = biliup*
+
+[options.entry_points]
+console_scripts =
+    biliup = biliup:_main

setup.py

@@ -0,0 +1,2 @@
+import setuptools
+setuptools.setup()