From 711408390f39a900316fb01be73d7fe4e1a13d8b Mon Sep 17 00:00:00 2001 From: ForgQi <34411314+ForgQi@users.noreply.github.com> Date: Wed, 20 Jan 2021 02:54:06 +0800 Subject: [PATCH] setup --- .gitignore | 1 + Bilibili.py => biliup/__init__.py | 11 ++----- biliup/__main__.py | 7 +++++ {common => biliup/common}/Daemon.py | 2 +- {common => biliup/common}/__init__.py | 4 +++ {common => biliup/common}/configlog.ini | 2 +- {common => biliup/common}/decorators.py | 2 +- {common => biliup/common}/event.py | 0 {common => biliup/common}/log.py | 0 {common => biliup/common}/reload.py | 16 +++++----- {common => biliup/common}/timer.py | 0 {engine => biliup/engine}/__init__.py | 14 ++++----- {engine => biliup/engine}/downloader.py | 4 +-- {engine => biliup/engine}/handler.py | 13 +++++---- {engine => biliup/engine}/plugins/__init__.py | 0 .../engine}/plugins/afreecaTV.py | 6 ++-- .../engine}/plugins/base_adapter.py | 6 ++-- {engine => biliup/engine}/plugins/bilibili.py | 4 +-- {engine => biliup/engine}/plugins/douyu.py | 6 ++-- {engine => biliup/engine}/plugins/general.py | 4 +-- {engine => biliup/engine}/plugins/huya.py | 6 ++-- {engine => biliup/engine}/plugins/twitch.py | 6 ++-- .../engine}/plugins/upload/__init__.py | 0 .../engine}/plugins/upload/bili_chrome.py | 14 ++++----- .../engine}/plugins/upload/bili_web.py | 29 ++++++++++--------- .../engine}/plugins/upload/slider/__init__.py | 0 {engine => biliup/engine}/uploader.py | 6 ++-- pyproject.toml | 3 ++ setup.cfg | 28 ++++++++++++++++++ setup.py | 2 ++ 30 files changed, 120 insertions(+), 76 deletions(-) rename Bilibili.py => biliup/__init__.py (75%) create mode 100644 biliup/__main__.py rename {common => biliup/common}/Daemon.py (99%) rename {common => biliup/common}/__init__.py (81%) rename {common => biliup/common}/configlog.ini (93%) rename {common => biliup/common}/decorators.py (98%) rename {common => biliup/common}/event.py (100%) rename {common => biliup/common}/log.py (100%) rename {common => biliup/common}/reload.py (84%) rename {common => biliup/common}/timer.py (100%) rename {engine => biliup/engine}/__init__.py (81%) rename {engine => biliup/engine}/downloader.py (90%) rename {engine => biliup/engine}/handler.py (91%) rename {engine => biliup/engine}/plugins/__init__.py (100%) rename {engine => biliup/engine}/plugins/afreecaTV.py (92%) rename {engine => biliup/engine}/plugins/base_adapter.py (97%) rename {engine => biliup/engine}/plugins/bilibili.py (86%) rename {engine => biliup/engine}/plugins/douyu.py (89%) rename {engine => biliup/engine}/plugins/general.py (91%) rename {engine => biliup/engine}/plugins/huya.py (85%) rename {engine => biliup/engine}/plugins/twitch.py (96%) rename {engine => biliup/engine}/plugins/upload/__init__.py (100%) rename {engine => biliup/engine}/plugins/upload/bili_chrome.py (95%) rename {engine => biliup/engine}/plugins/upload/bili_web.py (95%) rename {engine => biliup/engine}/plugins/upload/slider/__init__.py (100%) rename {engine => biliup/engine}/uploader.py (89%) create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 86895c0..3764f98 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ test.py /upload.py /work.py *.zip +*.egg-info \ No newline at end of file diff --git a/Bilibili.py b/biliup/__init__.py similarity index 75% rename from Bilibili.py rename to biliup/__init__.py index e16e1b7..546444f 100644 --- a/Bilibili.py +++ b/biliup/__init__.py @@ -1,15 +1,10 @@ -#!/usr/bin/python3 -# coding:utf8 import asyncio import sys -import common -from engine import main -from common.Daemon import Daemon +from .engine import main +from .common.Daemon import Daemon -if __name__ == '__main__': - - sys.excepthook = common.new_hook +def _main(): daemon = Daemon('watch_process.pid') if len(sys.argv) == 2: if 'start' == sys.argv[1]: diff --git a/biliup/__main__.py b/biliup/__main__.py new file mode 100644 index 0000000..7fe482d --- /dev/null +++ b/biliup/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/python3 +# coding:utf8 +from . import _main + + +if __name__ == '__main__': + _main() diff --git a/common/Daemon.py b/biliup/common/Daemon.py similarity index 99% rename from common/Daemon.py rename to biliup/common/Daemon.py index 3fcecd9..4724768 100644 --- a/common/Daemon.py +++ b/biliup/common/Daemon.py @@ -5,7 +5,7 @@ import sys import os import time import atexit -from engine import main +from ..engine import main logger = logging.getLogger('log01') diff --git a/common/__init__.py b/biliup/common/__init__.py similarity index 81% rename from common/__init__.py rename to biliup/common/__init__.py index 336491a..af8011b 100644 --- a/common/__init__.py +++ b/biliup/common/__init__.py @@ -1,6 +1,8 @@ import os from datetime import datetime, timezone, timedelta +from .log import SafeRotatingFileHandler import logging.config +import sys def time_now(): @@ -15,6 +17,8 @@ def new_hook(t, v, tb): logger.error("Uncaught exception:", exc_info=(t, v, tb)) +logging.SafeRotatingFileHandler = SafeRotatingFileHandler +sys.excepthook = new_hook log_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configlog.ini') logging.config.fileConfig(log_file_path) logger = logging.getLogger('log01') diff --git a/common/configlog.ini b/biliup/common/configlog.ini similarity index 93% rename from common/configlog.ini rename to biliup/common/configlog.ini index edd7e02..bde64a7 100644 --- a/common/configlog.ini +++ b/biliup/common/configlog.ini @@ -21,7 +21,7 @@ formatter=form02 args=(sys.stdout,) [handler_handler02] -class=common.log.SafeRotatingFileHandler +class=SafeRotatingFileHandler level=INFO formatter=form01 args=("ds_update.log", "W0", 1, 1) diff --git a/common/decorators.py b/biliup/common/decorators.py similarity index 98% rename from common/decorators.py rename to biliup/common/decorators.py index 19800a3..9499634 100644 --- a/common/decorators.py +++ b/biliup/common/decorators.py @@ -40,7 +40,7 @@ class Plugin: @classmethod def sorted_checker(cls, urls): - from engine.plugins import general + from ..engine.plugins import general curls = urls.copy() checker_plugins = {} for plugin in cls.download_plugins: diff --git a/common/event.py b/biliup/common/event.py similarity index 100% rename from common/event.py rename to biliup/common/event.py diff --git a/common/log.py b/biliup/common/log.py similarity index 100% rename from common/log.py rename to biliup/common/log.py diff --git a/common/reload.py b/biliup/common/reload.py similarity index 84% rename from common/reload.py rename to biliup/common/reload.py index c74c9a2..340dbfb 100644 --- a/common/reload.py +++ b/biliup/common/reload.py @@ -1,11 +1,10 @@ import asyncio import sys import os -import time import subprocess import logging -from common.timer import Timer +from .timer import Timer logger = logging.getLogger('log01') @@ -73,11 +72,12 @@ class AutoReload(Timer): for watched in self.watched: watched.stop() self.stop() - parent_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) # 获得所在的目录的父级目 - path = os.path.join(parent_path, 'Bilibili.py') - if sys.platform == 'win32': - args = ["python", path] - else: - args = [path, 'start'] + # parent_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) # 获得所在的目录的父级目 + # path = os.path.join(parent_path, '__main__.py') + # if sys.platform == 'win32': + # args = ["python", path] + # else: + # args = [path, 'start'] + args = ['biliup', 'start'] subprocess.Popen(args) return logger.info('重启') diff --git a/common/timer.py b/biliup/common/timer.py similarity index 100% rename from common/timer.py rename to biliup/common/timer.py diff --git a/engine/__init__.py b/biliup/engine/__init__.py similarity index 81% rename from engine/__init__.py rename to biliup/engine/__init__.py index 4c6c58a..e4f4c0c 100644 --- a/engine/__init__.py +++ b/biliup/engine/__init__.py @@ -2,10 +2,10 @@ import asyncio import yaml -from common.decorators import Plugin -from common.event import Event, event_manager -from common.reload import AutoReload -from common.timer import Timer +from ..common.decorators import Plugin +from ..common.event import Event, event_manager +from ..common.reload import AutoReload +from ..common.timer import Timer with open(r'config.yaml', encoding='utf-8') as stream: config = yaml.load(stream, Loader=yaml.FullLoader) @@ -20,17 +20,17 @@ def invert_dict(d: dict): async def main(): - import engine.plugins + from . import plugins streamers = config['streamers'] streamer_url = {k: v['url'] for k, v in streamers.items()} inverted_index = invert_dict(streamer_url) urls = list(inverted_index.keys()) url_status = dict.fromkeys(inverted_index, 0) - checker = Plugin(engine.plugins).sorted_checker(urls) + checker = Plugin(plugins).sorted_checker(urls) # 初始化事件管理器 event_manager.context = {**config, 'urls': urls, 'url_status': url_status, 'checker': checker, 'inverted_index': inverted_index, 'streamer_url': streamer_url} - from engine.handler import CHECK_UPLOAD, CHECK + from ..engine.handler import CHECK_UPLOAD, CHECK event_manager.start() async def check_timer(): diff --git a/engine/downloader.py b/biliup/engine/downloader.py similarity index 90% rename from engine/downloader.py rename to biliup/engine/downloader.py index 58d8292..8709a85 100644 --- a/engine/downloader.py +++ b/biliup/engine/downloader.py @@ -3,8 +3,8 @@ import re import time from urllib.error import HTTPError -from common.decorators import Plugin -from engine.plugins import general, BatchCheckBase +from ..common.decorators import Plugin +from .plugins import general, BatchCheckBase logger = logging.getLogger('log01') diff --git a/engine/handler.py b/biliup/engine/handler.py similarity index 91% rename from engine/handler.py rename to biliup/engine/handler.py index 1405e62..a377998 100644 --- a/engine/handler.py +++ b/biliup/engine/handler.py @@ -1,9 +1,10 @@ -import common -from common import logger -from common.event import Event, event_manager -from engine.downloader import download, check_url -from engine.plugins.upload import UploadBase -from engine.uploader import upload +# import common +from .. import common +from ..common import logger +from ..common.event import Event, event_manager +from .downloader import download, check_url +from .plugins.upload import UploadBase +from .uploader import upload CHECK = 'check' CHECK_UPLOAD = 'check_upload' diff --git a/engine/plugins/__init__.py b/biliup/engine/plugins/__init__.py similarity index 100% rename from engine/plugins/__init__.py rename to biliup/engine/plugins/__init__.py diff --git a/engine/plugins/afreecaTV.py b/biliup/engine/plugins/afreecaTV.py similarity index 92% rename from engine/plugins/afreecaTV.py rename to biliup/engine/plugins/afreecaTV.py index b4718ed..7ad3097 100644 --- a/engine/plugins/afreecaTV.py +++ b/biliup/engine/plugins/afreecaTV.py @@ -1,8 +1,8 @@ import requests -from common.decorators import Plugin -from engine.plugins import match1, logger -from engine.plugins.base_adapter import FFmpegdl +from ...common.decorators import Plugin +from ..plugins import match1, logger +from .base_adapter import FFmpegdl VALID_URL_BASE = r"https?://play\.afreecatv\.com/(?P\w+)(?:/\d+)?" diff --git a/engine/plugins/base_adapter.py b/biliup/engine/plugins/base_adapter.py similarity index 97% rename from engine/plugins/base_adapter.py rename to biliup/engine/plugins/base_adapter.py index a84e330..edb32c1 100644 --- a/engine/plugins/base_adapter.py +++ b/biliup/engine/plugins/base_adapter.py @@ -7,8 +7,8 @@ from threading import Event import streamlink import youtube_dl -import engine -from engine.plugins import logger +from .. import config +from ..plugins import logger fake_headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -157,7 +157,7 @@ class FFmpegdl(DownloadBase): self.opt_args = [] self.default_output_args = [ '-bsf:a', 'aac_adtstoasc', - '-fs', f"{engine.config.get('file_size') if engine.config.get('file_size') else '2621440000'}" + '-fs', f"{config.get('file_size') if config.get('file_size') else '2621440000'}" ] self.default_input_args = ['-headers', ''.join('%s: %s\r\n' % x for x in fake_headers.items()), '-reconnect_streamed', '1', '-reconnect_delay_max', '20', '-rw_timeout', '20000000'] diff --git a/engine/plugins/bilibili.py b/biliup/engine/plugins/bilibili.py similarity index 86% rename from engine/plugins/bilibili.py rename to biliup/engine/plugins/bilibili.py index 2b57b76..d80e795 100644 --- a/engine/plugins/bilibili.py +++ b/biliup/engine/plugins/bilibili.py @@ -1,7 +1,7 @@ from ykdl.common import url_to_module -from common.decorators import Plugin -from engine.plugins.base_adapter import FFmpegdl +from ...common.decorators import Plugin +from .base_adapter import FFmpegdl @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m|live)\.)?bilibili\.com') diff --git a/engine/plugins/douyu.py b/biliup/engine/plugins/douyu.py similarity index 89% rename from engine/plugins/douyu.py rename to biliup/engine/plugins/douyu.py index 7034ebf..ca12c79 100644 --- a/engine/plugins/douyu.py +++ b/biliup/engine/plugins/douyu.py @@ -3,9 +3,9 @@ import platform from ykdl.common import url_to_module from ykdl.util.jsengine import chakra_available, quickjs_available, external_interpreter -from common.decorators import Plugin -from engine.plugins import logger -from engine.plugins.base_adapter import FFmpegdl +from ...common.decorators import Plugin +from ..plugins import logger +from .base_adapter import FFmpegdl @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m)\.)?douyu\.com') diff --git a/engine/plugins/general.py b/biliup/engine/plugins/general.py similarity index 91% rename from engine/plugins/general.py rename to biliup/engine/plugins/general.py index 6e0c0f6..16bf13c 100644 --- a/engine/plugins/general.py +++ b/biliup/engine/plugins/general.py @@ -1,7 +1,7 @@ from ykdl.common import url_to_module -from engine.plugins import logger -from engine.plugins.base_adapter import YDownload, SDownload, FFmpegdl +from ..plugins import logger +from .base_adapter import YDownload, SDownload, FFmpegdl class Generic(FFmpegdl): diff --git a/engine/plugins/huya.py b/biliup/engine/plugins/huya.py similarity index 85% rename from engine/plugins/huya.py rename to biliup/engine/plugins/huya.py index 7ef0e4c..1df1814 100644 --- a/engine/plugins/huya.py +++ b/biliup/engine/plugins/huya.py @@ -4,9 +4,9 @@ import json import requests -from common.decorators import Plugin -from engine.plugins import match1, logger -from engine.plugins.base_adapter import fake_headers, FFmpegdl +from ...common.decorators import Plugin +from ..plugins import match1, logger +from .base_adapter import fake_headers, FFmpegdl @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m)\.)?huya\.com') diff --git a/engine/plugins/twitch.py b/biliup/engine/plugins/twitch.py similarity index 96% rename from engine/plugins/twitch.py rename to biliup/engine/plugins/twitch.py index a071121..603b1e2 100644 --- a/engine/plugins/twitch.py +++ b/biliup/engine/plugins/twitch.py @@ -3,9 +3,9 @@ from urllib.parse import urlencode import requests -from common.decorators import Plugin -from engine.plugins import BatchCheckBase, match1 -from engine.plugins.base_adapter import FFmpegdl +from ...common.decorators import Plugin +from ..plugins import BatchCheckBase, match1 +from .base_adapter import FFmpegdl VALID_URL_BASE = r'(?:https?://)?(?:(?:www|go|m)\.)?twitch\.tv/(?P[0-9_a-zA-Z]+)' _OPERATION_HASHES = { diff --git a/engine/plugins/upload/__init__.py b/biliup/engine/plugins/upload/__init__.py similarity index 100% rename from engine/plugins/upload/__init__.py rename to biliup/engine/plugins/upload/__init__.py diff --git a/engine/plugins/upload/bili_chrome.py b/biliup/engine/plugins/upload/bili_chrome.py similarity index 95% rename from engine/plugins/upload/bili_chrome.py rename to biliup/engine/plugins/upload/bili_chrome.py index c715474..fef965a 100644 --- a/engine/plugins/upload/bili_chrome.py +++ b/biliup/engine/plugins/upload/bili_chrome.py @@ -9,10 +9,10 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait -import engine -from common.decorators import Plugin -from engine.plugins.upload import UploadBase, logger -from engine.plugins.upload.slider import slider_cracker +from ... import config +from ....common.decorators import Plugin +from ..upload import UploadBase, logger +from .slider import slider_cracker @Plugin.upload("bilibili") @@ -54,7 +54,7 @@ class BiliChrome(UploadBase): options = webdriver.ChromeOptions() options.add_argument('headless') - self.driver = webdriver.Chrome(executable_path=engine.config.get('chromedriver_path'), chrome_options=options) + self.driver = webdriver.Chrome(executable_path=config.get('chromedriver_path'), chrome_options=options) # service_log_path=service_log_path) try: self.driver.get("https://www.bilibili.com") @@ -112,9 +112,9 @@ class BiliChrome(UploadBase): WebDriverWait(self.driver, 10).until( ec.presence_of_element_located((By.XPATH, r'//*[@id="login-username"]'))) username = self.driver.find_element_by_xpath(r'//*[@id="login-username"]') - username.send_keys(engine.config['user']['account']['username']) + username.send_keys(config['user']['account']['username']) password = self.driver.find_element_by_xpath('//*[@id="login-passwd"]') - password.send_keys(engine.config['user']['account']['password']) + password.send_keys(config['user']['account']['password']) self.driver.find_element_by_class_name("btn-login").click() # logger.info('第四步') # try: diff --git a/engine/plugins/upload/bili_web.py b/biliup/engine/plugins/upload/bili_web.py similarity index 95% rename from engine/plugins/upload/bili_web.py rename to biliup/engine/plugins/upload/bili_web.py index 2fce29f..64bd162 100644 --- a/engine/plugins/upload/bili_web.py +++ b/biliup/engine/plugins/upload/bili_web.py @@ -19,15 +19,15 @@ import rsa from requests import utils from requests.adapters import HTTPAdapter, Retry -import engine -from common.decorators import Plugin -from engine.plugins.upload import UploadBase, logger +from ... import config +from ....common.decorators import Plugin +from ..upload import UploadBase, logger @Plugin.upload(platform="bili_web") class BiliWeb(UploadBase): def __init__(self, principal, data, user, lines='AUTO', threads=3, tid=174, tags=None, cover_path=None): - super().__init__(principal, data, persistence_path='engine/bili.cookie') + super().__init__(principal, data, persistence_path='bili.cookie') if tags is None: tags = ['星际争霸2', '电子竞技'] self.user = user @@ -134,13 +134,16 @@ class BiliBili: r = response.json() if r['code'] != 0 and r.get('data') is None: raise RuntimeError(r) - for cookie in r['data']['cookie_info']['cookies']: - self.__session.cookies.set(cookie['name'], cookie['value']) - if 'bili_jct' == cookie['name']: - self.__bili_jct = cookie['value'] - self.cookies = self.__session.cookies.get_dict() - self.access_token = r['data']['token_info']['access_token'] - self.refresh_token = r['data']['token_info']['refresh_token'] + try: + for cookie in r['data']['cookie_info']['cookies']: + self.__session.cookies.set(cookie['name'], cookie['value']) + if 'bili_jct' == cookie['name']: + self.__bili_jct = cookie['value'] + self.cookies = self.__session.cookies.get_dict() + self.access_token = r['data']['token_info']['access_token'] + self.refresh_token = r['data']['token_info']['refresh_token'] + except: + raise RuntimeError(r) return r def login_by_cookies(self, cookie): @@ -374,14 +377,14 @@ class BiliBili: logger.info(f'用户权重: {user_weight} => 网页端分p数量受到限制使用客户端api端提交') if not self.access_token: - self.login_by_password(**engine.config['user']['account']) + self.login_by_password(**config['user']['account']) self.store() while True: ret = self.__session.post(f'http://member.bilibili.com/x/vu/client/add?access_key={self.access_token}', timeout=5, json=asdict(self.video)).json() if ret['code'] == -101: logger.info(f'刷新token{ret}') - self.login_by_password(**engine.config['user']['account']) + self.login_by_password(**config['user']['account']) self.store() continue break diff --git a/engine/plugins/upload/slider/__init__.py b/biliup/engine/plugins/upload/slider/__init__.py similarity index 100% rename from engine/plugins/upload/slider/__init__.py rename to biliup/engine/plugins/upload/slider/__init__.py diff --git a/engine/uploader.py b/biliup/engine/uploader.py similarity index 89% rename from engine/uploader.py rename to biliup/engine/uploader.py index 1c907b3..4821aaf 100644 --- a/engine/uploader.py +++ b/biliup/engine/uploader.py @@ -1,8 +1,8 @@ import inspect -from common import logger -from common.decorators import Plugin -import engine +from .. import engine +from ..common import logger +from ..common.decorators import Plugin def upload(platform, index, data): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..07de284 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..65c382e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,28 @@ +[metadata] +name = biliup +version = 0.0.1 + +[options] +packages = find: +python_requires = >= 3.7 +install_requires = + selenium >= 3.8.1 + psutil >= 5.4.6 + youtube_dl >= 2018.11.18 + Pillow >= 5.2.0 + aiohttp[speedups] + Requests >= 2.20.0 + PyYAML >= 4.2b1 + streamlink >= 0.14.2 + ykdl @ git+https://github.com/zhangn1985/ykdl.git + rsa >= 4.6 + +[options.package_data] +biliup = *.ini + +[options.packages.find] +include = biliup* + +[options.entry_points] +console_scripts = + biliup = biliup:_main \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..864b617 --- /dev/null +++ b/setup.py @@ -0,0 +1,2 @@ +import setuptools +setuptools.setup() \ No newline at end of file