From c18d409d76a79f8f9430c192788dd30031c3cccf Mon Sep 17 00:00:00 2001 From: ForgQi <34411314+ForgQi@users.noreply.github.com> Date: Tue, 20 Apr 2021 16:09:04 +0800 Subject: [PATCH] Move plugins --- biliup/__init__.py | 62 ++++++++++++------- biliup/__main__.py | 27 +++++++- biliup/common/Daemon.py | 9 ++- biliup/{engine => }/downloader.py | 2 +- biliup/engine/__init__.py | 37 +---------- biliup/{common => engine}/decorators.py | 2 +- biliup/{common => engine}/event.py | 3 - biliup/{engine => }/handler.py | 8 +-- biliup/{engine => }/plugins/__init__.py | 0 biliup/{engine => }/plugins/afreecaTV.py | 2 +- biliup/{engine => }/plugins/bilibili.py | 2 +- biliup/{engine => }/plugins/cc.py | 2 +- biliup/{engine => }/plugins/douyu.py | 2 +- biliup/{engine => }/plugins/general.py | 4 +- biliup/{engine => }/plugins/huya.py | 2 +- biliup/{engine => }/plugins/twitch.py | 2 +- .../{engine => }/plugins/upload/__init__.py | 0 .../plugins/upload/bili_chrome.py | 4 +- .../{engine => }/plugins/upload/bili_web.py | 4 +- .../plugins/upload/slider/__init__.py | 0 biliup/{engine => }/uploader.py | 6 +- setup.cfg | 2 +- 22 files changed, 94 insertions(+), 88 deletions(-) rename biliup/{engine => }/downloader.py (95%) rename biliup/{common => engine}/decorators.py (98%) rename biliup/{common => engine}/event.py (99%) rename biliup/{engine => }/handler.py (95%) rename biliup/{engine => }/plugins/__init__.py (100%) rename biliup/{engine => }/plugins/afreecaTV.py (97%) rename biliup/{engine => }/plugins/bilibili.py (93%) rename biliup/{engine => }/plugins/cc.py (96%) rename biliup/{engine => }/plugins/douyu.py (96%) rename biliup/{engine => }/plugins/general.py (98%) rename biliup/{engine => }/plugins/huya.py (95%) rename biliup/{engine => }/plugins/twitch.py (98%) rename biliup/{engine => }/plugins/upload/__init__.py (100%) rename biliup/{engine => }/plugins/upload/bili_chrome.py (99%) rename biliup/{engine => }/plugins/upload/bili_web.py (99%) rename biliup/{engine => }/plugins/upload/slider/__init__.py (100%) rename biliup/{engine => }/uploader.py (89%) diff --git a/biliup/__init__.py b/biliup/__init__.py index 24f08f3..966e3ba 100644 --- a/biliup/__init__.py +++ b/biliup/__init__.py @@ -1,25 +1,43 @@ import asyncio -import platform -import sys -from .engine import main -from .common.Daemon import Daemon + +from .common.reload import AutoReload +from .common.timer import Timer + +from .engine.event import EventManager, Event +from .engine import config, invert_dict, Plugin +from . import plugins -def _main(): - daemon = Daemon('watch_process.pid') - if platform.system() != 'Windows' and len(sys.argv) == 2: - if 'start' == sys.argv[1]: - daemon.start() - elif 'stop' == sys.argv[1]: - daemon.stop() - elif 'restart' == sys.argv[1]: - daemon.restart() - else: - print('unknown command') - sys.exit(2) - sys.exit(0) - elif platform.system() == 'Windows' or len(sys.argv) == 1: - asyncio.run(main()) - else: - print('usage: %s start|stop|restart' % sys.argv[0]) - sys.exit(2) +def create_event_manager(): + streamer_url = {k: v['url'] for k, v in config['streamers'].items()} + inverted_index = invert_dict(streamer_url) + urls = list(inverted_index.keys()) + # 初始化事件管理器 + app = EventManager(config) + app.context['urls'] = urls + app.context['url_status'] = dict.fromkeys(inverted_index, 0) + app.context['checker'] = Plugin(plugins).sorted_checker(urls) + app.context['inverted_index'] = inverted_index + app.context['streamer_url'] = streamer_url + return app + + +event_manager = create_event_manager() + + +async def main(): + from .handler import CHECK_UPLOAD, CHECK + + event_manager.start() + + async def check_timer(): + event_manager.send_event(Event(CHECK_UPLOAD)) + for k in event_manager.context['checker'].keys(): + event_manager.send_event(Event(CHECK, (k,))) + + # 初始化定时器 + timer = Timer(func=check_timer, interval=40) + + # 模块更新自动重启 + detector = AutoReload(event_manager, timer, interval=15) + await asyncio.gather(detector.astart(), timer.astart(), return_exceptions=True) diff --git a/biliup/__main__.py b/biliup/__main__.py index 7fe482d..7a0ddc8 100644 --- a/biliup/__main__.py +++ b/biliup/__main__.py @@ -1,6 +1,31 @@ #!/usr/bin/python3 # coding:utf8 -from . import _main +import platform +import sys +import asyncio + +from .common.Daemon import Daemon +from . import main + + +def _main(): + daemon = Daemon('watch_process.pid', main) + if platform.system() != 'Windows' and len(sys.argv) == 2: + if 'start' == sys.argv[1]: + daemon.start() + elif 'stop' == sys.argv[1]: + daemon.stop() + elif 'restart' == sys.argv[1]: + daemon.restart() + else: + print('unknown command') + sys.exit(2) + sys.exit(0) + elif platform.system() == 'Windows' or len(sys.argv) == 1: + asyncio.run(main()) + else: + print('usage: %s start|stop|restart' % sys.argv[0]) + sys.exit(2) if __name__ == '__main__': diff --git a/biliup/common/Daemon.py b/biliup/common/Daemon.py index 4724768..597bc76 100644 --- a/biliup/common/Daemon.py +++ b/biliup/common/Daemon.py @@ -5,20 +5,20 @@ import sys import os import time import atexit -from ..engine import main logger = logging.getLogger('log01') # python模拟linux的守护进程 class Daemon(object): - def __init__(self, pidfile, change_currentdirectory=False, stdin='/dev/null', stdout='/dev/null', + def __init__(self, pidfile, fn, change_currentdirectory=False, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # 需要获取调试信息,改为stdin='/dev/stdin', stdout='/dev/stdout', stderr='/dev/stderr',以root身份运行。 self.stdin = stdin self.stdout = stdout self.stderr = stderr self.pidfile = pidfile + self.fn = fn self.cd = change_currentdirectory def _daemonize(self): @@ -119,7 +119,6 @@ class Daemon(object): self.stop() self.start() - @staticmethod - def _run(): + def _run(self): """ run your fun""" - asyncio.run(main()) + asyncio.run(self.fn()) diff --git a/biliup/engine/downloader.py b/biliup/downloader.py similarity index 95% rename from biliup/engine/downloader.py rename to biliup/downloader.py index 8709a85..ca8b830 100644 --- a/biliup/engine/downloader.py +++ b/biliup/downloader.py @@ -3,7 +3,7 @@ import re import time from urllib.error import HTTPError -from ..common.decorators import Plugin +from .engine.decorators import Plugin from .plugins import general, BatchCheckBase logger = logging.getLogger('log01') diff --git a/biliup/engine/__init__.py b/biliup/engine/__init__.py index e4f4c0c..f0c48da 100644 --- a/biliup/engine/__init__.py +++ b/biliup/engine/__init__.py @@ -1,11 +1,5 @@ -import asyncio - import yaml - -from ..common.decorators import Plugin -from ..common.event import Event, event_manager -from ..common.reload import AutoReload -from ..common.timer import Timer +from .decorators import Plugin with open(r'config.yaml', encoding='utf-8') as stream: config = yaml.load(stream, Loader=yaml.FullLoader) @@ -19,31 +13,4 @@ def invert_dict(d: dict): return inverse_dict -async def main(): - from . import plugins - streamers = config['streamers'] - streamer_url = {k: v['url'] for k, v in streamers.items()} - inverted_index = invert_dict(streamer_url) - urls = list(inverted_index.keys()) - url_status = dict.fromkeys(inverted_index, 0) - checker = Plugin(plugins).sorted_checker(urls) - # 初始化事件管理器 - event_manager.context = {**config, 'urls': urls, 'url_status': url_status, - 'checker': checker, 'inverted_index': inverted_index, 'streamer_url': streamer_url} - from ..engine.handler import CHECK_UPLOAD, CHECK - event_manager.start() - - async def check_timer(): - event_manager.send_event(Event(CHECK_UPLOAD)) - for k in checker.keys(): - event_manager.send_event(Event(CHECK, (k,))) - - # 初始化定时器 - timer = Timer(func=check_timer, interval=40) - - # 模块更新自动重启 - detector = AutoReload(event_manager, timer, interval=15) - await asyncio.gather(detector.astart(), timer.astart(), return_exceptions=True) - - -__all__ = ['downloader', 'uploader', 'plugins', 'main'] +__all__ = ['config', 'invert_dict', 'Plugin'] diff --git a/biliup/common/decorators.py b/biliup/engine/decorators.py similarity index 98% rename from biliup/common/decorators.py rename to biliup/engine/decorators.py index 9499634..5bc9ca6 100644 --- a/biliup/common/decorators.py +++ b/biliup/engine/decorators.py @@ -40,7 +40,7 @@ class Plugin: @classmethod def sorted_checker(cls, urls): - from ..engine.plugins import general + from ..plugins import general curls = urls.copy() checker_plugins = {} for plugin in cls.download_plugins: diff --git a/biliup/common/event.py b/biliup/engine/event.py similarity index 99% rename from biliup/common/event.py rename to biliup/engine/event.py index 6596cf2..3510730 100644 --- a/biliup/common/event.py +++ b/biliup/engine/event.py @@ -162,6 +162,3 @@ class Event: type_: str # 事件类型 args: tuple = () dict: dict = field(default_factory=dict) # 字典用于保存具体的事件数据 - - -event_manager = EventManager() diff --git a/biliup/engine/handler.py b/biliup/handler.py similarity index 95% rename from biliup/engine/handler.py rename to biliup/handler.py index a377998..75bc4bb 100644 --- a/biliup/engine/handler.py +++ b/biliup/handler.py @@ -1,7 +1,7 @@ -# import common -from .. import common -from ..common import logger -from ..common.event import Event, event_manager +from . import common +from biliup import event_manager +from .common import logger +from .engine.event import Event from .downloader import download, check_url from .plugins.upload import UploadBase from .uploader import upload diff --git a/biliup/engine/plugins/__init__.py b/biliup/plugins/__init__.py similarity index 100% rename from biliup/engine/plugins/__init__.py rename to biliup/plugins/__init__.py diff --git a/biliup/engine/plugins/afreecaTV.py b/biliup/plugins/afreecaTV.py similarity index 97% rename from biliup/engine/plugins/afreecaTV.py rename to biliup/plugins/afreecaTV.py index ac4c203..215229c 100644 --- a/biliup/engine/plugins/afreecaTV.py +++ b/biliup/plugins/afreecaTV.py @@ -1,6 +1,6 @@ import requests -from ...common.decorators import Plugin +from ..engine.decorators import Plugin from ..plugins import match1, logger from .general import DownloadBase diff --git a/biliup/engine/plugins/bilibili.py b/biliup/plugins/bilibili.py similarity index 93% rename from biliup/engine/plugins/bilibili.py rename to biliup/plugins/bilibili.py index b5f0fd5..e1a7b3d 100644 --- a/biliup/engine/plugins/bilibili.py +++ b/biliup/plugins/bilibili.py @@ -1,6 +1,6 @@ from ykdl.common import url_to_module -from ...common.decorators import Plugin +from ..engine.decorators import Plugin from .general import DownloadBase diff --git a/biliup/engine/plugins/cc.py b/biliup/plugins/cc.py similarity index 96% rename from biliup/engine/plugins/cc.py rename to biliup/plugins/cc.py index cc725a3..b9bd4d3 100644 --- a/biliup/engine/plugins/cc.py +++ b/biliup/plugins/cc.py @@ -3,7 +3,7 @@ import re import requests -from ...common.decorators import Plugin +from ..engine.decorators import Plugin from . import logger, fake_headers from .general import DownloadBase diff --git a/biliup/engine/plugins/douyu.py b/biliup/plugins/douyu.py similarity index 96% rename from biliup/engine/plugins/douyu.py rename to biliup/plugins/douyu.py index b06661a..93a4a81 100644 --- a/biliup/engine/plugins/douyu.py +++ b/biliup/plugins/douyu.py @@ -3,7 +3,7 @@ import platform from ykdl.common import url_to_module from ykdl.util.jsengine import chakra_available, quickjs_available, external_interpreter -from ...common.decorators import Plugin +from ..engine.decorators import Plugin from ..plugins import logger from .general import DownloadBase diff --git a/biliup/engine/plugins/general.py b/biliup/plugins/general.py similarity index 98% rename from biliup/engine/plugins/general.py rename to biliup/plugins/general.py index ea9de40..9b3258f 100644 --- a/biliup/engine/plugins/general.py +++ b/biliup/plugins/general.py @@ -8,8 +8,8 @@ import streamlink import youtube_dl from ykdl.common import url_to_module -from .. import config -from ..plugins import logger, fake_headers +from ..engine import config +from . import logger, fake_headers class DownloadBase: diff --git a/biliup/engine/plugins/huya.py b/biliup/plugins/huya.py similarity index 95% rename from biliup/engine/plugins/huya.py rename to biliup/plugins/huya.py index 81dfc08..2c2a430 100644 --- a/biliup/engine/plugins/huya.py +++ b/biliup/plugins/huya.py @@ -4,7 +4,7 @@ import json import requests -from ...common.decorators import Plugin +from ..engine.decorators import Plugin from ..plugins import match1, logger, fake_headers from .general import DownloadBase diff --git a/biliup/engine/plugins/twitch.py b/biliup/plugins/twitch.py similarity index 98% rename from biliup/engine/plugins/twitch.py rename to biliup/plugins/twitch.py index b74fdbc..291fd6e 100644 --- a/biliup/engine/plugins/twitch.py +++ b/biliup/plugins/twitch.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode import requests -from ...common.decorators import Plugin +from ..engine.decorators import Plugin from ..plugins import BatchCheckBase, match1 from .general import DownloadBase diff --git a/biliup/engine/plugins/upload/__init__.py b/biliup/plugins/upload/__init__.py similarity index 100% rename from biliup/engine/plugins/upload/__init__.py rename to biliup/plugins/upload/__init__.py diff --git a/biliup/engine/plugins/upload/bili_chrome.py b/biliup/plugins/upload/bili_chrome.py similarity index 99% rename from biliup/engine/plugins/upload/bili_chrome.py rename to biliup/plugins/upload/bili_chrome.py index fef965a..7b65ced 100644 --- a/biliup/engine/plugins/upload/bili_chrome.py +++ b/biliup/plugins/upload/bili_chrome.py @@ -9,8 +9,8 @@ from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait -from ... import config -from ....common.decorators import Plugin +from ...engine import config +from ...engine.decorators import Plugin from ..upload import UploadBase, logger from .slider import slider_cracker diff --git a/biliup/engine/plugins/upload/bili_web.py b/biliup/plugins/upload/bili_web.py similarity index 99% rename from biliup/engine/plugins/upload/bili_web.py rename to biliup/plugins/upload/bili_web.py index 9b7b28e..73ec773 100644 --- a/biliup/engine/plugins/upload/bili_web.py +++ b/biliup/plugins/upload/bili_web.py @@ -19,8 +19,8 @@ import rsa from requests import utils from requests.adapters import HTTPAdapter, Retry -from ... import config -from ....common.decorators import Plugin +from ...engine import config +from ...engine.decorators import Plugin from ..upload import UploadBase, logger diff --git a/biliup/engine/plugins/upload/slider/__init__.py b/biliup/plugins/upload/slider/__init__.py similarity index 100% rename from biliup/engine/plugins/upload/slider/__init__.py rename to biliup/plugins/upload/slider/__init__.py diff --git a/biliup/engine/uploader.py b/biliup/uploader.py similarity index 89% rename from biliup/engine/uploader.py rename to biliup/uploader.py index 4821aaf..3f2ab69 100644 --- a/biliup/engine/uploader.py +++ b/biliup/uploader.py @@ -1,8 +1,8 @@ import inspect -from .. import engine -from ..common import logger -from ..common.decorators import Plugin +from biliup import engine +from .common import logger +from .engine.decorators import Plugin def upload(platform, index, data): diff --git a/setup.cfg b/setup.cfg index 84adf86..9d43b9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,4 +32,4 @@ include = biliup* [options.entry_points] console_scripts = - biliup = biliup:_main \ No newline at end of file + biliup = biliup.__main__:_main \ No newline at end of file