From bb253b3df9da6411cb9e7bd5e5bcf015ba5a6dca Mon Sep 17 00:00:00 2001 From: ForgQi <34411314+ForgQi@users.noreply.github.com> Date: Sat, 17 Nov 2018 11:20:41 +0800 Subject: [PATCH] add decorator --- Bilibili.py | 6 +- Engine/__init__.py | 93 --------------- Engine/kernel.py | 155 ------------------------- common/Daemon.py | 5 +- common/DesignPattern.py | 25 ---- common/configlog.ini | 4 +- common/event.py | 76 +++++++++++- common/timer.py | 2 + engine/__init__.py | 52 +++++++++ {Engine => engine}/downloader.py | 26 ++--- engine/handler.py | 154 ++++++++++++++++++++++++ {Engine => engine}/plugins/__init__.py | 0 {Engine => engine}/plugins/douyu.py | 4 +- {Engine => engine}/plugins/general.py | 2 +- {Engine => engine}/plugins/huya.py | 8 +- {Engine => engine}/plugins/panda.py | 2 +- {Engine => engine}/plugins/twitch.py | 60 +++++----- {Engine => engine}/slider/__init__.py | 0 {Engine => engine}/upload.py | 16 +-- requirements.txt | 2 +- 20 files changed, 350 insertions(+), 342 deletions(-) delete mode 100644 Engine/__init__.py delete mode 100644 Engine/kernel.py delete mode 100644 common/DesignPattern.py create mode 100644 engine/__init__.py rename {Engine => engine}/downloader.py (79%) create mode 100644 engine/handler.py rename {Engine => engine}/plugins/__init__.py (100%) rename {Engine => engine}/plugins/douyu.py (96%) rename {Engine => engine}/plugins/general.py (61%) rename {Engine => engine}/plugins/huya.py (85%) rename {Engine => engine}/plugins/panda.py (97%) rename {Engine => engine}/plugins/twitch.py (60%) rename {Engine => engine}/slider/__init__.py (100%) rename {Engine => engine}/upload.py (97%) diff --git a/Bilibili.py b/Bilibili.py index 398af28..d786f18 100644 --- a/Bilibili.py +++ b/Bilibili.py @@ -1,10 +1,12 @@ #!/usr/bin/python3 import sys import common -from Engine import main +from engine.handler import event_manager +from engine import main from common.Daemon import Daemon if __name__ == '__main__': + sys.excepthook = common.new_hook daemon = Daemon('watch_process.pid') @@ -20,7 +22,7 @@ if __name__ == '__main__': sys.exit(2) sys.exit(0) elif len(sys.argv) == 1: - main() + main(event_manager) else: print('usage: %s start|stop|restart' % sys.argv[0]) sys.exit(2) diff --git a/Engine/__init__.py b/Engine/__init__.py deleted file mode 100644 index a04df4d..0000000 --- a/Engine/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -from functools import partial -from multiprocessing.pool import Pool - -import yaml - -import common -from Engine.downloader import Extractor -from Engine.kernel import CallBack, modify, process, revise, free_upload, \ - callback2, all_check -from common.DesignPattern import Service -from common.event import Event -from common.reload import autoreload -from common.timer import Timer - -CHECK = 'check' -TO_MODIFY = 'to_modify' -DOWNLOAD_UPLOAD = 'download_upload' -BE_MODIFIED = 'be_modified' -UPLOAD = 'upload' - -# print(Urls, url_status, url_status_base) - -# def get_queue(q): -# process.q = q -with open(r'config.yaml', encoding='utf-8') as stream: - config = yaml.load(stream) - links_id = config['links_id'] - user_name = config['user_name'] - pass_word = config['pass_word'] - chromedrive_path = config['chromedrive_path'] - - -def getmany(): - urls = [] - urlstatus = {} - for k, v in links_id.items(): - urls += v - for url in v: - urlstatus[url] = 0 - return urls, urlstatus, urlstatus.copy() - - -Urls, url_status, url_status_base = getmany() -batches, onebyone = Extractor().sorted_checker(Urls) - - -def main(): - # pool = Pool(3, initializer=get_queue, initargs=(queue,)) - pool = Pool(3) - service_p = partial(Service, pool) - - # 初始化事件管理器 - event_manager = common.event.EventManager() - - # 初始化定时器 - timer_ = Timer(func=event_manager.send_event, args=(Event(CHECK),), interval=40) - - # 模块更新自动重启 - autoreload(event_manager, timer_, interval=15) - - # 监听器 - # check = Event(CHECK) - # ty = Event(TO_MODIFY) - # dd = Event(DOWNLOAD_UPLOAD) - # bd = Event(BE_MODIFIED) - # ud = Event(UPLOAD) - callback_2 = partial(callback2, event_manager) - - service_check = service_p(all_check, callback_2) - modify_p = partial(modify, event_manager) - service_download = service_p(process, CallBack(event_manager, Event(BE_MODIFIED)).send) - # service_upload = service_p(free_upload, None) - upload_p = partial(free_upload, event_manager) - - # 批量注册事件 - # Engine.downloader.Batch(event_manager, links_id).register() - - # 绑定事件和监听器响应函数 - event_manager.add_event_listener(CHECK, service_check.start) - event_manager.add_event_listener(TO_MODIFY, modify_p) - event_manager.add_event_listener(DOWNLOAD_UPLOAD, service_download.start) - event_manager.add_event_listener(BE_MODIFIED, revise) - event_manager.add_event_listener(UPLOAD, upload_p) - - event_manager.start() - timer_.start() - - # 添加事件队列 - # event_queue(links_id, queue) - # 定时推送事件 - - -__all__ = ['downloader', 'upload', 'plugins', 'main', 'links_id'] diff --git a/Engine/kernel.py b/Engine/kernel.py deleted file mode 100644 index 7494d92..0000000 --- a/Engine/kernel.py +++ /dev/null @@ -1,155 +0,0 @@ -import time -import requests -import Engine -import common -from Engine.downloader import download -from Engine.upload import Upload -from common import logger -from common.event import Event - - -def all_check(): - live = [] - for batch in Engine.batches: - try: - res = batch.check() - if res: - live += res - except requests.exceptions.ReadTimeout as timeout: - logger.error(batch.__class__.__module__ + ',ReadTimeout:' + str(timeout)) - except requests.exceptions.SSLError as sslerr: - logger.error(batch.__class__.__module__ + ',SSLError:' + str(sslerr)) - except requests.exceptions.ConnectionError as connerr: - logger.error(batch.__class__.__module__ + ',ConnectionError:' + str(connerr)) - except requests.exceptions.RequestException: - logger.exception(batch.__class__.__module__) - - for one in Engine.onebyone: - for url in one.__plugin__.url_list: - try: - if one.__plugin__('检测' + url, url).check_stream(): - live += [url] - except requests.exceptions.ReadTimeout as timeout: - logger.error(one.__plugin__.__module__ + ',ReadTimeout:' + str(timeout)) - except requests.exceptions.ConnectTimeout as timeout: - logger.error(one.__plugin__.__module__ + ',ConnectTimeout:' + str(timeout)) - except requests.exceptions.ConnectionError as connerr: - logger.error(one.__plugin__.__module__ + ',ConnectionError:' + str(connerr)) - except requests.exceptions.ChunkedEncodingError as ceer: - logger.error(one.__plugin__.__module__ + ',ChunkedEncodingError:' + str(ceer)) - except requests.exceptions.RequestException: - logger.exception(one.__plugin__.__module__) - - if url != one.__plugin__.url_list[-1]: - logger.debug('歇息会') - time.sleep(15) - return live - - -class CallBack(object): - def __init__(self, event_manager, event): - self.event_manager = event_manager - self.event = event - - def send(self, result): - # if result: - self.event.args = (result,) - self.event_manager.send_event(self.event) - - -def callback2(event_manager, result): - CallBack(event_manager, Event('to_modify')).send(result) - CallBack(event_manager, Event('upload')).send(result) - - -def modify(event_manager, live_m): - live_m, = live_m.args - live_d = {} - # global url_status - # print(live_m) - if live_m: - for live in live_m: - # print(live) - if Engine.url_status[live] == 1: - pass - # print('已开播正在下载') - else: - # print('刚刚开播，去下载') - - name = find_name(live) - - event_d = Event('download_upload') - event_d.args = (name, live, 'dl') - - event_manager.send_event(event_d) - # self.__run(live) - live_d[live] = 1 - Engine.url_status.update(live_d) - # url_status = {**url_status_base, **live_d} - else: - logger.debug('无人直播') - - -def process(name, url, mod): - try: - now = common.time_now() - if mod == 'dl': - download(name, url) - Upload(name).start(url, now) - elif mod == 'up': - Upload(name).start(url, now) - else: - return url - finally: - return url - - -def free(list_url): - status_num = list(map(lambda x: Engine.url_status.get(x), list_url)) - # print(status_num) - if 1 in status_num or 2 in status_num: - return False - else: - return True - - -def free_upload(event_manager, urls): - # urls, = urls.args - # print(urlstatus) - # try: - logger.debug(urls) - for title, v in Engine.links_id.items(): - # names = list(map(find_name, urls)) - url = v[0] - # if title not in names and url_status[url] == 0 and Upload(title, url).filter_file(): - if free(v) and Upload(title).filter_file(): - event_d = Event('download_upload') - event_d.args = (title, url, 'up') - - event_manager.send_event(event_d) - Engine.url_status[url] = 2 - # print('up') - - # Upload(title, url).start() - - # except: - # logger.exception() - # print('寻找结束') - - -def revise(url): - # if type(url) is dict: - # url = url['result'] - url, = url.args - if url: - # global url_status - # url_status = {**url_status, **{url: 0}} - Engine.url_status.update({url: 0}) - # print('更新字典') - # print(url_status) - - -def find_name(url): - for name in Engine.links_id: - if url in Engine.links_id[name]: - return name diff --git a/common/Daemon.py b/common/Daemon.py index 71935ef..245ba57 100644 --- a/common/Daemon.py +++ b/common/Daemon.py @@ -4,7 +4,8 @@ import sys import os import time import atexit -from Engine import main +from engine import main +from engine.handler import event_manager logger = logging.getLogger('log01') @@ -121,7 +122,7 @@ class Daemon(object): @staticmethod def _run(): """ run your fun""" - main() + main(event_manager) # if __name__ == '__main__': diff --git a/common/DesignPattern.py b/common/DesignPattern.py deleted file mode 100644 index ff7adc0..0000000 --- a/common/DesignPattern.py +++ /dev/null @@ -1,25 +0,0 @@ -def singleton(cls): - instance = cls() - cls.__call__ = lambda self: instance - return instance - - -# 用来包装需要多进程的函数（多进程执行避免主进程阻塞） -class Service: - def __init__(self, pool, func, callback): - # 事件处理进程池 - self.__pool = pool - self.__func = func - self.callback = callback - - def __run(self, args): - self.__pool.apply_async(func=self.__func, args=args, callback=self.callback) - # self.__pool.apply(func=self.__func, args=args) - - def start(self, event): - args = event.args - self.__run(args) - - def stop(self): - self.__pool.close() - self.__pool.join() diff --git a/common/configlog.ini b/common/configlog.ini index 223743b..e4666e1 100644 --- a/common/configlog.ini +++ b/common/configlog.ini @@ -31,9 +31,9 @@ args=("ds_update.log", "W0", 1, 1) keys=form01,form02 [formatter_form01] -format=%(asctime)s %(filename)s[line:%(lineno)d](Pid:%(process)s) %(levelname)s %(message)s +format=%(asctime)s %(filename)s[line:%(lineno)d](Pid:%(process)d Tid:%(thread)d) %(levelname)s %(message)s [formatter_form02] -format=%(filename)s%(lineno)d[%(levelname)s]%(message)s +format=%(filename)s%(lineno)d[%(levelname)s]Tname:%(threadName)s %(message)s diff --git a/common/event.py b/common/event.py index 19ce40a..4dac781 100644 --- a/common/event.py +++ b/common/event.py @@ -1,7 +1,10 @@ # encoding: UTF-8 # 系统模块 +import inspect +from concurrent.futures.thread import ThreadPoolExecutor from queue import Queue, Empty from threading import * +import functools class EventManager: @@ -13,11 +16,17 @@ class EventManager: self.__active = False # 事件处理线程 self.__thread = Thread(target=self.__run) + # 事件处理线程池 + self.__pool = ThreadPoolExecutor(3) + # 阻塞函数列表 + self.__block = [] # 这里的__handlers是一个字典，用来保存对应的事件的响应函数 # 其中每个键对应的值是一个列表，列表中保存了对该事件监听的响应函数，一对多 self.__handlers = {} + self.__method = {} + def __run(self): """引擎运行""" while self.__active is True: @@ -34,7 +43,10 @@ class EventManager: if event.type_ in self.__handlers: # 若存在，则按顺序将事件传递给处理函数执行 for handler in self.__handlers[event.type_]: - handler(event) + if handler.__qualname__ in self.__block: + self.__pool.submit(handler, event) + else: + handler(event) def start(self): """启动""" @@ -47,9 +59,11 @@ class EventManager: """停止""" # 将事件管理器设为停止 self.__active = False + self.__pool.shutdown() # 等待事件处理线程退出 self.__thread.join() + def add_event_listener(self, type_, handler): """绑定事件和监听器处理函数""" # 尝试获取该事件类型对应的处理函数列表，若无则创建 @@ -83,10 +97,66 @@ class EventManager: """发送事件，向事件队列中存入事件""" self.__eventQueue.put(event) + def register(self, type_, block=False): + classname = inspect.getouterframes(inspect.currentframe())[1][3] + + def callback(result): + if not result: + pass + elif isinstance(result, tuple): + for event in result: + self.send_event(event) + else: + self.send_event(result) + + def appendblock(fc, blk): + if blk: + self.__block.append(fc.__qualname__) + + if classname == '': + def decorator(func): + appendblock(func, block) + + @functools.wraps(func) + def wrapper(event): + _event = func(*event.args) + callback(_event) + return _event + + self.add_event_listener(type_, wrapper) + + return wrapper + else: + def decorator(func): + appendblock(func, block) + + self.__method.setdefault(type_, []) + self.__method[type_].append(func.__name__) + + @functools.wraps(func) + def wrapper(this, event): + _event = func(this, *event.args) + callback(_event) + return _event + + return wrapper + return decorator + + def server(self, *args): + def decorator(cls): + for type_ in self.__method: + for handler in self.__method[type_]: + self.add_event_listener(type_, getattr(cls(*args), handler)) + self.__method = {} + return cls + + return decorator + class Event: """事件对象""" + def __init__(self, type_=None): - self.type_ = type_ # 事件类型 - self.dict = {} # 字典用于保存具体的事件数据 + self.type_ = type_ # 事件类型 + self.dict = {} # 字典用于保存具体的事件数据 self.args = () diff --git a/common/timer.py b/common/timer.py index cce9b97..4beb84a 100644 --- a/common/timer.py +++ b/common/timer.py @@ -1,5 +1,7 @@ import threading +from common import logger + class Timer(object): def __init__(self, func, args=(), kwargs=None, interval=15): diff --git a/engine/__init__.py b/engine/__init__.py new file mode 100644 index 0000000..b08baef --- /dev/null +++ b/engine/__init__.py @@ -0,0 +1,52 @@ +import yaml +from common.event import Event +from common.reload import autoreload +from common.timer import Timer + + +def getmany(_links_id): + _urls = [] + urlstatus = {} + for k, v in _links_id.items(): + _urls += v + for url in v: + urlstatus[url] = 0 + return _urls, urlstatus, urlstatus.copy() + + +def find_name(url): + for name in links_id: + if url in links_id[name]: + return name + + +def main(event_manager): + # 初始化定时器 + timer_ = Timer(func=event_manager.send_event, args=(Event(CHECK),), interval=40) + + # 模块更新自动重启 + autoreload(event_manager, timer_, interval=15) + + event_manager.start() + timer_.start() + + +CHECK = 'check' +TO_MODIFY = 'to_modify' +DOWNLOAD_UPLOAD = 'download_upload' +BE_MODIFIED = 'be_modified' +UPLOAD = 'upload' + +# def get_queue(q): +# process.q = q +with open(r'config.yaml', encoding='utf-8') as stream: + config = yaml.load(stream) + links_id = config['links_id'] + user_name = config['user_name'] + pass_word = config['pass_word'] + chromedrive_path = config['chromedrive_path'] + +urls, url_status, url_status_base = getmany(links_id) +__all__ = ['downloader', 'upload', 'plugins', 'main', + 'urls', 'url_status', 'url_status_base', + 'CHECK', 'BE_MODIFIED', 'DOWNLOAD_UPLOAD', 'TO_MODIFY', 'UPLOAD'] diff --git a/Engine/downloader.py b/engine/downloader.py similarity index 79% rename from Engine/downloader.py rename to engine/downloader.py index 7bdb3da..1058771 100644 --- a/Engine/downloader.py +++ b/engine/downloader.py @@ -1,20 +1,20 @@ import pkgutil import importlib import re -import Engine.plugins.douyu -from Engine.plugins import general -from common.DesignPattern import singleton +import engine.plugins.douyu +from engine.plugins import general +# from common.DesignPattern import singleton -@singleton +# @singleton class Extractor(object): def __init__(self): self.plugins = [] - self.load_plugins(Engine.plugins.__path__[0]) + self.load_plugins(engine.plugins.__path__[0]) - def get_plugins(self): - # self.load_plugins(Engine.plugins.__path__[0]) - return self.plugins + # def get_plugins(self): + # # self.load_plugins(engine.plugins.__path__[0]) + # return self.plugins def load_plugins(self, path): """Attempt to load plugins from the path specified. @@ -24,7 +24,7 @@ class Extractor(object): """ for loader, name, ispkg in pkgutil.iter_modules([path]): # set the full plugin module name - module_name = "Engine.plugins.{0}".format(name) + module_name = "engine.plugins.{0}".format(name) # print(module_name) self.load_plugin(module_name) # print(self.plugins) @@ -61,16 +61,16 @@ class Extractor(object): if hasattr(plugin, signature): batches.append(plugin.BatchCheck(plugin.__plugin__.url_list)) else: - onebyone.append(plugin) + onebyone.append(plugin.__plugin__) general.__plugin__.url_list = curls - onebyone.append(general) - # onebyone.append(__import__('Engine.plugins.general', fromlist=['general',])) + onebyone.append(general.__plugin__) + # onebyone.append(__import__('engine.plugins.general', fromlist=['general',])) return batches, onebyone def download(fname, url): extractor = Extractor() - plugins = extractor.get_plugins() + plugins = extractor.plugins for plugin in plugins: if hasattr(plugin, "VALID_URL_BASE"): if re.match(plugin.VALID_URL_BASE, url): diff --git a/engine/handler.py b/engine/handler.py new file mode 100644 index 0000000..9879f68 --- /dev/null +++ b/engine/handler.py @@ -0,0 +1,154 @@ +import multiprocessing +import time +import requests +import engine +import common +from engine import CHECK, BE_MODIFIED, DOWNLOAD_UPLOAD, TO_MODIFY, UPLOAD, urls, url_status, url_status_base +from engine.downloader import download, Extractor +from engine.upload import Upload +from common import logger +from common.event import Event + +# 初始化事件管理器 + +event_manager = common.event.EventManager() + + +@event_manager.register(CHECK, block=True) +def all_check(): + batches, onebyone = Extractor().sorted_checker(urls) + live = [] + try: + for batch in batches: + try: + res = batch.check() + if res: + live += res + except requests.exceptions.ReadTimeout as timeout: + logger.error(batch.__class__.__module__ + ',ReadTimeout:' + str(timeout)) + except requests.exceptions.SSLError as sslerr: + logger.error(batch.__class__.__module__ + ',SSLError:' + str(sslerr)) + except requests.exceptions.ConnectionError as connerr: + logger.error(batch.__class__.__module__ + ',ConnectionError:' + str(connerr)) + except requests.exceptions.RequestException: + logger.exception(batch.__class__.__module__) + + for one in onebyone: + for url in one.url_list: + try: + if one('检测' + url, url).check_stream(): + live += [url] + except requests.exceptions.ReadTimeout as timeout: + logger.error(one.__plugin__.__module__ + ',ReadTimeout:' + str(timeout)) + except requests.exceptions.ConnectTimeout as timeout: + logger.error(one.__plugin__.__module__ + ',ConnectTimeout:' + str(timeout)) + except requests.exceptions.ConnectionError as connerr: + logger.error(one.__plugin__.__module__ + ',ConnectionError:' + str(connerr)) + except requests.exceptions.ChunkedEncodingError as ceer: + logger.error(one.__plugin__.__module__ + ',ChunkedEncodingError:' + str(ceer)) + except requests.exceptions.RequestException: + logger.exception(one.__plugin__.__module__) + + if url != one.url_list[-1]: + logger.debug('歇息会') + time.sleep(15) + finally: + event_t = Event(TO_MODIFY) + event_t.args = (live,) + event_u = Event(UPLOAD) + event_u.args = (live,) + return event_u, event_t + + +@event_manager.register(DOWNLOAD_UPLOAD, block=True) +def process(name, url, mod): + try: + now = common.time_now() + if mod == 'dl': + p = multiprocessing.Process(target=download, args=(name, url)) + p.start() + p.join() + # download(name, url) + Upload(name).start(url, now) + elif mod == 'up': + Upload(name).start(url, now) + else: + return url + finally: + event = Event(BE_MODIFIED) + event.args = (url,) + # return url + return event + + +@event_manager.server(urls, url_status, url_status_base) +class KernelFunc: + def __init__(self, _urls, _url_status, _url_status_base): + self.urls = _urls + self.url_status = _url_status + self.url_status_base = _url_status_base + + @event_manager.register(engine.TO_MODIFY) + def modify(self, live_m): + live_d = {} + # print(live_m) + if live_m: + event = [] + for live in live_m: + if self.url_status[live] == 1: + # 已开播正在下载 + print('已开播正在下载') + pass + else: + name = engine.find_name(live) + logger.debug(name + '刚刚开播，去下载') + event_d = Event(DOWNLOAD_UPLOAD) + event_d.args = (name, live, 'dl') + event.append(event_d) + + live_d[live] = 1 + self.url_status.update(live_d) + # url_status = {**url_status_base, **live_d} + return tuple(event) + + else: + logger.debug('无人直播') + + def free(self, list_url): + status_num = list(map(lambda x: self.url_status.get(x), list_url)) + # print(status_num) + if 1 in status_num or 2 in status_num: + return False + else: + return True + + @event_manager.register(engine.UPLOAD) + def free_upload(self, _urls): + logger.debug(_urls) + event = [] + for title, v in engine.links_id.items(): + # names = list(map(find_name, urls)) + url = v[0] + # if title not in names and url_status[url] == 0 and Upload(title, url).filter_file(): + if self.free(v) and Upload(title).filter_file(): + event_d = Event(DOWNLOAD_UPLOAD) + event_d.args = (title, url, 'up') + event.append(event_d) + + # self.event_manager.send_event(event_d) + self.url_status[url] = 2 + # print('up') + return tuple(event) + # Upload(title, url).start() + + # except: + # logger.exception() + # print('寻找结束') + + @event_manager.register(engine.BE_MODIFIED) + def revise(self, url): + if url: + # url_status = {**url_status, **{url: 0}} + self.url_status.update({url: 0}) + # print('更新字典') + # print(url_status) diff --git a/Engine/plugins/__init__.py b/engine/plugins/__init__.py similarity index 100% rename from Engine/plugins/__init__.py rename to engine/plugins/__init__.py diff --git a/Engine/plugins/douyu.py b/engine/plugins/douyu.py similarity index 96% rename from Engine/plugins/douyu.py rename to engine/plugins/douyu.py index c6df11d..d6fc5bf 100644 --- a/Engine/plugins/douyu.py +++ b/engine/plugins/douyu.py @@ -3,8 +3,8 @@ import json import re import time import requests -from Engine.plugins import FFmpegdl, match1 -from Engine.plugins.twitch import headers +from engine.plugins import FFmpegdl, match1 +from engine.plugins.twitch import headers from common import logger VALID_URL_BASE = r'(?:https?://)?(?:(?:www|m)\.)?douyu\.com' diff --git a/Engine/plugins/general.py b/engine/plugins/general.py similarity index 61% rename from Engine/plugins/general.py rename to engine/plugins/general.py index 0a5e700..d7553f4 100644 --- a/Engine/plugins/general.py +++ b/engine/plugins/general.py @@ -1,4 +1,4 @@ -from Engine.plugins import SDownload +from engine.plugins import SDownload class Generic(SDownload): diff --git a/Engine/plugins/huya.py b/engine/plugins/huya.py similarity index 85% rename from Engine/plugins/huya.py rename to engine/plugins/huya.py index 77f985d..0ad9e12 100644 --- a/Engine/plugins/huya.py +++ b/engine/plugins/huya.py @@ -1,9 +1,9 @@ import json import requests -import Engine.plugins -import Engine.plugins.douyu -from Engine.plugins import FFmpegdl +import engine.plugins +import engine.plugins.douyu +from engine.plugins import FFmpegdl from common import logger VALID_URL_BASE = r'(?:https?://)?(?:(?:www|m)\.)?huya\.com' @@ -21,7 +21,7 @@ class Huya(FFmpegdl): res = requests.get(self.url, timeout=5, headers=user_agent) res.close() data = res.text - huya = Engine.plugins.match1(data, r'({"sCdnType":"TX".*?})') + huya = engine.plugins.match1(data, r'({"sCdnType":"TX".*?})') if huya: huyajson = json.loads(huya) self.ydl_opts["absurl"] = huyajson["sFlvUrl"] + '/' + huyajson["sStreamName"] + '.' + \ diff --git a/Engine/plugins/panda.py b/engine/plugins/panda.py similarity index 97% rename from Engine/plugins/panda.py rename to engine/plugins/panda.py index deb2196..b953de1 100644 --- a/Engine/plugins/panda.py +++ b/engine/plugins/panda.py @@ -1,5 +1,5 @@ import requests -from Engine.plugins import BatchCheckBase, SDownload +from engine.plugins import BatchCheckBase, SDownload from common import logger VALID_URL_BASE = r'(?:https?://)?(?:www\.)?panda\.tv/(?P[0-9]+)' diff --git a/Engine/plugins/twitch.py b/engine/plugins/twitch.py similarity index 60% rename from Engine/plugins/twitch.py rename to engine/plugins/twitch.py index 5202ab5..c5d144a 100644 --- a/Engine/plugins/twitch.py +++ b/engine/plugins/twitch.py @@ -1,7 +1,7 @@ # import json # import re import requests -from Engine.plugins import YDownload, BatchCheckBase +from engine.plugins import YDownload, BatchCheckBase from common import logger headers = { @@ -16,38 +16,10 @@ class Twitch(YDownload): def __init__(self, fname, url, suffix='mp4'): YDownload.__init__(self, fname, url, suffix=suffix) - # def check_stream(self): - # - # check_url = re.sub(r'.*twitch.tv', 'https://api.twitch.tv/kraken/streams', self.url) - # try: - # res = requests.get(check_url, headers=headers) - # res.close() - # except requests.exceptions.SSLError: - # logger.error('获取流信息发生错误') - # logger.error(requests.exceptions.SSLError, exc_info=True) - # return None - # except requests.exceptions.ConnectionError: - # logger.exception('During handling of the above exception, another exception occurred:') - # return None - # - # try: - # s = json.loads(res.text) - # # s = res.json() https://api.twitch.tv/kraken/streams/ - # except json.decoder.JSONDecodeError: - # logger.exception('Expecting value') - # return None - # print(self.fname) - # try: - # stream = s['stream'] - # except KeyError: - # logger.error(KeyError, exc_info=True) - # return None - # return stream - def dl(self): info_list = self.get_sinfo() - if self.fname in ['星际2ByuN武圣人族天梯第一视角', '星际2INnoVation吕布卫星人族天梯第一视角', '星际2Maru人族天梯第一视角']: + if self.fname not in ['星际2PartinG跳跳胖丁神族天梯第一视角']: pass elif '720p' in info_list: self.ydl_opts['format'] = '720p' @@ -57,6 +29,34 @@ class Twitch(YDownload): super().dl() +# def check_stream(self): +# +# check_url = re.sub(r'.*twitch.tv', 'https://api.twitch.tv/kraken/streams', self.url) +# try: +# res = requests.get(check_url, headers=headers) +# res.close() +# except requests.exceptions.SSLError: +# logger.error('获取流信息发生错误') +# logger.error(requests.exceptions.SSLError, exc_info=True) +# return None +# except requests.exceptions.ConnectionError: +# logger.exception('During handling of the above exception, another exception occurred:') +# return None +# +# try: +# s = json.loads(res.text) +# # s = res.json() https://api.twitch.tv/kraken/streams/ +# except json.decoder.JSONDecodeError: +# logger.exception('Expecting value') +# return None +# print(self.fname) +# try: +# stream = s['stream'] +# except KeyError: +# logger.error(KeyError, exc_info=True) +# return None +# return stream + class BatchCheck(BatchCheckBase): def __init__(self, urls): BatchCheckBase.__init__(self, pattern_id=VALID_URL_BASE, urls=urls) diff --git a/Engine/slider/__init__.py b/engine/slider/__init__.py similarity index 100% rename from Engine/slider/__init__.py rename to engine/slider/__init__.py diff --git a/Engine/upload.py b/engine/upload.py similarity index 97% rename from Engine/upload.py rename to engine/upload.py index 7addeb4..18c5cda 100644 --- a/Engine/upload.py +++ b/engine/upload.py @@ -3,9 +3,9 @@ import os from selenium import webdriver import selenium.common import time -import Engine -import Engine.kernel -from Engine.slider import slider_cracker +import engine +import engine.handler +from engine.slider import slider_cracker from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as ec @@ -24,7 +24,7 @@ class Upload(object): # @property # def file_name(self): - # now = Engine.work.time_now() + # now = engine.work.time_now() # file_name = '%s%s' % (now, self.title) # return file_name @@ -86,7 +86,7 @@ class Upload(object): def upload(self, title_, file_list, link): - filename = 'Engine/bilibili.cookie' + filename = 'engine/bilibili.cookie' # title_ = self.r_title videopath = self.assemble_videopath(file_list) @@ -94,7 +94,7 @@ class Upload(object): options = webdriver.ChromeOptions() options.add_argument('headless') - driver = webdriver.Chrome(executable_path=Engine.chromedrive_path, chrome_options=options) + driver = webdriver.Chrome(executable_path=engine.chromedrive_path, chrome_options=options) # service_log_path=service_log_path) try: # service_log_path = "{}/chromedriver.log".format('/home') @@ -243,10 +243,10 @@ class Upload(object): ec.presence_of_element_located((By.XPATH, r'//*[@id="login-username"]'))) username = driver.find_element_by_xpath(r'//*[@id="login-username"]') - username.send_keys(Engine.user_name) + username.send_keys(engine.user_name) password = driver.find_element_by_xpath('//*[@id="login-passwd"]') - password.send_keys(Engine.pass_word) + password.send_keys(engine.pass_word) # logger.info('第四步') # try: cracker = slider_cracker(driver) diff --git a/requirements.txt b/requirements.txt index 031ea21..d723cff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ selenium==3.8.1 psutil==5.4.6 -youtube_dl==2018.10.5 +youtube_dl==2018.11.7 Pillow==5.2.0 Requests==2.20.0 PyYAML==3.12