Add log config

README.md

@@ -11,7 +11,7 @@
 相关配置示例在config.yaml文件中，如直播间地址，b站账号密码
 
 演示视频：[BV1ip4y1x7Gi](https://www.bilibili.com/video/BV1ip4y1x7Gi)
-## 1. 安装使用
+## INSTALLATION
 1. 创建配置文件**config.yaml**，内容参照**config(demo).yaml**（或可调整配置后直接修改文件名）
 2. 安装 __FFmpeg__, __pip__
 3. 安装 __biliup__：
@@ -25,13 +25,15 @@ $ biliup stop
 $ biliup restart
 # 查看版本
 $ biliup --version
+# 显示帮助
+$ biliup -h
 ```
 
 Linux下以daemon进程启动，录像和日志文件保存在执行目录下，程序执行过程可查看日志文件。
 `ps -A | grep biliup` 查看进程是否启动成功。
 
 
-## 2. Docker使用 🔨 
+## Docker使用 🔨 
 ### 方式一
 ```bash
 vim /host/path/config.yaml
@@ -49,7 +51,7 @@ sudo docker ps (找到你的imageId)
 sudo docker exec -it imageId /bin/bash     
 ```
 
-## 3. 源码使用
+## 源码使用
 * 下载源码: git clone https://github.com/ForgQi/bilibiliupload.git
 * 安装: `pip3 install -e .` 或者 `pip3 install -r requirements.txt`
 * 启动: `python3 -m biliup`

biliup/__init__.py

@@ -7,7 +7,7 @@ from .engine.event import EventManager, Event
 from .engine import config, invert_dict, Plugin
 from . import plugins
 
-__version__ = "0.0.8"
+__version__ = "0.0.9"
 
 
 def create_event_manager():

biliup/__main__.py

@@ -1,34 +1,83 @@
 #!/usr/bin/python3
 # coding:utf8
+import argparse
+import logging.config
 import platform
-import sys
 import asyncio
+import sys
 
+from biliup import config
 from .common.Daemon import Daemon
 from . import main, __version__
 
+LOG_CONF = {
+    'version': 1,
+    'formatters': {
+        'verbose': {
+            'format': "%(asctime)s %(filename)s[line:%(lineno)d](Pid:%(process)d "
+                      "Tname:%(threadName)s) %(levelname)s %(message)s",
+            # 'datefmt': "%Y-%m-%d %H:%M:%S"
+        },
+        'simple': {
+            'format': '%(filename)s%(lineno)d[%(levelname)s]Tname:%(threadName)s %(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': logging.DEBUG,
+            'class': 'logging.StreamHandler',
+            'stream': sys.stdout,
+            'formatter': 'simple'
+        },
+        'file': {
+            'level': logging.DEBUG,
+            'class': 'biliup.common.log.SafeRotatingFileHandler',
+            'when': 'W0',
+            'interval': 1,
+            'backupCount': 1,
+            'filename': 'ds_update.log',
+            'formatter': 'verbose'
+        }
+    },
+    'root': {
+        'handlers': ['console'],
+        'level': logging.INFO,
+    },
+    'loggers': {
+        'biliup': {
+            'handlers': ['file'],
+            'level': logging.INFO,
+        },
+    }
+}
 
-def _main():
+
+def arg_parser():
     daemon = Daemon('watch_process.pid', main)
-    if platform.system() != 'Windows' and len(sys.argv) == 2:
-        if 'start' == sys.argv[1]:
-            daemon.start()
-        elif 'stop' == sys.argv[1]:
-            daemon.stop()
-        elif 'restart' == sys.argv[1]:
-            daemon.restart()
-        elif '--version' == sys.argv[1]:
-            print(__version__)
-        else:
-            print('unknown command')
-            sys.exit(2)
-        sys.exit(0)
-    elif platform.system() == 'Windows' or len(sys.argv) == 1:
+    parser = argparse.ArgumentParser(description='Stream download and upload, not only for bilibili.')
+    parser.add_argument('--version', action='version', version=f"v{__version__}")
+    parser.add_argument('-v', '--verbose', action="store_const", const=logging.DEBUG, default=logging.INFO,
+                        help="Increase output verbosity")
+    subparsers = parser.add_subparsers(help='Windows does not support this sub-command.')
+    # create the parser for the "start" command
+    parser_start = subparsers.add_parser('start', help='Run as a daemon process.')
+    parser_start.set_defaults(func=daemon.start)
+    parser_stop = subparsers.add_parser('stop', help='Stop daemon according to "watch_process.pid".')
+    parser_stop.set_defaults(func=daemon.stop)
+    parser_restart = subparsers.add_parser('restart')
+    parser_restart.set_defaults(func=daemon.restart)
+    parser.set_defaults(func=None)
+    args = parser.parse_args()
+
+    LOG_CONF['loggers']['biliup']['level'] = args.verbose
+    LOG_CONF['root']['level'] = args.verbose
+    logging.config.dictConfig({**LOG_CONF, **config['LOGGING']})
+
+    if platform.system() == 'Windows' or args.func is None:
         asyncio.run(main())
     else:
-        print('usage: %s start|stop|restart' % sys.argv[0])
-        sys.exit(2)
+        args.func()
 
 
 if __name__ == '__main__':
-    _main()
+    arg_parser()

biliup/common/Daemon.py

@@ -6,7 +6,7 @@ import os
 import time
 import atexit
 
-logger = logging.getLogger('log01')
+logger = logging.getLogger('biliup')
 
 
 # python模拟linux的守护进程

biliup/common/__init__.py

@@ -1,7 +1,5 @@
-import os
+import logging
 from datetime import datetime, timezone, timedelta
-from .log import SafeRotatingFileHandler
-import logging.config
 import sys
 
 
@@ -11,14 +9,13 @@ def time_now():
     # now = bj_dt.strftime('%Y{0}%m{1}%d{2}').format(*'...')
     now = bj_dt.strftime('%Y.%m.%d')
     return now
+# logging.SafeRotatingFileHandler = SafeRotatingFileHandler
+# log_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configlog.ini')
+# logging.config.fileConfig(log_file_path)
 
 
 def new_hook(t, v, tb):
-    logger.error("Uncaught exception:", exc_info=(t, v, tb))
+    logging.getLogger('biliup').error("Uncaught exception:", exc_info=(t, v, tb))
 
 
-logging.SafeRotatingFileHandler = SafeRotatingFileHandler
 sys.excepthook = new_hook
-log_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configlog.ini')
-logging.config.fileConfig(log_file_path)
-logger = logging.getLogger('log01')

biliup/common/reload.py

@@ -6,7 +6,7 @@ import logging
 
 from .timer import Timer
 
-logger = logging.getLogger('log01')
+logger = logging.getLogger('biliup')
 
 
 def has_extension(fname_list, *extension):

biliup/downloader.py

@@ -6,7 +6,7 @@ from urllib.error import HTTPError
 from .engine.decorators import Plugin
 from .plugins import general, BatchCheckBase
 
-logger = logging.getLogger('log01')
+logger = logging.getLogger('biliup')
 
 
 def download(fname, url, **kwargs):

biliup/engine/download.py

@@ -7,7 +7,7 @@ import time
 from biliup import config
 from ..plugins import fake_headers
 
-logger = logging.getLogger('log01')
+logger = logging.getLogger('biliup')
 
 
 class DownloadBase:

biliup/engine/upload.py

@@ -1,7 +1,7 @@
 import logging
 import os
 
-logger = logging.getLogger('log01')
+logger = logging.getLogger('biliup')
 
 
 class UploadBase:

biliup/handler.py

@@ -1,6 +1,7 @@
+import logging
+
 from . import common
 from biliup import event_manager, config
-from .common import logger
 from .engine.event import Event
 from .downloader import download, check_url
 from .engine.upload import UploadBase
@@ -12,6 +13,7 @@ TO_MODIFY = 'to_modify'
 DOWNLOAD = 'download'
 BE_MODIFIED = 'be_modified'
 UPLOAD = 'upload'
+logger = logging.getLogger('biliup')
 
 
 @event_manager.register(DOWNLOAD, block=True)

biliup/plugins/__init__.py

@@ -1,7 +1,7 @@
 import logging
 import re
 
-logger = logging.getLogger('log01')
+logger = logging.getLogger('biliup')
 
 
 class BatchCheckBase:

biliup/plugins/acfun.py

@@ -2,18 +2,13 @@ import random
 import string
 import json
 import requests
-from . import match1, logger
+from . import logger
 from ..engine.decorators import Plugin
 from ..engine.download import DownloadBase
 
 
-def get_random_name(l):
-    return random.choice(string.ascii_lowercase) + \
-        ''.join(random.sample(string.ascii_letters + string.digits, l - 1))
-
-
 @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m|live)\.)?acfun\.cn')
-class acfun(DownloadBase):
+class Acfun(DownloadBase):
     def __init__(self, fname, url, suffix='flv'):
         super().__init__(fname, url, suffix)
 
@@ -23,10 +18,14 @@ class acfun(DownloadBase):
             return False
         rid = self.url.split("acfun.cn/live/")[1]
         did = "web_"+get_random_name(16)
-        headers1 = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.67"}
+        headers1 = {
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                          "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.67"
+        }
         cookies1 = dict(_did=did)
         data1 = {'sid': 'acfun.api.visitor'}
-        r1 = requests.post("https://id.app.acfun.cn/rest/app/visitor/login", headers=headers1, data=data1, cookies=cookies1)
+        r1 = requests.post("https://id.app.acfun.cn/rest/app/visitor/login",
+                           headers=headers1, data=data1, cookies=cookies1)
         userId = r1.json()['userId']
         visitorSt = r1.json()['acfun.api.visitor_st']
         params = {
@@ -39,14 +38,21 @@ class acfun(DownloadBase):
         }
         data2 = {'authorId': rid, 'pullStreamType': 'FLV'}
         headers2 = {
-            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.67",
+            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                          "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.67",
             "Referer": "https://live.acfun.cn/"
         }
-        r2 = requests.post("https://api.kuaishouzt.com/rest/zt/live/web/startPlay", headers=headers2, data=data2, params=params)
+        r2 = requests.post("https://api.kuaishouzt.com/rest/zt/live/web/startPlay",
+                           headers=headers2, data=data2, params=params)
         if r2.json()['result'] != 1:
             logger.debug(r2.json()['error_msg'])
             return False
         d = r2.json()['data']['videoPlayRes']
         e = json.loads(d)['liveAdaptiveManifest'][0]['adaptationSet']['representation']
         self.raw_stream_url = e[-1]['url']
-        return True
+        return True
+
+
+def get_random_name(l):
+    return random.choice(string.ascii_lowercase) + \
+        ''.join(random.sample(string.ascii_letters + string.digits, l - 1))

biliup/plugins/douyin.py

@@ -4,8 +4,10 @@ import json
 from . import logger
 from ..engine.decorators import Plugin
 from ..engine.download import DownloadBase
+
+
 @Plugin.download(regexp=r'(?:https?://)?(?:(?:www|m|live)\.)?douyin\.com')
-class acfun(DownloadBase):
+class Douyin(DownloadBase):
     def __init__(self, fname, url, suffix='flv'):
         super().__init__(fname, url, suffix)
 
@@ -14,7 +16,8 @@ class acfun(DownloadBase):
             logger.debug("直播间地址错误")
             return False
         rid = self.url.split("live.douyin.com/")[1]
-        r1 = requests.get('https://live.douyin.com/'+rid).text.split('<script id="RENDER_DATA" type="application/json">')[1].split('</script>')[0]
+        r1 = requests.get('https://live.douyin.com/' + rid).text \
+            .split('<script id="RENDER_DATA" type="application/json">')[1].split('</script>')[0]
         r2 = urllib.request.unquote(r1)
         r3 = json.loads(r2)['routeInitialProps']['errorType']
         if r3 != 'none':
@@ -29,6 +32,6 @@ class acfun(DownloadBase):
         for k in r5:
             if i < 1:
                 r6 = k
-                i = i+1
+                i = i + 1
         self.raw_stream_url = r5[r6]
         return True

biliup/uploader.py

@@ -1,9 +1,11 @@
 import inspect
+import logging
 
 from biliup import engine
-from .common import logger
 from .engine.decorators import Plugin
 
+logger = logging.getLogger('biliup')
+
 
 def upload(platform, index, data):
     """

config(demo).yaml

@@ -47,6 +47,34 @@ pool1_size: 3
 pool2_size: 3
 # 检测源码文件变化间隔，单位：秒，检测源码到变化后，程序会在空闲时自动重启
 check_sourcecode: 15
-
+# 日志输出配置
+LOGGING:
+    formatters:
+        verbose:
+            format: '%(asctime)s %(filename)s[line:%(lineno)d](Pid:%(process)d Tname:%(threadName)s) %(levelname)s %(message)s'
+            datefmt: '%Y-%m-%d %H:%M:%S'
+        simple:
+            format: '%(filename)s%(lineno)d[%(levelname)s]Tname:%(threadName)s %(message)s'
+    handlers:
+        console:
+            level: DEBUG
+            class: logging.StreamHandler
+            formatter: simple
+            stream: ext://sys.stdout
+        file:
+            level: DEBUG
+            class: biliup.common.log.SafeRotatingFileHandler
+            when: W0
+            interval: 1
+            backupCount: 1
+            filename: ds_update.log
+            formatter: verbose
+    root:
+        handlers: [ console ]
+        level: INFO
+    loggers:
+        biliup:
+            handlers: [ file ]
+            level: INFO
 # 默认通过网页接口上传,可选通过操作chrome上传,此时需要填写chromedriver路径
 #chromedriver_path: /usr/local/bin/chromedriver

setup.cfg

@@ -32,4 +32,4 @@ include = biliup*
 
 [options.entry_points]
 console_scripts =
-    biliup = biliup.__main__:_main
+    biliup = biliup.__main__:arg_parser