fix(douyin-danmaku): include signature parameter (#967)

* fix(douyin-danmaku): include signature parameter

* cleanup

* cleanup

* use random ua

biliup/Danmaku/douyin.py

@@ -1,5 +1,6 @@
 # 抖音的弹幕录制参考了 https://github.com/LyzenX/DouyinLiveRecorder 和 https://github.com/YunzhiYike/live-tool
 # 2023.07.14：KNaiFen：这部分代码参考了https://github.com/SmallPeaches/DanmakuRender
+# 2024.06.22: 添加来自 https://github.com/hua0512/stream-rec 修改后的 webmssdk.js，以计算 signature
 
 import gzip
 
@@ -8,13 +9,15 @@ import json
 from urllib.parse import unquote
 from biliup.config import config
 from .douyin_util.dy_pb2 import ChatMessage, PushFrame, Response
-from biliup.plugins import match1, random_user_agent
+from biliup.plugins import match1
 from google.protobuf import json_format
+import logging
 
+logger = logging.getLogger('biliup')
 
 class Douyin:
     headers = {
-        'user-agent': random_user_agent(),
+        # 'user-agent': random_user_agent(),
         'Referer': 'https://live.douyin.com/',
         'Cookie': config.get('user', {}).get('douyin_cookie', '')
     }
@@ -25,6 +28,8 @@ class Douyin:
     async def get_ws_info(url, context):
         async with aiohttp.ClientSession() as session:
             from biliup.plugins.douyin import DouyinUtils
+            from .douyin_util import DouyinDanmakuUtils
+            Douyin.headers['user-agent'] = DouyinUtils.DOUYIN_USER_AGENT
             if "/user/" in url:
                 async with session.get(url, headers=Douyin.headers, timeout=5) as resp:
                     user_page = await resp.text()
@@ -44,9 +49,59 @@ class Douyin:
                     DouyinUtils.build_request_url(f"https://live.douyin.com/webcast/room/web/enter/?web_rid={room_id}"),
                     headers=Douyin.headers, timeout=5) as resp:
                 room_info = json.loads(await resp.text())['data']['data'][0]
-                url = DouyinUtils.build_request_url(
-                    f"wss://webcast3-ws-web-lf.douyin.com/webcast/im/push/v2/?room_id={room_info['id_str']}&compress=gzip&signature=00000000")
-                return url, []
+
+            USER_UNIQUE_ID = DouyinDanmakuUtils.get_user_unique_id()
+            VERSION_CODE = 180800 # https://lf-cdn-tos.bytescm.com/obj/static/webcast/douyin_live/7697.782665f8.js -> a.ry
+            WEBCAST_SDK_VERSION = "1.0.14-beta.0" # https://lf-cdn-tos.bytescm.com/obj/static/webcast/douyin_live/7697.782665f8.js -> ee.VERSION
+            # logger.info(f"user_unique_id: {USER_UNIQUE_ID}")
+            sig_params = {
+                "live_id": "1",
+                "aid": "6383",
+                "version_code": VERSION_CODE,
+                "webcast_sdk_version": WEBCAST_SDK_VERSION,
+                "room_id": room_info['id_str'],
+                "sub_room_id": "",
+                "sub_channel_id": "",
+                "did_rule": "3",
+                "user_unique_id": USER_UNIQUE_ID,
+                "device_platform": "web",
+                "device_type": "",
+                "ac": "",
+                "identity": "audience"
+            }
+            signature = DouyinDanmakuUtils.get_signature(DouyinDanmakuUtils.get_x_ms_stub(sig_params))
+            # logger.info(f"signature: {signature}")
+            webcast5_params = {
+                "room_id": room_info['id_str'],
+                "compress": 'gzip',
+                # "app_name": "douyin_web",
+                "version_code": VERSION_CODE,
+                "webcast_sdk_version": WEBCAST_SDK_VERSION,
+                # "update_version_code": "1.0.14-beta.0",
+                # "cookie_enabled": "true",
+                # "screen_width": "1920",
+                # "screen_height": "1080",
+                # "browser_online": "true",
+                # "tz_name": "Asia/Shanghai",
+                # "cursor": "t-1718899404570_r-1_d-1_u-1_h-7382616636258522175",
+                # "internal_ext": "internal_src:dim|wss_push_room_id:7382580251462732598|wss_push_did:7344670681018189347|first_req_ms:1718899404493|fetch_time:1718899404570|seq:1|wss_info:0-1718899404570-0-0|wrds_v:7382616716703957597",
+                # "host": "https://live.douyin.com",
+                "live_id": "1",
+                "did_rule": "3",
+                # "endpoint": "live_pc",
+                # "support_wrds": "1",
+                "user_unique_id": USER_UNIQUE_ID,
+                # "im_path": "/webcast/im/fetch/",
+                "identity": "audience",
+                # "need_persist_msg_count": "15",
+                # "insert_task_id": "",
+                # "live_reason": "",
+                # "heartbeatDuration": "0",
+                "signature": signature,
+            }
+            wss_url = f"wss://webcast5-ws-web-lf.douyin.com/webcast/im/push/v2/?{'&'.join([f'{k}={v}' for k, v in webcast5_params.items()])}"
+            url = DouyinUtils.build_request_url(wss_url)
+            return url, []
 
     @staticmethod
     def decode_msg(data):
@@ -73,6 +128,7 @@ class Douyin:
                 data = json_format.MessageToDict(chat_message, preserving_proto_field_name=True)
                 # name = data['user']['nickName']
                 content = data['content']
+                # print(content)
                 # msg_dict = {"time": now, "name": name, "content": content, "msg_type": "danmaku", "color": "ffffff"}
                 msg_dict = {"content": content, "msg_type": "danmaku"}
                 msgs.append(msg_dict)

biliup/Danmaku/douyin_util/__init__.py

@@ -1,17 +1,47 @@
-# 抖音的弹幕录制参考了 https://github.com/LyzenX/DouyinLiveRecorder 和 https://github.com/YunzhiYike/live-tool
+import hashlib
+import logging
+import os
 
-from datetime import datetime
-import threading
-import asyncio
-import gzip
-import re
-import time
-import re
-import requests
-import urllib
-import json
+logger = logging.getLogger('biliup')
 
-import websocket
-from google.protobuf import json_format
+class DouyinDanmakuUtils:
+    @staticmethod
+    def load_webmssdk(js_file: str):
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        js_path = os.path.join(dir_path, js_file)
+        with open(js_path, 'r', encoding='utf-8') as f:
+            return f.read()
 
-from .dy_pb2 import PushFrame, Response, ChatMessage
+    @staticmethod
+    def get_user_unique_id() -> str:
+        import random
+        return str(random.randint(7300000000000000000, 7999999999999999999))
+
+    @staticmethod
+    def get_x_ms_stub(params: dict) -> str:
+        sig_params = ','.join([f'{k}={v}' for k, v in params.items()])
+        return hashlib.md5(sig_params.encode()).hexdigest()
+
+    @staticmethod
+    def get_signature(x_ms_stub: str):
+        from biliup.plugins.douyin import DouyinUtils
+        try:
+            import jsengine
+            ctx = jsengine.jsengine()
+            js_dom = f"""
+document = {{}}
+window = {{}}
+navigator = {{
+  'userAgent': '{DouyinUtils.DOUYIN_USER_AGENT}'
+}}
+""".strip()
+            js_enc = DouyinDanmakuUtils.load_webmssdk('webmssdk.js')
+            final_js = js_dom + js_enc
+            ctx.eval(final_js)
+            function_caller = f"get_sign('{x_ms_stub}')"
+            signature = ctx.eval(function_caller)
+            # print("signature: ", signature)
+            return signature
+        except:
+            logger.exception("get_signature error")
+        return "00000000"

biliup/Danmaku/douyin_util/get_signature_test.py

@@ -0,0 +1,29 @@
+DOUYIN_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
+
+def load_webmssdk(js_file: str):
+    with open(js_file, 'r', encoding='utf-8') as f:
+        return f.read()
+
+def get_signature(X_MS_STUB: str):
+    try:
+        import jsengine
+        ctx = jsengine.jsengine()
+        js_dom = f"""
+document = {{}}
+window = {{}}
+navigator = {{
+'userAgent': '{DOUYIN_USER_AGENT}'
+}}
+""".strip()
+        js_enc = load_webmssdk('biliup\Danmaku\douyin_util\webmssdk.js')
+        final_js = js_dom + js_enc
+        ctx.eval(final_js)
+        function_caller = f"get_sign('{X_MS_STUB}')"
+        signature = ctx.eval(function_caller)
+        print("signature: ", signature)
+        return signature
+    except:
+        raise
+    return "00000000"
+
+get_signature("69a78110dbe05a916c750237d701907e")

biliup/__main__.py

@@ -45,13 +45,14 @@ def arg_parser():
         from biliup.database.db import SessionLocal, init
         # 初始化数据库
         with SessionLocal() as db:
-            try:
-                config.load(args.config)
-                # 如果配置文件不存在，不修改已有数据库
-                if init(args.no_http):
-                    config.save_to_db(db)
-            except FileNotFoundError:
-                print(f'新版本不依赖配置文件，请访问 WebUI 修改配置')
+            if args.no_http or args.config:
+                try:
+                    config.load(args.config)
+                    # 如果配置文件不存在，不修改已有数据库
+                    if init(args.no_http):
+                        config.save_to_db(db)
+                except FileNotFoundError:
+                    print(f'新版本不依赖配置文件，请访问 WebUI 修改配置')
             config.load_from_db(db)
         # db.remove()
         LOG_CONF.update(config.get('LOGGING', {}))

biliup/plugins/douyin.py

@@ -5,7 +5,7 @@ from urllib.parse import unquote, urlparse, parse_qs, urlencode, urlunparse
 import requests
 
 from biliup.common.util import client
-from . import logger, match1
+from . import logger, match1, random_user_agent
 from biliup.config import config
 from biliup.Danmaku import DanmakuClient
 from ..common.tools import NamedLock
@@ -18,6 +18,7 @@ class Douyin(DownloadBase):
     def __init__(self, fname, url, suffix='flv'):
         super().__init__(fname, url, suffix)
         self.douyin_danmaku = config.get('douyin_danmaku', False)
+        self.fake_headers['user-agent'] = DouyinUtils.DOUYIN_USER_AGENT
         self.fake_headers['referer'] = "https://live.douyin.com/"
         self.fake_headers['cookie'] = config.get('user', {}).get('douyin_cookie', '')
 
@@ -125,12 +126,26 @@ class Douyin(DownloadBase):
 
     def danmaku_init(self):
         if self.douyin_danmaku:
-            self.danmaku = DanmakuClient(self.url, self.gen_download_filename())
+            try:
+                import jsengine
+                try:
+                    jsengine.jsengine()
+                    self.danmaku = DanmakuClient(self.url, self.gen_download_filename())
+                except jsengine.exceptions.RuntimeError as e:
+                    extra_msg = "如需录制抖音弹幕，"
+                    logger.error(f"\n{e}\n{extra_msg}请至少安装一个 Javascript 解释器，如 pip install quickjs")
+            except:
+                pass
 
 
 class DouyinUtils:
     # 抖音ttwid
     _douyin_ttwid: Optional[str] = None
+    # DOUYIN_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
+    DOUYIN_USER_AGENT = random_user_agent()
+    DOUYIN_HTTP_HEADERS = {
+        'User-Agent': DOUYIN_USER_AGENT
+    }
 
     @staticmethod
     def get_ttwid() -> Optional[str]:
@@ -148,8 +163,8 @@ class DouyinUtils:
         existing_params['device_platform'] = ['web']
         existing_params['browser_language'] = ['zh-CN']
         existing_params['browser_platform'] = ['Win32']
-        existing_params['browser_name'] = ['Chrome']
-        existing_params['browser_version'] = ['92.0.4515.159']
+        existing_params['browser_name'] = [DouyinUtils.DOUYIN_USER_AGENT.split('/')[0]]
+        existing_params['browser_version'] = [DouyinUtils.DOUYIN_USER_AGENT.split(existing_params['browser_name'][0])[-1][1:]]
         new_query_string = urlencode(existing_params, doseq=True)
         new_url = urlunparse((
             parsed_url.scheme,

biliup/plugins/douyu.py

@@ -61,9 +61,10 @@ class Douyu(DownloadBase):
             try:
                 import jsengine
                 try:
-                    ctx = jsengine.jsengine()
+                    jsengine.jsengine()
                 except jsengine.exceptions.RuntimeError as e:
-                    logger.error(f"{e}\n请至少安装一个 Javascript 解释器，如 pip install quickjs")
+                    extra_msg = "如需录制斗鱼直播，"
+                    logger.error(f"\n{e}\n{extra_msg}请至少安装一个 Javascript 解释器，如 pip install quickjs")
                     return False
             except:
                 logger.exception(f"{self.plugin_msg}: ")