Add Cos Uploader (#98)

Co-authored-by: luckycat <yanghong502gmail.com>

.gitignore

@@ -8,6 +8,8 @@
 *.mp4
 *.part
 old/
+venv./
+debug.py
 config.yaml
 chromedriver.*
 /configlog.ini

biliup/plugins/bili_webup.py

@@ -237,9 +237,14 @@ class BiliBili:
             if lines == 'qn':
                 self._auto_os = {"os": "upos", "query": "upcdn=qn&probe_version=20200810",
                                  "probe_url": "//upos-sz-upcdnqn.bilivideo.com/OK"}
+            if lines == 'cos':
+                self._auto_os = {"os": "cos", "query": "upcdn=qn&probe_version=20200810",
+                                 "probe_url": "//upos-sz-upcdnqn.bilivideo.com/OK"}
             logger.info(f"线路选择 => {self._auto_os['os']}: {self._auto_os['query']}. time: {self._auto_os.get('cost')}")
         if self._auto_os['os'] == 'upos':
             upload = self.upos
+        elif self._auto_os['os'] == 'cos':
+            upload = self.cos
         elif self._auto_os['os'] == 'kodo':
             upload = self.kodo
         elif self._auto_os['os'] == "gcs":
@@ -264,7 +269,77 @@ class BiliBili:
                 f"https://member.bilibili.com/preupload?{self._auto_os['query']}", params=query,
                 timeout=5)
             return asyncio.run(upload(f, total_size, ret.json(), tasks=tasks))
-
+    async def cos(self,file,total_size,ret,chunk_size=4194304,tasks=3):
+        filename = file.name
+        accelerate = config["accelerate"]
+        url = ret["url"]
+        if accelerate!="AUTO":
+            accelerate="ap-"+accelerate
+            url=url.replace("accelerate",accelerate)
+        biz_id = ret["biz_id"]
+        post_headers={
+            "Authorization":ret["post_auth"],
+        }
+        put_headers={
+            "Authorization":ret["put_auth"],
+        }
+        import xmltodict
+        upload_id =  xmltodict.parse(self.__session.post(f'{url}?uploads&output=json', timeout=5,
+                                        headers=post_headers).content)["InitiateMultipartUploadResult"]["UploadId"]
+        # 开始上传
+        parts = []  # 分块信息
+        chunks = math.ceil(total_size / chunk_size)  # 获取分块数量
+        async def upload_chunk(session, chunks_data, params):
+            async with session.put(url, params=params, raise_for_status=True,
+                                   data=chunks_data, headers=put_headers) as r:
+                end = time.perf_counter() - start
+                parts.append({"Part":{"PartNumber": params['chunk'] + 1, "ETag": r.headers['Etag']}})                
+                sys.stdout.write(f"\r{params['end'] / 1000 / 1000 / end:.2f}MB/s "
+                                 f"=> {params['partNumber'] / chunks:.1%}")
+        start = time.perf_counter()
+        await self._upload({
+            'uploadId': upload_id,
+            'chunks': chunks,
+            'total': total_size
+        }, file, chunk_size, upload_chunk, tasks=tasks)
+        cost = time.perf_counter() - start
+        fetch_headers={
+            "X-Upos-Fetch-Source":ret["fetch_headers"]["X-Upos-Fetch-Source"],
+            "X-Upos-Auth":ret["fetch_headers"]["X-Upos-Auth"],
+            "Fetch-Header-Authorization":ret["fetch_headers"]["Fetch-Header-Authorization"]
+        }
+        ii = 0
+        import dict2xml
+        parts=sorted(parts,key=lambda x:x['Part']['PartNumber'])
+        xml=dict2xml.Converter(newlines=False).build(parts)
+        xml="<CompleteMultipartUpload>"+xml+"</CompleteMultipartUpload>"
+        while ii <= 3:
+            try:
+                r = self.__session.post(url, params={'uploadId': upload_id},data=xml, headers=post_headers, timeout=15)
+                if r.status_code == 200:  
+                    break
+            except(urllib3.exceptions.ReadTimeoutError, urllib3.exceptions.MaxRetryError,
+                   requests.exceptions.ConnectionError):
+                ii += 1
+                logger.info("请求合并分片出现问题，尝试重连，次数：" + str(ii))
+                time.sleep(15)
+        if r.status_code !=200:
+            raise Exception(r.text)
+        ii=0
+        while ii <= 3:
+            try:
+                r= self.__session.post("https:"+ret["fetch_url"], headers=fetch_headers, timeout=15).json()
+                if r.get('OK') == 1:         
+                    logger.info(f'{filename} uploaded >> {total_size / 1000 / 1000 / cost:.2f}MB/s. {r}')
+                    return {"title": splitext(filename)[0], "filename": ret["bili_filename"], "desc": ""}
+            except(urllib3.exceptions.ReadTimeoutError, urllib3.exceptions.MaxRetryError,
+                   requests.exceptions.ConnectionError):
+                ii += 1
+                logger.info("上传出现问题，尝试重连，次数：" + str(ii))
+                time.sleep(15)
+        if r.get('OK') != 1:
+            raise Exception(r)
+        
     async def kodo(self, file, total_size, ret, chunk_size=4194304, tasks=3):
         filename = file.name
         bili_filename = ret['bili_filename']

config(demo).yaml

@@ -13,8 +13,10 @@ user: # 在填了cookies的情况下优先使用cookies上传，如需使用用
 
 # 选择全局默认上传插件，Noop为不上传，但会执行后处理
 #uploader: Noop
-# b站上传线路选择，默认为自动模式，目前可手动切换为bda2, kodo, ws, qn
-lines: AUTO
+#腾讯云对象存储内网加速地域选择，在上传线路为cos该配置生效，目前有AUTO，shanghai，guangzhou,AUTO为使用全球加速线路，其他为对应地域的内网线路，
+accelerate: shanghai
+# b站上传线路选择，默认为自动模式，目前可手动切换为bda2, kodo, ws, qn,cos
+lines: cos
 # b站提交接口，默认自动选择，可选web，client
 submit_api: client
 # 单文件并发上传数，未达到带宽上限时增大此值可提高上传速度