From c3418969d5422cbecd0e63fbfcff9727c517d6b7 Mon Sep 17 00:00:00 2001 From: ForgQi <34411314+ForgQi@users.noreply.github.com> Date: Thu, 1 Mar 2018 15:23:22 +0800 Subject: [PATCH] =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E7=89=881.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + .idea/workspace.xml | 144 ++++++++++----------- README.md | 14 ++ configlog.ini | 4 +- download.py | 306 +++++++++----------------------------------- start.py | 15 ++- upload.py | 95 ++------------ work.py | 11 +- 8 files changed, 178 insertions(+), 413 deletions(-) diff --git a/.gitignore b/.gitignore index 3c5b53a..0fd22c0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ external.py AUTO.log ds_update ds_update.log +__pycache__/work.cpython-36.pyc +.idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 7e023cb..59b7f9e 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,7 +1,14 @@ - + + + + + + + + - + - - + + @@ -26,7 +33,7 @@ - + @@ -38,8 +45,8 @@ - - + + @@ -50,18 +57,18 @@ - - + + - + - - + + @@ -70,8 +77,8 @@ - - + + @@ -118,18 +125,18 @@ - - - + - - + - + - + @@ -592,13 +598,6 @@ - - - - - - - @@ -621,7 +620,6 @@ - @@ -629,7 +627,6 @@ - @@ -637,21 +634,13 @@ - - - - - - - - - + - + @@ -659,36 +648,17 @@ - + - - - - - - - - - - - - - - - - - - - - - + + - - + + @@ -697,13 +667,39 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 018b8c3..e3003c9 100644 --- a/README.md +++ b/README.md @@ -1 +1,15 @@ # bilibiliupload + +Linux模拟CTRL-c + + kill -SIGINT PID + +后台运行 + nohup python3 start.py >/dev/null 2>&1 & + +查看系统文件句柄使用 + lsof -n |awk '{print $2}'|sort|uniq -c |sort -nr|more + +requests.exceptions.ConnectionError: HTTPSConnectionPool(host='api.twitch.tv', port=443): Max retries exceeded with url: /kraken/streams/innovation_s2 (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 110] Connection timed out',)) + +youtube_dl.utils.DownloadError: ERROR: Live stream is offline \ No newline at end of file diff --git a/configlog.ini b/configlog.ini index 3de0767..90dc040 100644 --- a/configlog.ini +++ b/configlog.ini @@ -24,14 +24,14 @@ args=(sys.stdout,) class=work.SafeRotatingFileHandler level=INFO formatter=form01 -args=("ds_update.log", "D", 1, 3) +args=("ds_update.log", "W0", 1, 1) [formatters] keys=form01,form02 [formatter_form01] -format=%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s +format=%(asctime)s %(filename)s[line:%(lineno)d]（Pid：%(process)s） %(levelname)s %(message)s [formatter_form02] format=%(lineno)d[%(levelname)s]%(message)s diff --git a/download.py b/download.py index c05fead..3e1973b 100644 --- a/download.py +++ b/download.py @@ -1,5 +1,4 @@ from upload import * -from multiprocessing import Manager import os import requests import json @@ -10,13 +9,21 @@ import youtube_dl # links = ['www.twitch.tv/innovation_s2','www.panda.tv/1160340','www.twitch.tv/sc2soo','www.panda.tv/1150595','www.twitch.tv/kimdaeyeob3'] -root_url = ['www.twitch.tv/', 'www.panda.tv/', 'https://api.twitch.tv/kraken/streams/'] +root_url = ['https://www.twitch.tv/', 'https://www.panda.tv/', 'https://api.twitch.tv/kraken/streams/'] links_id = { - '星际2innovation吕布卫星人族天梯第一视角': ['innovation_s2', '1160340'], - '星际2soo输本虫族天梯第一视角': ['sc2soo', '1150595'], - '星际2stats拔本神族天梯第一视角': ['kimdaeyeob3'], - '星际2sos狗哥神族天梯第一视角': ['', '1160930'] + '星际2Innovation吕布卫星人族天梯第一视角': ['innovation_s2', '1160340'], + '星际2soO输本虫族天梯第一视角': ['sc2soo', '1150595'], + '星际2sOs狗哥神族天梯第一视角': ['', '1160930'], + '星际2Stats拔本神族天梯第一视角': ['kimdaeyeob3'], + '星际2Dark暗本虫族天梯第一视角': ['qkrfuddn0'], + '星际2Scarlett噶姐虫族天梯第一视角': ['scarlettm'], + '星际2GuMiho砸本人族天梯第一视角': ['gumiho'], + '星际2Maru人族天梯第一视角': ['maru072'], + '星际2ByuN武圣人族天梯第一视角': ['byunprime'], + '星际2小herO神族天梯第一视角': ['dmadkr0818'], + '星际2Zest神族天梯第一视角': ['sc2_zest'], + '星际2PartinG跳跳胖丁神族天梯第一视角':['partingthebigboy'] # 'test':['maru072','37229'], # 'test1':['byunprime','10003'] } @@ -25,163 +32,47 @@ headers = { 'client-id': 'jzkbprff40iqj646a697cyrvl0zt2m6' } -# log_fmt = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s' -# formatter = logging.Formatter(log_fmt) -# log_file_handler = TimedRotatingFileHandler(filename="ds_update.log", when="D", interval=1, backupCount=2) -# # log_file_handler.suffix = "%Y-%m-%d.log" -# # log_file_handler.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$") -# log_file_handler.setFormatter(formatter) -# logging.basicConfig(level=logging.INFO) -# logger = logging.getLogger(__name__) -# logger.addHandler(log_file_handler) - -# def logg(): -# # logger = logging.getLogger(__name__) -# logger.info('111') -# class SizeError(Exception): -# def __init__(self, info): -# Exception.__init__(self) -# self.errorinfo = info -# -# def __str__(self): -# return "SizeError:%s" % (self.errorinfo) - -# -# def kill_child_processes(parent_pid, file_name_, sig=signal.SIGINT): -# file_name_ = file_name_+'.part' -# last_file_size = 0.0 -# while True: -# time.sleep(10) -# if os.path.isfile(file_name_): -# file_size = os.path.getsize(file_name_)/1024/1024/1024 -# file_sizes = os.path.getsize(file_name_) -# if float(file_sizes) == last_file_size: -# try: -# parent = psutil.Process(parent_pid) -# except psutil.NoSuchProcess: -# return -# children = parent.children(recursive=True) -# for process in children: -# # print(process) -# process.send_signal(sig) -# logger.info('下载卡死'+file_name_) -# break -# -# -# last_file_size = file_sizes -# -# if float(file_size) >= 3.9: -# try: -# parent = psutil.Process(parent_pid) -# except psutil.NoSuchProcess: -# return -# children = parent.children(recursive=True) -# for process in children: -# # print(process) -# process.send_signal(sig) -# print('分段下载') -# logger.info('分段下载'+file_name_) -# break -# else: -# os._exit(0) - - -# print(file_name_, '文件大小', file_size) # 单位GB -# try: -# parent = psutil.Process(parent_pid) -# except psutil.NoSuchProcess: -# return -# children = parent.children(recursive=True) -# for process in children: -# print(process) -# # process.send_signal(sig) - - -# def monitoring(q): -# while True: -# # print('开始监测') -# pid, file_name = q.get() -# time.sleep(5) -# # print('获取到{0}，{1}'.format(pid,file_name)) -# p = Process(target=kill_child_processes, args=(pid, file_name)) -# p.start() - - -# def uploads(file_name_, url_): -# logger.info('准备上传'+file_name_[:-4]) -# if os.path.isfile(file_name_): -# os.rename(file_name_, file_name_[:-4] + str(time.time())[:10] + file_name_[-4:]) -# file_list = get_file(file_name_) -# logger.debug('获取%s文件列表' % file_name_[:-4]) -# videopath = '' -# root = os.getcwd() -# for i in range(len(file_list)): -# file = file_list[i] -# videopath += root + '/' + file + '\n' -# videopath = videopath.rstrip() -# upload(video_path=videopath, link=url_, title_=file_name_[:-4]) -# -# for r in file_list: -# os.remove(r) -# logger.info('删除-' + r) -# def download_part(): -# try: -# with youtube_dl.YoutubeDL(ydl_opts) as ydl: -# ydl.download([url_]) -# except SizeError: -# -# async def download(file_name_,url_, ydl_): -# if os.path.isfile(file_name_): -# file_size = os.path.getsize(file_name_) -# print('文件大小',file_size) -# print(os.path.isfile(file_name_)) -# await asyncio.sleep(3)#ydl_.download([url_]) -# -# -# async def file_size(file_name_): -# file_size_ = os.path.getsize(file_name_) -# print('文件大小', file_size_) - - -# def is_err(file_name_): -# if os.path.getsize(file_name_) / 1024 / 1024 >= 5: -# raise SizeError('文件大小超出限制') - - -def get_twitch_stream(url, value): +def get_twitch_stream(url, key): try: res = requests.get(url, headers=headers) res.close() + except requests.exceptions.ConnectionError: + logger.exception('During handling of the above exception, another exception occurred:') + return None except requests.exceptions.SSLError: logger.error('获取流信息发生错误') logger.error(requests.exceptions.SSLError, exc_info = True) - return 'None' + return None try: s = json.loads(res.text) # s = res.json() except json.decoder.JSONDecodeError: logger.exception('Expecting value') - return 'None' - print(value) + return None + print(key) try: stream = s['stream'] except KeyError : logger.error(KeyError, exc_info = True) - return 'None' + return None return stream -def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_): +def download_twitch_stream(dict, q, confirm_url, key_, url_, tfile_name_, value_=None): + status_ = get_twitch_stream(confirm_url,key_) # print(status_) if str(status_) != 'None': - value_ = dict[key_] - dict.pop(key_) + if value_ is None: + value_ = dict[key_] + dict.pop(key_) + else: + logger.info('准备递归下载'+key_) print('开始下载twitch', key_) logger.info('开始下载twitch：' + key_) ydl_opts = { 'outtmpl': tfile_name_, - 'format': '720p' + 'format': '720p60' # 'keep_fragments':True # 'postprocessors': [{ # 'key': 'FFmpegFixupM3u8', @@ -191,11 +82,7 @@ def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_): } try: with youtube_dl.YoutubeDL(ydl_opts) as ydl: - # loop = asyncio.get_event_loop() - # tasks = [download(file_name_+'.part',url_,ydl),file_size(file_name_+'.part')] - # loop.run_until_complete(download(file_name_+'.part',url_,ydl)) - # loop.run_until_complete(asyncio.wait(tasks)) - # loop.close() + pid = os.getpid() q.put([pid, tfile_name_]) @@ -210,17 +97,16 @@ def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_): # print('分段下载') try: - logger.info('更名{0}'.format(tfile_name_ + '.part')) os.rename(tfile_name_ + '.part', tfile_name_[:-4] + str(time.time())[:10] + tfile_name_[-4:]) + logger.info('更名{0}为{1}+时间'.format(tfile_name_ + '.part', tfile_name_)) except FileExistsError: os.rename(tfile_name_ + '.part', tfile_name_[:-4] + str(time.time())[:10] + tfile_name_[-4:]) + + download_twitch_stream(dict, q, confirm_url, key_, url_, tfile_name_, value_) + finally: dict[key_] = value_ - - # print('下载完成') - # logger.info('下载完成'+key_) - # uploads(tfile_name_, url_) - # print(links_id) + logger.info('退出下载') elif len(dict[key_]) == 1: value_ = dict[key_] @@ -228,21 +114,13 @@ def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_): supplemental_upload(dict, tfile_name_, key_, url_, value_) - # try: - # for f in os.listdir(): - # if tfile_name_[:-15] in f: - # logger.info('补充上传' + key_) - # uploads(tfile_name_, url_) - # os._exit(0) - # finally: - # dict[key_] = value_ - -def download_panda_stream(dict, q, key_, url_, pfile_name_): - value_ = dict[key_] - dict.pop(key_) - # print('开始下载', key_) - # logger.debug('开始下载pandaTV' + key_) +def download_panda_stream(dict, q, key_, url_, pfile_name_, value_=None): + if value_ is None: + value_ = dict[key_] + dict.pop(key_) + else: + logger.info('准备递归下载'+key_) list_info = [] with youtube_dl.YoutubeDL() as ydl: try: @@ -255,15 +133,6 @@ def download_panda_stream(dict, q, key_, url_, pfile_name_): # logger.debug('准备补充上传'+key_) supplemental_upload(dict, pfile_name_, key_, url_, value_) return - # try: - # for f in os.listdir(): - # if pfile_name_[:-15] in f: - # logger.info('补充上传' + key_) - # uploads(pfile_name_, url_) - # break - # finally: - # dict[key_] = value_ - # os._exit(0) if 'HD-flv' in list_info: ydl_opts = { @@ -286,105 +155,52 @@ def download_panda_stream(dict, q, key_, url_, pfile_name_): pid = os.getpid() # print(pid) q.put([pid, pfile_name_]) + + # if os.path.isfile(pfile_name_): + # os.rename(pfile_name_, pfile_name_ + '.part') + # logger.info('存在{0}更名为{1}'.format(pfile_name_, pfile_name_ + '.part')) + logger.info('开始下载pandaTV' + key_) ydl.download([url_]) print('下载完成') logger.info('下载完成' + key_) - if os.path.isfile(pfile_name_): - os.rename(pfile_name_, pfile_name_ + '.part') - logger.info('存在{0}更名为{1}'.format(pfile_name_,pfile_name_ + '.part' )) - with youtube_dl.YoutubeDL() as ydl: try: info = ydl.extract_info(url_, download=False) - for i in info['formats']: - list_info.append(i['format_id']) except youtube_dl.utils.DownloadError: uploads(pfile_name_, url_) + return + logger.info('实际未下载完成' + key_) + + if os.path.isfile(pfile_name_): + os.rename(pfile_name_, pfile_name_[:-4] + str(time.time())[:10] + pfile_name_[-4:]) + logger.info('存在{0}更名为{1}'.format(pfile_name_, pfile_name_[:-4] + '时间' + pfile_name_[-4:])) + + download_panda_stream(dict, q, key_, url_, pfile_name_, value_) except KeyboardInterrupt: # except youtube_dl.utils.DownloadError: - # print('分段下载') try: - logger.info('更名{0}'.format(pfile_name_+ '.part')) + # logger.info('更名{0}'.format(pfile_name_+ '.part')) os.rename(pfile_name_ + '.part', pfile_name_[:-4] + str(time.time())[:10] + pfile_name_[-4:]) + logger.info('下载进程发生KeyboardInterrupt更名{0}为{1}+时间'.format(pfile_name_ + '.part', pfile_name_)) except FileExistsError: os.rename(pfile_name_ + '.part', pfile_name_[:-4] + str(time.time())[:10] + pfile_name_[-4:]) + download_panda_stream(dict, q, key_, url_, pfile_name_, value_) finally: dict[key_] = value_ - # print('下载完成') - # logger.info('下载完成'+key_) - - # os.rename(file_name_, file_name_[:-4] + str(time.time())[:10] + '.mp4') - # file_list = get_file(file_name_) - # videopath = '' - # root = os.getcwd() - # for i in range(len(file_list)): - # file = file_list[i] - # videopath += root + '/' + file + '\n' - # videopath = videopath.rstrip() - # upload(video_path=videopath, link=url_, title_=file_name_[:-4]) - # - # for r in file_list: - # os.remove(r) - # logger.info('删除-'+r) - # print(links_id) - - # uploads(pfile_name_,url_) + logger.info('退出下载') -def download_stream(dict, q, status_, key_, twitch_url_, panda_url_, tfile_name_, pfile_name_): +def download_stream(dict, q, confirm_url, key_, twitch_url_, panda_url_, tfile_name_, pfile_name_): # print('子进程id%s'%(os.getpid())) - download_twitch_stream(dict, q, status_, key_, twitch_url_, tfile_name_) + download_twitch_stream(dict, q, confirm_url, key_, twitch_url_, tfile_name_) download_panda_stream(dict, q, key_, panda_url_, pfile_name_) + if __name__ == '__main__': - manager = Manager() - # logger.info('123') - # d = manager.dict(links_id) - # while True: - # utc_dt = datetime.utcnow().replace(tzinfo=timezone.utc) - # bj_dt = utc_dt.astimezone(timezone(timedelta(hours=8))) - # now = bj_dt.strftime('%Y{0}%m{1}%d{2}').format(*'年月日') - # - # for key in d.copy():(status) - # - # # print(key) - # file_name = '%s%s.mp4' % (key, now) - # if len(links_id[key]) == 2: - # twitch_url = root_url[0]+links_id[key][0] - # panda_url = root_url[1]+links_id[key][1] - # confirm_url = root_url[2]+links_id[key][0] - # status = get_twitch_stream(confirm_url,key) - # # print # print('父进程id%s'%(os.getpid())) - # p = Process(target=download_stream, args=(d,status,key,twitch_url,panda_url,file_name)) - # p.start() - # - # elif len(links_id[key]) ==1: - # twitch_url = root_url[0]+links_id[key][0] - # confirm_url = root_url[2] + links_id[key][0] - # status = get_twitch_stream(confirm_url, key) - # p = Process(target=download_twitch_stream, args=(d,status, key,twitch_url,file_name)) - # p.start() - # time.sleep(5) - # print('test2018年01月22日0473.mp4') - # try: - # os.rename('1','2') - # except FileExistsError: - # print('test2018年01月22日.mp4') - # root = os.getcwd() - # print(root) - # print(os.listdir()) - # get_twitch_stream('https://api.twitch.tv/kraken/streams/',1) - # a = 'test2018年01月17日.mp4' - # print(a[-4:]) - # a = ['111','2'] - # if '11' in a: - # print('yes') - # else: - # print('88') get_twitch_stream('https://api.twitch.tv/kraken/streams/1160340','233') diff --git a/start.py b/start.py index 8925163..4e5043e 100644 --- a/start.py +++ b/start.py @@ -2,13 +2,16 @@ import logging.config logging.config.fileConfig('configlog.ini') if __name__ == '__main__': from datetime import datetime, timedelta, timezone - from multiprocessing import Pool + from multiprocessing import Pool,Manager from download import * from work import * + import sys # logging.basicConfig(filename='AUTO.log', level=logging.INFO) # logging.basicConfig(format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',\ # datefmt='%a, %d %b %Y %H:%M',level=logging.INFO) + sys.excepthook = new_hook + manager = Manager() d = manager.dict(links_id) @@ -34,11 +37,11 @@ if __name__ == '__main__': panda_url = root_url[1] + links_id[key][1] if links_id[key][0] != '': confirm_url = root_url[2] + links_id[key][0] - status = get_twitch_stream(confirm_url, key) + # status = get_twitch_stream(confirm_url, key) # print(status) # print('父进程id%s'%(os.getpid())) res = pool.apply_async(func=download_stream, - args=(d, queue, status, key, twitch_url, panda_url, tfile_name, pfile_name)) + args=(d, queue, confirm_url, key, twitch_url, panda_url, tfile_name, pfile_name)) # p = Process(target=download_stream, args=(d, queue, status,key,twitch_url,panda_url,tfile_name, pfile_name)) # p.start() else: @@ -50,13 +53,13 @@ if __name__ == '__main__': elif len(links_id[key]) == 1: twitch_url = root_url[0] + links_id[key][0] confirm_url = root_url[2] + links_id[key][0] - status = get_twitch_stream(confirm_url, key) + # status = get_twitch_stream(confirm_url, key) res = pool.apply_async(func=download_twitch_stream, - args=(d, queue, status, key, twitch_url, tfile_name)) + args=(d, queue, confirm_url, key, twitch_url, tfile_name)) # p = Process(target=download_twitch_stream, args=(d, queue, status, key,twitch_url,tfile_name)) # p.start() # print(res.get()) - time.sleep(60) + time.sleep(50) # pool.close() # pool.join() diff --git a/upload.py b/upload.py index 819216f..d347e3b 100644 --- a/upload.py +++ b/upload.py @@ -1,5 +1,6 @@ import os from selenium import webdriver +import selenium import time from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait @@ -9,15 +10,7 @@ import logging # 10800 18000 4110 -# log_fmt = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s' -# formatter = logging.Formatter(log_fmt) -# log_file_handler = TimedRotatingFileHandler(filename="ds_update.log", when="D", interval=1, backupCount=2) -# # log_file_handler.suffix = "%Y-%m-%d.log" -# # log_file_handler.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$") -# log_file_handler.setFormatter(formatter) -# logging.basicConfig(level=logging.INFO) -# logger = logging.getLogger(__name__) -# logger.addHandler(log_file_handler) + logger = logging.getLogger('log01') # def excision(Path): @@ -57,48 +50,8 @@ def upload(video_path, link, title_): service_log_path=service_log_path) # driver = webdriver.Chrome(executable_path=r'D:\bilibiliupload\chromedriver.exe',service_log_path=service_log_path) - driver.get("https://www.bilibili.com") - # cookie = { - # 'fts':'1511094141', - # 'buvid3':'0E299359-8055-4E58-9E82-DDF2F325CCAF26543infoc', - # 'UM_distinctid':'15fdc423182b22-07a38f91352cb7-7b113d-144000-15fdc423183a13', - # 'pgv_pvi':'3700880384', - # 'rpdid':'olklsxmolldosooiiqlpw', - # 'sid':'8k9huzd0', - # 'finger':'edc6ecda', - # 'DedeUserID':'274986200', - # 'DedeUserID__ckMd5':'1206e39bc044fe98', - # 'SESSDATA':'3e02f3d0%2C1517749345%2C489ac7f7', - # 'bili_jct':'4a71e9d389551c4bb7a9ebd05ac7dda0', - # 'member_v2':'1', - # 'pgv_si':'s2364394496', - # '_dfcaptcha':'8c4345e68b07f547aa3f246b7b058cd1', - # 'LIVE_BUVID':'a23404d890e536c9a93b72e5e50be145', - # 'LIVE_BUVID__ckMd5':'d5ea746722041186' - # } - # 964b42c0 - # bahkjm41 - # 1515221245 - # EB49C459-95CA-415C-AA48-B3FE5B46BD9D31043infoc - # 3e02f3d0%2C1517813257%2C06f75553 - # 1ef343ac61192cd3aa59ca9aea289ff3 - # c25a35691b8ef1fc2f2566a1c4df50b4 - # AUTO1115152212636167 - - - # 814ce92affe021564ee8f5ceca806c4a - # cookies = [{'name': 'finger', 'value': 'edc6ecda', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': 'sid', 'value': '8k9huzd0', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': 'fts', 'value': '1511094141', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': 'buvid3', 'value': '0E299359-8055-4E58-9E82-DDF2F325CCAF26543infoc', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': 'DedeUserID', 'value': '274986200', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': 'DedeUserID__ckMd5', 'value': '1206e39bc044fe98', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': 'SESSDATA', 'value': '3e02f3d0%2C1517749345%2C489ac7f7', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': True}, - # {'name': 'bili_jct', 'value': '4a71e9d389551c4bb7a9ebd05ac7dda0', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}, - # {'name': '_dfcaptcha', 'value': '8c4345e68b07f547aa3f246b7b058cd1', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': True}, - # {'name': 'LIVE_BUVID', 'value': 'a23404d890e536c9a93b72e5e50be145', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}] - # driver.delete_all_cookies() + # driver.delete_all_cookies() cookies = [ {'domain': '.bilibili.com', 'expiry': 1611825707.027044, 'httpOnly': False, 'name': 'buvid3', 'path': '/', 'secure': False, 'value': 'A558A031-653B-4890-917D-C205F81B84F93220infoc'}, @@ -125,36 +78,18 @@ def upload(video_path, link, title_): # cookie.pop('domain') # print(cookie) driver.add_cookie(cookie) - # driver.add_cookie({'name': 'finger', 'value': 'edc6ecda'}) - # driver.add_cookie({'name': 'fts', 'value': '1511094141'}) - # driver.add_cookie({'name': 'buvid3', 'value': '0E299359-8055-4E58-9E82-DDF2F325CCAF26543infoc'}) - # # driver.add_cookie({'name': 'UM_distinctid', 'value': '15fdc423182b22-07a38f91352cb7-7b113d-144000-15fdc423183a13'}) - # # driver.add_cookie({'name': 'pgv_pvi', 'value': '3700880384'}) - # # driver.add_cookie({'name': 'rpdid', 'value': 'olklsxmolldosooiiqlpw'}) - # driver.add_cookie({'name': 'sid', 'value': '8k9huzd0'}) - # driver.add_cookie({'name': 'DedeUserID', 'value': '274986200'}) - # driver.add_cookie({'name': 'DedeUserID__ckMd5', 'value': '1206e39bc044fe98'}) - # driver.add_cookie({'name': 'SESSDATA', 'value': '3e02f3d0%2C1517749345%2C489ac7f7'}) - # driver.add_cookie({'name': 'bili_jct', 'value': '814ce92affe021564ee8f5ceca806c4a'}) - # # driver.add_cookie({'name': 'member_v2', 'value': '1'}) - # # driver.add_cookie({'name': 'pgv_si', 'value': 's2364394496'}) - # driver.add_cookie({'name': '_dfcaptcha', 'value': '8c4345e68b07f547aa3f246b7b058cd1'}) - # driver.add_cookie({'name': 'LIVE_BUVID', 'value': 'a23404d890e536c9a93b72e5e50be145'}) - # # driver.add_cookie({'name': 'LIVE_BUVID__ckMd5', 'value': 'd5ea746722041186'}) + # cookie = driver.get_cookies() # print(cookie) # driver.get("https://www.bilibili.com") driver.get("https://member.bilibili.com/video/upload.html") - # driver.add_cookie(cookie) - # time.sleep(20) - # cookie = driver.get_cookies() - # print(cookie) + # print(driver.title) element = WebDriverWait(driver, 10).until( EC.presence_of_element_located((By.NAME, 'file'))) upload = driver.find_element_by_name('file') - # time.sleep(5) + print(driver.title) # logger.info(driver.title) logger.info('开始上传' + title_) @@ -186,15 +121,11 @@ def upload(video_path, link, title_): # driver.execute_script('arguments[0].style.display="none";',hid) # hid_1 = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[1]') # driver.execute_script('arguments[0].style.display="none";',hid_1) - # - # time.sleep(1) + # 点击转载 click = driver.find_element_by_xpath( '//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[2]/div') - # click = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[1]/div[1]/div[2]/div[1]') - # click = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[2]/input') - # click = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[1]/input') - # click.send_keys(Keys.ENTER) + click.click() # driver.execute_script('arguments[0].className = "is-checked";',click) # print(click.is_selected()) @@ -203,9 +134,7 @@ def upload(video_path, link, title_): # click_1.click() # driver.find_elements_by_class_name('is-checked')[1].click() - # click_1 = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[3]/div[1]/div[2]/label/input') - # click_1 = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[3]/div[1]/div[2]/label/div') - # click_1.click() + # 输入转载来源 Input = driver.find_element_by_xpath( '//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[2]/div[2]/input') @@ -224,7 +153,7 @@ def upload(video_path, link, title_): title.send_keys(title_) text = driver.find_element_by_xpath( - '//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[2]/div[2]/div[2]/div/div/input') + '//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[2]/div[2]/div[2]/div[last()]/div/input') text.send_keys('星际争霸2') text.send_keys(Keys.ENTER) time.sleep(0.5) @@ -250,7 +179,7 @@ def upload(video_path, link, title_): print('稿件提交完成！') logger.info('%s提交完成！' % title_) - except Exception: + except selenium.common.exceptions.NoSuchElementException: logger.exception('发生错误') finally: driver.quit() @@ -315,3 +244,5 @@ if __name__ == '__main__': # for i in range(1): # print(i) os.rename('3', '4') + + diff --git a/work.py b/work.py index 875c1f4..e6239ff 100644 --- a/work.py +++ b/work.py @@ -1,6 +1,5 @@ import logging import signal -from logging import FileHandler from logging.handlers import TimedRotatingFileHandler from multiprocessing import Process import time @@ -62,10 +61,10 @@ def kill_child_processes(parent_pid, file_name_, sig=signal.SIGINT): # print(process) process.send_signal(sig) logger.info('下载卡死' + file_name_) - time.sleep(2) + time.sleep(0.5) if os.path.isfile(file_name_): - logger.info('卡死下载进程可能未成功退出') - break + logger.info('卡死下载进程可能未成功退出PID:'+parent_pid+','+children) + continue else: logger.info('卡死下载进程成功退出') break @@ -104,6 +103,10 @@ def monitoring(q): p.start() +def new_hook(t, v, tb): + logger.error("Uncaught exception：", exc_info=(t, v, tb)) + + class SafeRotatingFileHandler(TimedRotatingFileHandler): def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False): TimedRotatingFileHandler.__init__(self, filename, when, interval, backupCount, encoding, delay, utc)