okooooooo/biliup
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

正式版1.0

Browse Source
This commit is contained in:
ForgQi committed 2018-03-01 15:23:22 +08:00
1 parent 06b57e40e5
commit c3418969d5
8 files changed
+178 -413
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+2
View File
@@ -10,3 +10,5 @@ external.py
AUTO.log
ds_update
ds_update.log
__pycache__/work.cpython-36.pyc
.idea/workspace.xml
.idea/workspace.xml
Generated
+70 -74
View File
@@ -1,7 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="58acc6c6-8ea5-48bb-95b2-31fcec9a4358" name="Default" comment="" />
<list default="true" id="58acc6c6-8ea5-48bb-95b2-31fcec9a4358" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/configlog.ini" afterPath="$PROJECT_DIR$/configlog.ini" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/download.py" afterPath="$PROJECT_DIR$/download.py" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/start.py" afterPath="$PROJECT_DIR$/start.py" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/upload.py" afterPath="$PROJECT_DIR$/upload.py" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/work.py" afterPath="$PROJECT_DIR$/work.py" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
@@ -11,11 +18,11 @@
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="375">
<file leaf-file-name="upload.py" pinned="false" current-in-tab="true">
<file leaf-file-name="upload.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/upload.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="286">
<caret line="189" column="23" lean-forward="false" selection-start-line="189" selection-start-column="23" selection-end-line="189" selection-end-column="23" />
<state relative-caret-position="431">
<caret line="245" column="4" lean-forward="false" selection-start-line="245" selection-start-column="4" selection-end-line="245" selection-end-column="4" />
<folding>
<element signature="e#0#9#0" expanded="true" />
</folding>
@@ -26,7 +33,7 @@
<file leaf-file-name="get_cookie.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/get_cookie.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<state relative-caret-position="69">
<caret line="4" column="38" lean-forward="false" selection-start-line="4" selection-start-column="38" selection-end-line="4" selection-end-column="38" />
<folding>
<element signature="e#0#30#0" expanded="true" />
@@ -38,8 +45,8 @@
<file leaf-file-name="download.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/download.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
<caret line="208" column="46" lean-forward="true" selection-start-line="208" selection-start-column="46" selection-end-line="208" selection-end-column="46" />
<state relative-caret-position="247">
<caret line="195" column="25" lean-forward="false" selection-start-line="195" selection-start-column="25" selection-end-line="195" selection-end-column="25" />
<folding>
<element signature="e#0#20#0" expanded="true" />
</folding>
@@ -50,18 +57,18 @@
<file leaf-file-name="configlog.ini" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/configlog.ini">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-46">
<caret line="10" column="14" lean-forward="false" selection-start-line="10" selection-start-column="14" selection-end-line="10" selection-end-column="14" />
<state relative-caret-position="637">
<caret line="33" column="53" lean-forward="false" selection-start-line="33" selection-start-column="53" selection-end-line="33" selection-end-column="53" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="start.py" pinned="false" current-in-tab="false">
<file leaf-file-name="start.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/start.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="408">
<caret line="58" column="25" lean-forward="false" selection-start-line="58" selection-start-column="25" selection-end-line="58" selection-end-column="25" />
<state relative-caret-position="92">
<caret line="4" column="44" lean-forward="false" selection-start-line="4" selection-start-column="44" selection-end-line="4" selection-end-column="44" />
<folding />
</state>
</provider>
@@ -70,8 +77,8 @@
<file leaf-file-name="work.py" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/work.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="592">
<caret line="63" column="45" lean-forward="true" selection-start-line="63" selection-start-column="45" selection-end-line="63" selection-end-column="45" />
<state relative-caret-position="-1454">
<caret line="45" column="21" lean-forward="false" selection-start-line="45" selection-start-column="21" selection-end-line="45" selection-end-column="21" />
<folding>
<element signature="e#0#14#0" expanded="true" />
</folding>
@@ -118,18 +125,18 @@
<option value="$PROJECT_DIR$/get_cookie.py" />
<option value="$PROJECT_DIR$/total/__init__.py" />
<option value="$PROJECT_DIR$/total/download.py" />
<option value="$PROJECT_DIR$/start.py" />
<option value="$PROJECT_DIR$/configlog.ini" />
<option value="$PROJECT_DIR$/download.py" />
<option value="$PROJECT_DIR$/work.py" />
<option value="$PROJECT_DIR$/upload.py" />
<option value="$PROJECT_DIR$/configlog.ini" />
<option value="$PROJECT_DIR$/work.py" />
<option value="$PROJECT_DIR$/download.py" />
<option value="$PROJECT_DIR$/start.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="1">
<option name="x" value="566" />
<option name="y" value="74" />
<option name="width" value="1352" />
<component name="ProjectFrameBounds">
<option name="x" value="433" />
<option name="y" value="92" />
<option name="width" value="1362" />
<option name="height" value="879" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
@@ -148,8 +155,8 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="Scratches" />
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
@@ -164,7 +171,7 @@
</panes>
</component>
<component name="PropertiesComponent">
<property name="last_opened_file_path" value="D:/test" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RecentsManager">
@@ -184,7 +191,7 @@
</list>
</option>
</component>
<component name="RunManager" selected="Python.download">
<component name="RunManager" selected="Python.upload">
<configuration name="download" type="PythonConfigurationType" factoryName="Python" temporary="true">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
@@ -284,15 +291,14 @@
<servers />
</component>
<component name="ToolWindowManager">
<frame x="566" y="74" width="1352" height="879" extended-state="1" />
<editor active="true" />
<frame x="433" y="92" width="1362" height="879" extended-state="0" />
<layout>
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.27621284" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.2833851" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="WINDOWED" visible="false" show_stripe_button="true" weight="0.32968536" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" x="542" y="621" width="1143" height="252" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="WINDOWED" visible="false" show_stripe_button="true" weight="0.32968536" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" x="409" y="634" width="1143" height="252" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="WINDOWED" visible="false" show_stripe_button="true" weight="0.32968536" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" x="33" y="260" width="1274" height="433" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
@@ -592,13 +598,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/AUTO.log">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="17" lean-forward="false" selection-start-line="4" selection-start-column="17" selection-end-line="4" selection-end-column="17" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/python-skeletons/multiprocessing/managers.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="218">
@@ -621,7 +620,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="227">
<caret line="116" column="6" lean-forward="false" selection-start-line="116" selection-start-column="6" selection-end-line="116" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
@@ -629,7 +627,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2435">
<caret line="415" column="6" lean-forward="false" selection-start-line="415" selection-start-column="6" selection-end-line="415" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
@@ -637,21 +634,13 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="292">
<caret line="1019" column="47" lean-forward="false" selection-start-line="1019" selection-start-column="47" selection-end-line="1019" selection-end-column="47" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/ds_update">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/ds_update" />
<entry file="file://$PROJECT_DIR$/get_cookie.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<state relative-caret-position="69">
<caret line="4" column="38" lean-forward="false" selection-start-line="4" selection-start-column="38" selection-end-line="4" selection-end-column="38" />
<folding>
<element signature="e#0#30#0" expanded="true" />
@@ -659,36 +648,17 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.py">
<entry file="file://$PROJECT_DIR$/AUTO.log">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="408">
<caret line="58" column="25" lean-forward="false" selection-start-line="58" selection-start-column="25" selection-end-line="58" selection-end-column="25" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/configlog.ini">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-46">
<caret line="10" column="14" lean-forward="false" selection-start-line="10" selection-start-column="14" selection-end-line="10" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/download.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
<caret line="208" column="46" lean-forward="true" selection-start-line="208" selection-start-column="46" selection-end-line="208" selection-end-column="46" />
<folding>
<element signature="e#0#20#0" expanded="true" />
</folding>
<state relative-caret-position="92">
<caret line="4" column="17" lean-forward="false" selection-start-line="4" selection-start-column="17" selection-end-line="4" selection-end-column="17" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/work.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="592">
<caret line="63" column="45" lean-forward="true" selection-start-line="63" selection-start-column="45" selection-end-line="63" selection-end-column="45" />
<state relative-caret-position="-1454">
<caret line="45" column="21" lean-forward="false" selection-start-line="45" selection-start-column="21" selection-end-line="45" selection-end-column="21" />
<folding>
<element signature="e#0#14#0" expanded="true" />
</folding>
@@ -697,13 +667,39 @@
</entry>
<entry file="file://$PROJECT_DIR$/upload.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="286">
<caret line="189" column="23" lean-forward="false" selection-start-line="189" selection-start-column="23" selection-end-line="189" selection-end-column="23" />
<state relative-caret-position="431">
<caret line="245" column="4" lean-forward="false" selection-start-line="245" selection-start-column="4" selection-end-line="245" selection-end-column="4" />
<folding>
<element signature="e#0#9#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/configlog.ini">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="637">
<caret line="33" column="53" lean-forward="false" selection-start-line="33" selection-start-column="53" selection-end-line="33" selection-end-column="53" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/download.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="247">
<caret line="195" column="25" lean-forward="false" selection-start-line="195" selection-start-column="25" selection-end-line="195" selection-end-column="25" />
<folding>
<element signature="e#0#20#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/start.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="44" lean-forward="false" selection-start-line="4" selection-start-column="44" selection-end-line="4" selection-end-column="44" />
<folding />
</state>
</provider>
</entry>
</component>
</project>
README.md
+14
View File
@@ -1 +1,15 @@
# bilibiliupload
Linux模拟CTRL-c
kill -SIGINT PID
后台运行
nohup python3 start.py >/dev/null 2>&1 &
查看系统文件句柄使用
lsof -n |awk '{print $2}'|sort|uniq -c |sort -nr|more
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='api.twitch.tv', port=443): Max retries exceeded with url: /kraken/streams/innovation_s2 (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7fedd8032a20>: Failed to establish a new connection: [Errno 110] Connection timed out',))
youtube_dl.utils.DownloadError: ERROR: Live stream is offline
configlog.ini
+2 -2
View File
@@ -24,14 +24,14 @@ args=(sys.stdout,)
class=work.SafeRotatingFileHandler
level=INFO
formatter=form01
args=("ds_update.log", "D", 1, 3)
args=("ds_update.log", "W0", 1, 1)
[formatters]
keys=form01,form02
[formatter_form01]
format=%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s
format=%(asctime)s %(filename)s[line:%(lineno)d]（Pid：%(process)s） %(levelname)s %(message)s
[formatter_form02]
format=%(lineno)d[%(levelname)s]%(message)s
download.py
+61 -245
View File
@@ -1,5 +1,4 @@
from upload import *
from multiprocessing import Manager
import os
import requests
import json
@@ -10,13 +9,21 @@ import youtube_dl
# links = ['www.twitch.tv/innovation_s2','www.panda.tv/1160340','www.twitch.tv/sc2soo','www.panda.tv/1150595','www.twitch.tv/kimdaeyeob3']
root_url = ['www.twitch.tv/', 'www.panda.tv/', 'https://api.twitch.tv/kraken/streams/']
root_url = ['https://www.twitch.tv/', 'https://www.panda.tv/', 'https://api.twitch.tv/kraken/streams/']
links_id = {
'星际2innovation吕布卫星人族天梯第一视角': ['innovation_s2', '1160340'],
'星际2soo输本虫族天梯第一视角': ['sc2soo', '1150595'],
'星际2stats拔本神族天梯第一视角': ['kimdaeyeob3'],
'星际2sos狗哥神族天梯第一视角': ['', '1160930']
'星际2Innovation吕布卫星人族天梯第一视角': ['innovation_s2', '1160340'],
'星际2soO输本虫族天梯第一视角': ['sc2soo', '1150595'],
'星际2sOs狗哥神族天梯第一视角': ['', '1160930'],
'星际2Stats拔本神族天梯第一视角': ['kimdaeyeob3'],
'星际2Dark暗本虫族天梯第一视角': ['qkrfuddn0'],
'星际2Scarlett噶姐虫族天梯第一视角': ['scarlettm'],
'星际2GuMiho砸本人族天梯第一视角': ['gumiho'],
'星际2Maru人族天梯第一视角': ['maru072'],
'星际2ByuN武圣人族天梯第一视角': ['byunprime'],
'星际2小herO神族天梯第一视角': ['dmadkr0818'],
'星际2Zest神族天梯第一视角': ['sc2_zest'],
'星际2PartinG跳跳胖丁神族天梯第一视角':['partingthebigboy']
# 'test':['maru072','37229'],
# 'test1':['byunprime','10003']
}
@@ -25,163 +32,47 @@ headers = {
'client-id': 'jzkbprff40iqj646a697cyrvl0zt2m6'
}
# log_fmt = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s'
# formatter = logging.Formatter(log_fmt)
# log_file_handler = TimedRotatingFileHandler(filename="ds_update.log", when="D", interval=1, backupCount=2)
# # log_file_handler.suffix = "%Y-%m-%d.log"
# # log_file_handler.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$")
# log_file_handler.setFormatter(formatter)
# logging.basicConfig(level=logging.INFO)
# logger = logging.getLogger(__name__)
# logger.addHandler(log_file_handler)
# def logg():
# # logger = logging.getLogger(__name__)
# logger.info('111')
# class SizeError(Exception):
# def __init__(self, info):
# Exception.__init__(self)
# self.errorinfo = info
#
# def __str__(self):
# return "SizeError:%s" % (self.errorinfo)
#
# def kill_child_processes(parent_pid, file_name_, sig=signal.SIGINT):
# file_name_ = file_name_+'.part'
# last_file_size = 0.0
# while True:
# time.sleep(10)
# if os.path.isfile(file_name_):
# file_size = os.path.getsize(file_name_)/1024/1024/1024
# file_sizes = os.path.getsize(file_name_)
# if float(file_sizes) == last_file_size:
# try:
# parent = psutil.Process(parent_pid)
# except psutil.NoSuchProcess:
# return
# children = parent.children(recursive=True)
# for process in children:
# # print(process)
# process.send_signal(sig)
# logger.info('下载卡死'+file_name_)
# break
#
#
# last_file_size = file_sizes
#
# if float(file_size) >= 3.9:
# try:
# parent = psutil.Process(parent_pid)
# except psutil.NoSuchProcess:
# return
# children = parent.children(recursive=True)
# for process in children:
# # print(process)
# process.send_signal(sig)
# print('分段下载')
# logger.info('分段下载'+file_name_)
# break
# else:
# os._exit(0)
# print(file_name_, '文件大小', file_size) # 单位GB
# try:
# parent = psutil.Process(parent_pid)
# except psutil.NoSuchProcess:
# return
# children = parent.children(recursive=True)
# for process in children:
# print(process)
# # process.send_signal(sig)
# def monitoring(q):
# while True:
# # print('开始监测')
# pid, file_name = q.get()
# time.sleep(5)
# # print('获取到{0}，{1}'.format(pid,file_name))
# p = Process(target=kill_child_processes, args=(pid, file_name))
# p.start()
# def uploads(file_name_, url_):
# logger.info('准备上传'+file_name_[:-4])
# if os.path.isfile(file_name_):
# os.rename(file_name_, file_name_[:-4] + str(time.time())[:10] + file_name_[-4:])
# file_list = get_file(file_name_)
# logger.debug('获取%s文件列表' % file_name_[:-4])
# videopath = ''
# root = os.getcwd()
# for i in range(len(file_list)):
# file = file_list[i]
# videopath += root + '/' + file + '\n'
# videopath = videopath.rstrip()
# upload(video_path=videopath, link=url_, title_=file_name_[:-4])
#
# for r in file_list:
# os.remove(r)
# logger.info('删除-' + r)
# def download_part():
# try:
# with youtube_dl.YoutubeDL(ydl_opts) as ydl:
# ydl.download([url_])
# except SizeError:
#
# async def download(file_name_,url_, ydl_):
# if os.path.isfile(file_name_):
# file_size = os.path.getsize(file_name_)
# print('文件大小',file_size)
# print(os.path.isfile(file_name_))
# await asyncio.sleep(3)#ydl_.download([url_])
#
#
# async def file_size(file_name_):
# file_size_ = os.path.getsize(file_name_)
# print('文件大小', file_size_)
# def is_err(file_name_):
# if os.path.getsize(file_name_) / 1024 / 1024 >= 5:
# raise SizeError('文件大小超出限制')
def get_twitch_stream(url, value):
def get_twitch_stream(url, key):
try:
res = requests.get(url, headers=headers)
res.close()
except requests.exceptions.ConnectionError:
logger.exception('During handling of the above exception, another exception occurred:')
return None
except requests.exceptions.SSLError:
logger.error('获取流信息发生错误')
logger.error(requests.exceptions.SSLError, exc_info = True)
return 'None'
return None
try:
s = json.loads(res.text)
# s = res.json()
except json.decoder.JSONDecodeError:
logger.exception('Expecting value')
return 'None'
print(value)
return None
print(key)
try:
stream = s['stream']
except KeyError :
logger.error(KeyError, exc_info = True)
return 'None'
return None
return stream
def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_):
def download_twitch_stream(dict, q, confirm_url, key_, url_, tfile_name_, value_=None):
status_ = get_twitch_stream(confirm_url,key_)
# print(status_)
if str(status_) != 'None':
value_ = dict[key_]
dict.pop(key_)
if value_ is None:
value_ = dict[key_]
dict.pop(key_)
else:
logger.info('准备递归下载'+key_)
print('开始下载twitch', key_)
logger.info('开始下载twitch：' + key_)
ydl_opts = {
'outtmpl': tfile_name_,
'format': '720p'
'format': '720p60'
# 'keep_fragments':True
# 'postprocessors': [{
# 'key': 'FFmpegFixupM3u8',
@@ -191,11 +82,7 @@ def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_):
}
try:
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
# loop = asyncio.get_event_loop()
# tasks = [download(file_name_+'.part',url_,ydl),file_size(file_name_+'.part')]
# loop.run_until_complete(download(file_name_+'.part',url_,ydl))
# loop.run_until_complete(asyncio.wait(tasks))
# loop.close()
pid = os.getpid()
q.put([pid, tfile_name_])
@@ -210,17 +97,16 @@ def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_):
# print('分段下载')
try:
logger.info('更名{0}'.format(tfile_name_ + '.part'))
os.rename(tfile_name_ + '.part', tfile_name_[:-4] + str(time.time())[:10] + tfile_name_[-4:])
logger.info('更名{0}为{1}+时间'.format(tfile_name_ + '.part', tfile_name_))
except FileExistsError:
os.rename(tfile_name_ + '.part', tfile_name_[:-4] + str(time.time())[:10] + tfile_name_[-4:])
download_twitch_stream(dict, q, confirm_url, key_, url_, tfile_name_, value_)
finally:
dict[key_] = value_
# print('下载完成')
# logger.info('下载完成'+key_)
# uploads(tfile_name_, url_)
# print(links_id)
logger.info('退出下载')
elif len(dict[key_]) == 1:
value_ = dict[key_]
@@ -228,21 +114,13 @@ def download_twitch_stream(dict, q, status_, key_, url_, tfile_name_):
supplemental_upload(dict, tfile_name_, key_, url_, value_)
# try:
# for f in os.listdir():
# if tfile_name_[:-15] in f:
# logger.info('补充上传' + key_)
# uploads(tfile_name_, url_)
# os._exit(0)
# finally:
# dict[key_] = value_
def download_panda_stream(dict, q, key_, url_, pfile_name_):
value_ = dict[key_]
dict.pop(key_)
# print('开始下载', key_)
# logger.debug('开始下载pandaTV' + key_)
def download_panda_stream(dict, q, key_, url_, pfile_name_, value_=None):
if value_ is None:
value_ = dict[key_]
dict.pop(key_)
else:
logger.info('准备递归下载'+key_)
list_info = []
with youtube_dl.YoutubeDL() as ydl:
try:
@@ -255,15 +133,6 @@ def download_panda_stream(dict, q, key_, url_, pfile_name_):
# logger.debug('准备补充上传'+key_)
supplemental_upload(dict, pfile_name_, key_, url_, value_)
return
# try:
# for f in os.listdir():
# if pfile_name_[:-15] in f:
# logger.info('补充上传' + key_)
# uploads(pfile_name_, url_)
# break
# finally:
# dict[key_] = value_
# os._exit(0)
if 'HD-flv' in list_info:
ydl_opts = {
@@ -286,105 +155,52 @@ def download_panda_stream(dict, q, key_, url_, pfile_name_):
pid = os.getpid()
# print(pid)
q.put([pid, pfile_name_])
# if os.path.isfile(pfile_name_):
# os.rename(pfile_name_, pfile_name_ + '.part')
# logger.info('存在{0}更名为{1}'.format(pfile_name_, pfile_name_ + '.part'))
logger.info('开始下载pandaTV' + key_)
ydl.download([url_])
print('下载完成')
logger.info('下载完成' + key_)
if os.path.isfile(pfile_name_):
os.rename(pfile_name_, pfile_name_ + '.part')
logger.info('存在{0}更名为{1}'.format(pfile_name_,pfile_name_ + '.part' ))
with youtube_dl.YoutubeDL() as ydl:
try:
info = ydl.extract_info(url_, download=False)
for i in info['formats']:
list_info.append(i['format_id'])
except youtube_dl.utils.DownloadError:
uploads(pfile_name_, url_)
return
logger.info('实际未下载完成' + key_)
if os.path.isfile(pfile_name_):
os.rename(pfile_name_, pfile_name_[:-4] + str(time.time())[:10] + pfile_name_[-4:])
logger.info('存在{0}更名为{1}'.format(pfile_name_, pfile_name_[:-4] + '时间' + pfile_name_[-4:]))
download_panda_stream(dict, q, key_, url_, pfile_name_, value_)
except KeyboardInterrupt:
# except youtube_dl.utils.DownloadError:
# print('分段下载')
try:
logger.info('更名{0}'.format(pfile_name_+ '.part'))
# logger.info('更名{0}'.format(pfile_name_+ '.part'))
os.rename(pfile_name_ + '.part', pfile_name_[:-4] + str(time.time())[:10] + pfile_name_[-4:])
logger.info('下载进程发生KeyboardInterrupt更名{0}为{1}+时间'.format(pfile_name_ + '.part', pfile_name_))
except FileExistsError:
os.rename(pfile_name_ + '.part', pfile_name_[:-4] + str(time.time())[:10] + pfile_name_[-4:])
download_panda_stream(dict, q, key_, url_, pfile_name_, value_)
finally:
dict[key_] = value_
# print('下载完成')
# logger.info('下载完成'+key_)
# os.rename(file_name_, file_name_[:-4] + str(time.time())[:10] + '.mp4')
# file_list = get_file(file_name_)
# videopath = ''
# root = os.getcwd()
# for i in range(len(file_list)):
# file = file_list[i]
# videopath += root + '/' + file + '\n'
# videopath = videopath.rstrip()
# upload(video_path=videopath, link=url_, title_=file_name_[:-4])
#
# for r in file_list:
# os.remove(r)
# logger.info('删除-'+r)
# print(links_id)
# uploads(pfile_name_,url_)
logger.info('退出下载')
def download_stream(dict, q, status_, key_, twitch_url_, panda_url_, tfile_name_, pfile_name_):
def download_stream(dict, q, confirm_url, key_, twitch_url_, panda_url_, tfile_name_, pfile_name_):
# print('子进程id%s'%(os.getpid()))
download_twitch_stream(dict, q, status_, key_, twitch_url_, tfile_name_)
download_twitch_stream(dict, q, confirm_url, key_, twitch_url_, tfile_name_)
download_panda_stream(dict, q, key_, panda_url_, pfile_name_)
if __name__ == '__main__':
manager = Manager()
# logger.info('123')
# d = manager.dict(links_id)
# while True:
# utc_dt = datetime.utcnow().replace(tzinfo=timezone.utc)
# bj_dt = utc_dt.astimezone(timezone(timedelta(hours=8)))
# now = bj_dt.strftime('%Y{0}%m{1}%d{2}').format(*'年月日')
#
# for key in d.copy():(status)
#
# # print(key)
# file_name = '%s%s.mp4' % (key, now)
# if len(links_id[key]) == 2:
# twitch_url = root_url[0]+links_id[key][0]
# panda_url = root_url[1]+links_id[key][1]
# confirm_url = root_url[2]+links_id[key][0]
# status = get_twitch_stream(confirm_url,key)
# # print # print('父进程id%s'%(os.getpid()))
# p = Process(target=download_stream, args=(d,status,key,twitch_url,panda_url,file_name))
# p.start()
#
# elif len(links_id[key]) ==1:
# twitch_url = root_url[0]+links_id[key][0]
# confirm_url = root_url[2] + links_id[key][0]
# status = get_twitch_stream(confirm_url, key)
# p = Process(target=download_twitch_stream, args=(d,status, key,twitch_url,file_name))
# p.start()
# time.sleep(5)
# print('test2018年01月22日0473.mp4')
# try:
# os.rename('1','2')
# except FileExistsError:
# print('test2018年01月22日.mp4')
# root = os.getcwd()
# print(root)
# print(os.listdir())
# get_twitch_stream('https://api.twitch.tv/kraken/streams/',1)
# a = 'test2018年01月17日.mp4'
# print(a[-4:])
# a = ['111','2']
# if '11' in a:
# print('yes')
# else:
# print('88')
get_twitch_stream('https://api.twitch.tv/kraken/streams/1160340','233')
start.py
+9 -6
View File
@@ -2,13 +2,16 @@ import logging.config
logging.config.fileConfig('configlog.ini')
if __name__ == '__main__':
from datetime import datetime, timedelta, timezone
from multiprocessing import Pool
from multiprocessing import Pool,Manager
from download import *
from work import *
import sys
# logging.basicConfig(filename='AUTO.log', level=logging.INFO)
# logging.basicConfig(format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',\
# datefmt='%a, %d %b %Y %H:%M',level=logging.INFO)
sys.excepthook = new_hook
manager = Manager()
d = manager.dict(links_id)
@@ -34,11 +37,11 @@ if __name__ == '__main__':
panda_url = root_url[1] + links_id[key][1]
if links_id[key][0] != '':
confirm_url = root_url[2] + links_id[key][0]
status = get_twitch_stream(confirm_url, key)
# status = get_twitch_stream(confirm_url, key)
# print(status)
# print('父进程id%s'%(os.getpid()))
res = pool.apply_async(func=download_stream,
args=(d, queue, status, key, twitch_url, panda_url, tfile_name, pfile_name))
args=(d, queue, confirm_url, key, twitch_url, panda_url, tfile_name, pfile_name))
# p = Process(target=download_stream, args=(d, queue, status,key,twitch_url,panda_url,tfile_name, pfile_name))
# p.start()
else:
@@ -50,13 +53,13 @@ if __name__ == '__main__':
elif len(links_id[key]) == 1:
twitch_url = root_url[0] + links_id[key][0]
confirm_url = root_url[2] + links_id[key][0]
status = get_twitch_stream(confirm_url, key)
# status = get_twitch_stream(confirm_url, key)
res = pool.apply_async(func=download_twitch_stream,
args=(d, queue, status, key, twitch_url, tfile_name))
args=(d, queue, confirm_url, key, twitch_url, tfile_name))
# p = Process(target=download_twitch_stream, args=(d, queue, status, key,twitch_url,tfile_name))
# p.start()
# print(res.get())
time.sleep(60)
time.sleep(50)
# pool.close()
# pool.join()
upload.py
+13 -82
View File
@@ -1,5 +1,6 @@
import os
from selenium import webdriver
import selenium
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
@@ -9,15 +10,7 @@ import logging
# 10800 18000 4110
# log_fmt = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s'
# formatter = logging.Formatter(log_fmt)
# log_file_handler = TimedRotatingFileHandler(filename="ds_update.log", when="D", interval=1, backupCount=2)
# # log_file_handler.suffix = "%Y-%m-%d.log"
# # log_file_handler.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$")
# log_file_handler.setFormatter(formatter)
# logging.basicConfig(level=logging.INFO)
# logger = logging.getLogger(__name__)
# logger.addHandler(log_file_handler)
logger = logging.getLogger('log01')
# def excision(Path):
@@ -57,48 +50,8 @@ def upload(video_path, link, title_):
service_log_path=service_log_path)
# driver = webdriver.Chrome(executable_path=r'D:\bilibiliupload\chromedriver.exe',service_log_path=service_log_path)
driver.get("https://www.bilibili.com")
# cookie = {
# 'fts':'1511094141',
# 'buvid3':'0E299359-8055-4E58-9E82-DDF2F325CCAF26543infoc',
# 'UM_distinctid':'15fdc423182b22-07a38f91352cb7-7b113d-144000-15fdc423183a13',
# 'pgv_pvi':'3700880384',
# 'rpdid':'olklsxmolldosooiiqlpw',
# 'sid':'8k9huzd0',
# 'finger':'edc6ecda',
# 'DedeUserID':'274986200',
# 'DedeUserID__ckMd5':'1206e39bc044fe98',
# 'SESSDATA':'3e02f3d0%2C1517749345%2C489ac7f7',
# 'bili_jct':'4a71e9d389551c4bb7a9ebd05ac7dda0',
# 'member_v2':'1',
# 'pgv_si':'s2364394496',
# '_dfcaptcha':'8c4345e68b07f547aa3f246b7b058cd1',
# 'LIVE_BUVID':'a23404d890e536c9a93b72e5e50be145',
# 'LIVE_BUVID__ckMd5':'d5ea746722041186'
# }
# 964b42c0
# bahkjm41
# 1515221245
# EB49C459-95CA-415C-AA48-B3FE5B46BD9D31043infoc
# 3e02f3d0%2C1517813257%2C06f75553
# 1ef343ac61192cd3aa59ca9aea289ff3
# c25a35691b8ef1fc2f2566a1c4df50b4
# AUTO1115152212636167
# 814ce92affe021564ee8f5ceca806c4a
# cookies = [{'name': 'finger', 'value': 'edc6ecda', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': 'sid', 'value': '8k9huzd0', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': 'fts', 'value': '1511094141', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': 'buvid3', 'value': '0E299359-8055-4E58-9E82-DDF2F325CCAF26543infoc', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': 'DedeUserID', 'value': '274986200', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': 'DedeUserID__ckMd5', 'value': '1206e39bc044fe98', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': 'SESSDATA', 'value': '3e02f3d0%2C1517749345%2C489ac7f7', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': True},
# {'name': 'bili_jct', 'value': '4a71e9d389551c4bb7a9ebd05ac7dda0', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False},
# {'name': '_dfcaptcha', 'value': '8c4345e68b07f547aa3f246b7b058cd1', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': True},
# {'name': 'LIVE_BUVID', 'value': 'a23404d890e536c9a93b72e5e50be145', 'path': '/', 'domain': '.bilibili.com', 'expiry': None, 'secure': False, 'httpOnly': False}]
# driver.delete_all_cookies()
# driver.delete_all_cookies()
cookies = [
{'domain': '.bilibili.com', 'expiry': 1611825707.027044, 'httpOnly': False, 'name': 'buvid3', 'path': '/',
'secure': False, 'value': 'A558A031-653B-4890-917D-C205F81B84F93220infoc'},
@@ -125,36 +78,18 @@ def upload(video_path, link, title_):
# cookie.pop('domain')
# print(cookie)
driver.add_cookie(cookie)
# driver.add_cookie({'name': 'finger', 'value': 'edc6ecda'})
# driver.add_cookie({'name': 'fts', 'value': '1511094141'})
# driver.add_cookie({'name': 'buvid3', 'value': '0E299359-8055-4E58-9E82-DDF2F325CCAF26543infoc'})
# # driver.add_cookie({'name': 'UM_distinctid', 'value': '15fdc423182b22-07a38f91352cb7-7b113d-144000-15fdc423183a13'})
# # driver.add_cookie({'name': 'pgv_pvi', 'value': '3700880384'})
# # driver.add_cookie({'name': 'rpdid', 'value': 'olklsxmolldosooiiqlpw'})
# driver.add_cookie({'name': 'sid', 'value': '8k9huzd0'})
# driver.add_cookie({'name': 'DedeUserID', 'value': '274986200'})
# driver.add_cookie({'name': 'DedeUserID__ckMd5', 'value': '1206e39bc044fe98'})
# driver.add_cookie({'name': 'SESSDATA', 'value': '3e02f3d0%2C1517749345%2C489ac7f7'})
# driver.add_cookie({'name': 'bili_jct', 'value': '814ce92affe021564ee8f5ceca806c4a'})
# # driver.add_cookie({'name': 'member_v2', 'value': '1'})
# # driver.add_cookie({'name': 'pgv_si', 'value': 's2364394496'})
# driver.add_cookie({'name': '_dfcaptcha', 'value': '8c4345e68b07f547aa3f246b7b058cd1'})
# driver.add_cookie({'name': 'LIVE_BUVID', 'value': 'a23404d890e536c9a93b72e5e50be145'})
# # driver.add_cookie({'name': 'LIVE_BUVID__ckMd5', 'value': 'd5ea746722041186'})
# cookie = driver.get_cookies()
# print(cookie)
# driver.get("https://www.bilibili.com")
driver.get("https://member.bilibili.com/video/upload.html")
# driver.add_cookie(cookie)
# time.sleep(20)
# cookie = driver.get_cookies()
# print(cookie)
# print(driver.title)
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.NAME, 'file')))
upload = driver.find_element_by_name('file')
# time.sleep(5)
print(driver.title)
# logger.info(driver.title)
logger.info('开始上传' + title_)
@@ -186,15 +121,11 @@ def upload(video_path, link, title_):
# driver.execute_script('arguments[0].style.display="none";',hid)
# hid_1 = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[1]')
# driver.execute_script('arguments[0].style.display="none";',hid_1)
#
# time.sleep(1)
# 点击转载
click = driver.find_element_by_xpath(
'//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[2]/div')
# click = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[1]/div[1]/div[2]/div[1]')
# click = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[2]/input')
# click = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[1]/label[1]/input')
# click.send_keys(Keys.ENTER)
click.click()
# driver.execute_script('arguments[0].className = "is-checked";',click)
# print(click.is_selected())
@@ -203,9 +134,7 @@ def upload(video_path, link, title_):
# click_1.click()
# driver.find_elements_by_class_name('is-checked')[1].click()
# click_1 = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[3]/div[1]/div[2]/label/input')
# click_1 = driver.find_element_by_xpath('//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[3]/div[1]/div[2]/label/div')
# click_1.click()
# 输入转载来源
Input = driver.find_element_by_xpath(
'//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[1]/div/div[2]/div[2]/input')
@@ -224,7 +153,7 @@ def upload(video_path, link, title_):
title.send_keys(title_)
text = driver.find_element_by_xpath(
'//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[2]/div[2]/div[2]/div/div/input')
'//*[@id="app"]/div[2]/div[2]/div/div[2]/div[2]/div[2]/div[1]/div[4]/div[2]/div[2]/div[2]/div[last()]/div/input')
text.send_keys('星际争霸2')
text.send_keys(Keys.ENTER)
time.sleep(0.5)
@@ -250,7 +179,7 @@ def upload(video_path, link, title_):
print('稿件提交完成！')
logger.info('%s提交完成！' % title_)
except Exception:
except selenium.common.exceptions.NoSuchElementException:
logger.exception('发生错误')
finally:
driver.quit()
@@ -315,3 +244,5 @@ if __name__ == '__main__':
# for i in range(1):
# print(i)
os.rename('3', '4')
work.py
+7 -4
View File
@@ -1,6 +1,5 @@
import logging
import signal
from logging import FileHandler
from logging.handlers import TimedRotatingFileHandler
from multiprocessing import Process
import time
@@ -62,10 +61,10 @@ def kill_child_processes(parent_pid, file_name_, sig=signal.SIGINT):
# print(process)
process.send_signal(sig)
logger.info('下载卡死' + file_name_)
time.sleep(2)
time.sleep(0.5)
if os.path.isfile(file_name_):
logger.info('卡死下载进程可能未成功退出')
break
logger.info('卡死下载进程可能未成功退出PID:'+parent_pid+','+children)
continue
else:
logger.info('卡死下载进程成功退出')
break
@@ -104,6 +103,10 @@ def monitoring(q):
p.start()
def new_hook(t, v, tb):
logger.error("Uncaught exception：", exc_info=(t, v, tb))
class SafeRotatingFileHandler(TimedRotatingFileHandler):
def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False):
TimedRotatingFileHandler.__init__(self, filename, when, interval, backupCount, encoding, delay, utc)