ninjajiraiya
/
devine
forked from DRMTalks/devine
1
0
Fork 0
Code Pull Requests Activity

Add basic control file to Requests and Curl-Impersonate downloaders

This commit is contained in:
rlaphoenix 2024-01-23 10:06:42 +00:00
parent c53330046c
commit de7122a179
2 changed files with 88 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
devine/core/downloaders/curl_impersonate.py
View File

@ -69,8 +69,19 @@ def curl_impersonate(
for url, out_path in uri: for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
attempts = 1
control_file = out_path.with_name(f"{out_path.name}.!dev")
if control_file.exists():
# consider the file corrupt if the control file exists
# TODO: Design a control file format so we know how much of the file is missing
out_path.unlink(missing_ok=True)
control_file.unlink()
elif out_path.exists():
continue
control_file.write_bytes(b"")
attempts = 1
try:
while True: while True:
try: try:
stream = session.get(url, stream=True) stream = session.get(url, stream=True)
@ -101,6 +112,8 @@ def curl_impersonate(
raise e raise e
time.sleep(RETRY_WAIT) time.sleep(RETRY_WAIT)
attempts += 1 attempts += 1
finally:
control_file.unlink()
return 0 return 0

15
devine/core/downloaders/requests.py
View File

@ -65,8 +65,19 @@ def requests(
for url, out_path in uri: for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
attempts = 1
control_file = out_path.with_name(f"{out_path.name}.!dev")
if control_file.exists():
# consider the file corrupt if the control file exists
# TODO: Design a control file format so we know how much of the file is missing
out_path.unlink(missing_ok=True)
control_file.unlink()
elif out_path.exists():
continue
control_file.write_bytes(b"")
attempts = 1
try:
while True: while True:
try: try:
stream = session.get(url, stream=True) stream = session.get(url, stream=True)
@ -103,6 +114,8 @@ def requests(
raise e raise e
time.sleep(RETRY_WAIT) time.sleep(RETRY_WAIT)
attempts += 1 attempts += 1
finally:
control_file.unlink()
return 0 return 0