ninjajiraiya
/
devine
forked from DRMTalks/devine
1
0
Fork 0
Code Pull Requests Activity

Add retry attempts to Requests downloader

This commit is contained in:
rlaphoenix 2024-01-09 12:08:49 +00:00
parent fa3cee11b7
commit 552a0f13a5
1 changed files with 32 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
devine/core/downloaders/requests.py
View File

@ -7,6 +7,11 @@ from requests import Session
from requests.cookies import RequestsCookieJar from requests.cookies import RequestsCookieJar
from rich import filesize from rich import filesize
from devine.core.constants import DOWNLOAD_CANCELLED
MAX_ATTEMPTS = 5
RETRY_WAIT = 2
def requests( def requests(
uri: Union[str, list[str]], uri: Union[str, list[str]],
@ -59,7 +64,10 @@ def requests(
for url, out_path in uri: for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
attempts = 1
try:
stream = session.get(url, stream=True) stream = session.get(url, stream=True)
stream.raise_for_status()
with open(out_path, "wb") as f: with open(out_path, "wb") as f:
written = 0 written = 0
for chunk in stream.iter_content(chunk_size=1024): for chunk in stream.iter_content(chunk_size=1024):
@ -79,6 +87,12 @@ def requests(
progress(downloaded=f"{filesize.decimal(download_speed)}/s") progress(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now last_speed_refresh = now
download_sizes.clear() download_sizes.clear()
break
except Exception as e:
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
raise e
time.sleep(RETRY_WAIT)
attempts += 1
return 0 return 0