forked from DRMTalks/devine
Add retry attempts to Requests downloader
This commit is contained in:
parent
fa3cee11b7
commit
552a0f13a5
|
@ -7,6 +7,11 @@ from requests import Session
|
|||
from requests.cookies import RequestsCookieJar
|
||||
from rich import filesize
|
||||
|
||||
from devine.core.constants import DOWNLOAD_CANCELLED
|
||||
|
||||
MAX_ATTEMPTS = 5
|
||||
RETRY_WAIT = 2
|
||||
|
||||
|
||||
def requests(
|
||||
uri: Union[str, list[str]],
|
||||
|
@ -59,7 +64,10 @@ def requests(
|
|||
|
||||
for url, out_path in uri:
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
attempts = 1
|
||||
try:
|
||||
stream = session.get(url, stream=True)
|
||||
stream.raise_for_status()
|
||||
with open(out_path, "wb") as f:
|
||||
written = 0
|
||||
for chunk in stream.iter_content(chunk_size=1024):
|
||||
|
@ -79,6 +87,12 @@ def requests(
|
|||
progress(downloaded=f"{filesize.decimal(download_speed)}/s")
|
||||
last_speed_refresh = now
|
||||
download_sizes.clear()
|
||||
break
|
||||
except Exception as e:
|
||||
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
|
||||
raise e
|
||||
time.sleep(RETRY_WAIT)
|
||||
attempts += 1
|
||||
|
||||
return 0
|
||||
|
||||
|
|
Loading…
Reference in New Issue