ninjajiraiya
/
devine
forked from DRMTalks/devine
1
0
Fork 0
Code Pull Requests Activity

Add basic control file to Requests and Curl-Impersonate downloaders

This commit is contained in:
rlaphoenix 2024-01-23 10:06:42 +00:00
parent c53330046c
commit de7122a179
2 changed files with 88 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
devine/core/downloaders/curl_impersonate.py
View File

@ -69,38 +69,51 @@ def curl_impersonate(
for url, out_path in uri: for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
control_file = out_path.with_name(f"{out_path.name}.!dev")
if control_file.exists():
# consider the file corrupt if the control file exists
# TODO: Design a control file format so we know how much of the file is missing
out_path.unlink(missing_ok=True)
control_file.unlink()
elif out_path.exists():
continue
control_file.write_bytes(b"")
attempts = 1 attempts = 1
try:
while True:
try:
stream = session.get(url, stream=True)
stream.raise_for_status()
with open(out_path, "wb") as f:
written = 0
for chunk in stream.iter_content(chunk_size=1024):
download_size = len(chunk)
f.write(chunk)
written += download_size
if progress:
progress(advance=1)
while True: now = time.time()
try: time_since = now - last_speed_refresh
stream = session.get(url, stream=True)
stream.raise_for_status()
with open(out_path, "wb") as f:
written = 0
for chunk in stream.iter_content(chunk_size=1024):
download_size = len(chunk)
f.write(chunk)
written += download_size
if progress:
progress(advance=1)
now = time.time() download_sizes.append(download_size)
time_since = now - last_speed_refresh if time_since > 5 or download_size < 1024:
data_size = sum(download_sizes)
download_sizes.append(download_size) download_speed = data_size / (time_since or 1)
if time_since > 5 or download_size < 1024: progress(downloaded=f"{filesize.decimal(download_speed)}/s")
data_size = sum(download_sizes) last_speed_refresh = now
download_speed = data_size / (time_since or 1) download_sizes.clear()
progress(downloaded=f"{filesize.decimal(download_speed)}/s") break
last_speed_refresh = now except Exception as e:
download_sizes.clear() out_path.unlink(missing_ok=True)
break if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
except Exception as e: raise e
out_path.unlink(missing_ok=True) time.sleep(RETRY_WAIT)
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS: attempts += 1
raise e finally:
time.sleep(RETRY_WAIT) control_file.unlink()
attempts += 1
return 0 return 0

79
devine/core/downloaders/requests.py
View File

@ -65,44 +65,57 @@ def requests(
for url, out_path in uri: for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
control_file = out_path.with_name(f"{out_path.name}.!dev")
if control_file.exists():
# consider the file corrupt if the control file exists
# TODO: Design a control file format so we know how much of the file is missing
out_path.unlink(missing_ok=True)
control_file.unlink()
elif out_path.exists():
continue
control_file.write_bytes(b"")
attempts = 1 attempts = 1
try:
while True:
try:
stream = session.get(url, stream=True)
stream.raise_for_status()
while True: if len(uri) == 1 and progress:
try: content_length = int(stream.headers.get("Content-Length", "0"))
stream = session.get(url, stream=True) if content_length > 0:
stream.raise_for_status() progress(total=math.ceil(content_length / 1024))
if len(uri) == 1 and progress: with open(out_path, "wb") as f:
content_length = int(stream.headers.get("Content-Length", "0")) written = 0
if content_length > 0: for chunk in stream.iter_content(chunk_size=1024):
progress(total=math.ceil(content_length / 1024)) download_size = len(chunk)
f.write(chunk)
written += download_size
if progress:
progress(advance=1)
with open(out_path, "wb") as f: now = time.time()
written = 0 time_since = now - last_speed_refresh
for chunk in stream.iter_content(chunk_size=1024):
download_size = len(chunk)
f.write(chunk)
written += download_size
if progress:
progress(advance=1)
now = time.time() download_sizes.append(download_size)
time_since = now - last_speed_refresh if time_since > 5 or download_size < 1024:
data_size = sum(download_sizes)
download_sizes.append(download_size) download_speed = data_size / (time_since or 1)
if time_since > 5 or download_size < 1024: progress(downloaded=f"{filesize.decimal(download_speed)}/s")
data_size = sum(download_sizes) last_speed_refresh = now
download_speed = data_size / (time_since or 1) download_sizes.clear()
progress(downloaded=f"{filesize.decimal(download_speed)}/s") break
last_speed_refresh = now except Exception as e:
download_sizes.clear() out_path.unlink(missing_ok=True)
break if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
except Exception as e: raise e
out_path.unlink(missing_ok=True) time.sleep(RETRY_WAIT)
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS: attempts += 1
raise e finally:
time.sleep(RETRY_WAIT) control_file.unlink()
attempts += 1
return 0 return 0