ninjajiraiya
/
devine
forked from DRMTalks/devine
1
0
Fork 0
Code Pull Requests Activity

Remove file size check from Requests downloader 

We cannot actually do this check. The Content-Length value will be the size after being further encoded or compressed. While we can find out what it was compressed with via the Content-Encoding header, we cannot match the downloaded length with the Content-Length header as requests will automatically decompress/decode according to the Content-Encoding header.
This commit is contained in:
rlaphoenix 2023-05-19 22:11:05 +01:00
parent 8ada6165e3
commit 8df04de1ea
1 changed files with 0 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
devine/core/downloaders/requests.py
View File

@ -57,7 +57,6 @@ def requests(
for url, out_path in uri: for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
stream = session.get(url, stream=True) stream = session.get(url, stream=True)
file_size = stream.headers.get("Content-Length")
with open(out_path, "wb") as f: with open(out_path, "wb") as f:
written = 0 written = 0
for chunk in stream.iter_content(chunk_size=1024): for chunk in stream.iter_content(chunk_size=1024):
@ -77,9 +76,6 @@ def requests(
progress(downloaded=f"{filesize.decimal(download_speed)}/s") progress(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now last_speed_refresh = now
download_sizes.clear() download_sizes.clear()
if file_size and written < int(file_size):
raise ValueError(
f"{url} finished downloading unexpectedly, got {decimal(written)}/{decimal(int(file_size))}")
return 0 return 0