From 8df04de1eac9a7e060b5f6a5b4e164c151531cc6 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Fri, 19 May 2023 22:11:05 +0100 Subject: [PATCH] Remove file size check from Requests downloader We cannot actually do this check. The Content-Length value will be the size after being further encoded or compressed. While we can find out what it was compressed with via the Content-Encoding header, we cannot match the downloaded length with the Content-Length header as requests will automatically decompress/decode according to the Content-Encoding header. --- devine/core/downloaders/requests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py index d2d2fa5..1ddf599 100644 --- a/devine/core/downloaders/requests.py +++ b/devine/core/downloaders/requests.py @@ -57,7 +57,6 @@ def requests( for url, out_path in uri: out_path.parent.mkdir(parents=True, exist_ok=True) stream = session.get(url, stream=True) - file_size = stream.headers.get("Content-Length") with open(out_path, "wb") as f: written = 0 for chunk in stream.iter_content(chunk_size=1024): @@ -77,9 +76,6 @@ def requests( progress(downloaded=f"{filesize.decimal(download_speed)}/s") last_speed_refresh = now download_sizes.clear() - if file_size and written < int(file_size): - raise ValueError( - f"{url} finished downloading unexpectedly, got {decimal(written)}/{decimal(int(file_size))}") return 0