From 2a4e9505f14557d0b229e9b6d52bfae4575e79e3 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Tue, 16 May 2023 20:47:26 +0100 Subject: [PATCH] Remove unnecessary HEAD calls in requests downloader HEAD requests were made to sum a total file size of the download operation. However, the downloader is may be used on URLs where the content is not segmented media. Therefore, the server may not support or respond with the Content-Length header which causes the requests downloader to crash before it even gets a chance to begin downloading. Even still, this total size value isn't really necessary, and would cause possibly 100s of HEAD requests (in quick succession of each other) on segmented sources. It would also add up-front delay before it actually starts to download. --- devine/core/downloaders/requests.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py index b837716..2d1f7f8 100644 --- a/devine/core/downloaders/requests.py +++ b/devine/core/downloaders/requests.py @@ -48,13 +48,8 @@ def requests( if proxy: session.proxies.update({"all": proxy}) - total_size = sum( - int(session.head(url).headers["Content-Length"]) - for url, _ in uri - ) - if progress: - progress(total=total_size) + progress(total=len(uri)) download_sizes = [] last_speed_refresh = time.time() @@ -70,7 +65,7 @@ def requests( f.write(chunk) written += download_size if progress: - progress(advance=download_size) + progress(advance=1) now = time.time() time_since = now - last_speed_refresh