From 36b070f729897fa69f844f9b08c34d766f264887 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Wed, 13 Mar 2024 17:12:06 +0000 Subject: [PATCH 1/2] fix(requests): Manually compute default max_workers or pool size is None --- devine/core/downloaders/requests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py index 54ec0f7..cfaece8 100644 --- a/devine/core/downloaders/requests.py +++ b/devine/core/downloaders/requests.py @@ -1,4 +1,5 @@ import math +import os import time from concurrent import futures from concurrent.futures.thread import ThreadPoolExecutor @@ -196,6 +197,9 @@ def requests( if not isinstance(urls, list): urls = [urls] + if not max_workers: + max_workers = min(32, (os.cpu_count() or 1) + 4) + urls = [ dict( save_path=save_path, -- 2.34.1 From e7294c95d14c67d933d8a2571729c9431523e7c5 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Wed, 13 Mar 2024 17:15:13 +0000 Subject: [PATCH 2/2] fix(requests): Block until connection freed if too many connections --- devine/core/downloaders/requests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py index cfaece8..c6d23fc 100644 --- a/devine/core/downloaders/requests.py +++ b/devine/core/downloaders/requests.py @@ -218,7 +218,8 @@ def requests( session = Session() session.mount("https://", HTTPAdapter( pool_connections=max_workers, - pool_maxsize=max_workers + pool_maxsize=max_workers, + pool_block=True )) session.mount("http://", session.adapters["https://"]) -- 2.34.1