From 36b070f729897fa69f844f9b08c34d766f264887 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Wed, 13 Mar 2024 17:12:06 +0000 Subject: [PATCH] fix(requests): Manually compute default max_workers or pool size is None --- devine/core/downloaders/requests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py index 54ec0f7..cfaece8 100644 --- a/devine/core/downloaders/requests.py +++ b/devine/core/downloaders/requests.py @@ -1,4 +1,5 @@ import math +import os import time from concurrent import futures from concurrent.futures.thread import ThreadPoolExecutor @@ -196,6 +197,9 @@ def requests( if not isinstance(urls, list): urls = [urls] + if not max_workers: + max_workers = min(32, (os.cpu_count() or 1) + 4) + urls = [ dict( save_path=save_path,