Merge pull request 'master' (#3 ) from DRMTalks/devine:master into master

Reviewed-on: #3
fix(requests): Block until connection freed if too many connections
2024-03-14 03:25:35 +00:00 · 2024-03-13 17:15:13 +00:00 · 2024-03-13 17:12:06 +00:00
1 changed files with 6 additions and 1 deletions
--- a/devine/core/downloaders/requests.py
+++ b/devine/core/downloaders/requests.py
@ -1,4 +1,5 @@
 import math
+import os
 import time
 from concurrent import futures
 from concurrent.futures.thread import ThreadPoolExecutor
@ -196,6 +197,9 @@ def requests(
    if not isinstance(urls, list):
        urls = [urls]

+    if not max_workers:
+        max_workers = min(32, (os.cpu_count() or 1) + 4)
+
    urls = [
        dict(
            save_path=save_path,
@ -214,7 +218,8 @@ def requests(
    session = Session()
    session.mount("https://", HTTPAdapter(
        pool_connections=max_workers,
-        pool_maxsize=max_workers
+        pool_maxsize=max_workers,
+        pool_block=True
    ))
    session.mount("http://", session.adapters["https://"])
Author	SHA1	Message	Date
Ninja Jiraiya	2d7440ed95	Merge pull request 'master' (#3 ) from DRMTalks/devine:master into master Reviewed-on: #3	2024-03-14 03:25:35 +00:00
rlaphoenix	e7294c95d1	fix(requests): Block until connection freed if too many connections	2024-03-13 17:15:13 +00:00
rlaphoenix	36b070f729	fix(requests): Manually compute default max_workers or pool size is None	2024-03-13 17:12:06 +00:00