From 1bff87bd70ce83e3a0ead374d3e2d62b0ca39aec Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Tue, 12 Mar 2024 01:06:42 +0000 Subject: [PATCH] fix(requests): Set HTTP pool connections/maxsize to max workers This allows requests to open and save/cache up to *max_workers* amount of TCP connections. In most situations it will still only save and re-use one TCP Connection since it always tries to re-use the connection if one is available. However, in situations where downloads are from more than 10 Host/Port combinations (the default pool connections/maxsize) then this will improve download speeds. --- devine/core/downloaders/requests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py index 296b7dd..54ec0f7 100644 --- a/devine/core/downloaders/requests.py +++ b/devine/core/downloaders/requests.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any, Generator, MutableMapping, Optional, Union from requests import Session +from requests.adapters import HTTPAdapter from rich import filesize from devine.core.constants import DOWNLOAD_CANCELLED @@ -211,6 +212,12 @@ def requests( ] session = Session() + session.mount("https://", HTTPAdapter( + pool_connections=max_workers, + pool_maxsize=max_workers + )) + session.mount("http://", session.adapters["https://"]) + if headers: headers = { k: v