fix(requests): Set HTTP pool connections/maxsize to max workers

This allows requests to open and save/cache up to *max_workers* amount of TCP connections. In most situations it will still only save and re-use one TCP Connection since it always tries to re-use the connection if one is available.

However, in situations where downloads are from more than 10 Host/Port combinations (the default pool connections/maxsize) then this will improve download speeds.
This commit is contained in:
rlaphoenix 2024-03-12 01:06:42 +00:00
parent 5376e4c042
commit 1bff87bd70
1 changed files with 7 additions and 0 deletions

View File

@ -7,6 +7,7 @@ from pathlib import Path
from typing import Any, Generator, MutableMapping, Optional, Union
from requests import Session
from requests.adapters import HTTPAdapter
from rich import filesize
from devine.core.constants import DOWNLOAD_CANCELLED
@ -211,6 +212,12 @@ def requests(
]
session = Session()
session.mount("https://", HTTPAdapter(
pool_connections=max_workers,
pool_maxsize=max_workers
))
session.mount("http://", session.adapters["https://"])
if headers:
headers = {
k: v