mirror of https://github.com/devine-dl/devine.git
fix(requests): Set HTTP pool connections/maxsize to max workers
This allows requests to open and save/cache up to *max_workers* amount of TCP connections. In most situations it will still only save and re-use one TCP Connection since it always tries to re-use the connection if one is available. However, in situations where downloads are from more than 10 Host/Port combinations (the default pool connections/maxsize) then this will improve download speeds.
This commit is contained in:
parent
5376e4c042
commit
1bff87bd70
|
@ -7,6 +7,7 @@ from pathlib import Path
|
|||
from typing import Any, Generator, MutableMapping, Optional, Union
|
||||
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter
|
||||
from rich import filesize
|
||||
|
||||
from devine.core.constants import DOWNLOAD_CANCELLED
|
||||
|
@ -211,6 +212,12 @@ def requests(
|
|||
]
|
||||
|
||||
session = Session()
|
||||
session.mount("https://", HTTPAdapter(
|
||||
pool_connections=max_workers,
|
||||
pool_maxsize=max_workers
|
||||
))
|
||||
session.mount("http://", session.adapters["https://"])
|
||||
|
||||
if headers:
|
||||
headers = {
|
||||
k: v
|
||||
|
|
Loading…
Reference in New Issue