Compare commits

...

3 Commits

Author SHA1 Message Date
Ninja Jiraiya 905f5706eb Merge pull request 'master' (#1) from DRMTalks/devine:master into master
Reviewed-on: #1
2024-03-12 02:57:18 +00:00
rlaphoenix 1bff87bd70 fix(requests): Set HTTP pool connections/maxsize to max workers
This allows requests to open and save/cache up to *max_workers* amount of TCP connections. In most situations it will still only save and re-use one TCP Connection since it always tries to re-use the connection if one is available.

However, in situations where downloads are from more than 10 Host/Port combinations (the default pool connections/maxsize) then this will improve download speeds.
2024-03-12 01:06:42 +00:00
rlaphoenix 5376e4c042 refactor(Service): Go back to the default pool_maxsize in Session
The pool_maxsize value here isn't actually doing much. It should have also been applied to pool_connections. What we realistically needed was just pool_block to prevent opening too much connections (causing a warning). The default pool_connections=10 and pool_maxsize=10 is fine. The downloader doesn't currently use this value.
2024-03-12 00:59:30 +00:00
2 changed files with 7 additions and 3 deletions

View File

@ -7,6 +7,7 @@ from pathlib import Path
from typing import Any, Generator, MutableMapping, Optional, Union from typing import Any, Generator, MutableMapping, Optional, Union
from requests import Session from requests import Session
from requests.adapters import HTTPAdapter
from rich import filesize from rich import filesize
from devine.core.constants import DOWNLOAD_CANCELLED from devine.core.constants import DOWNLOAD_CANCELLED
@ -211,6 +212,12 @@ def requests(
] ]
session = Session() session = Session()
session.mount("https://", HTTPAdapter(
pool_connections=max_workers,
pool_maxsize=max_workers
))
session.mount("http://", session.adapters["https://"])
if headers: if headers:
headers = { headers = {
k: v k: v

View File

@ -98,9 +98,6 @@ class Service(metaclass=ABCMeta):
backoff_factor=0.2, backoff_factor=0.2,
status_forcelist=[429, 500, 502, 503, 504] status_forcelist=[429, 500, 502, 503, 504]
), ),
# 16 connections is used for byte-ranged downloads
# double it to allow for 16 non-related connections
pool_maxsize=16 * 2,
pool_block=True pool_block=True
)) ))
session.mount("http://", session.adapters["https://"]) session.mount("http://", session.adapters["https://"])