From 10285c3819af6cc49a582d9e4f00a80776491806 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Tue, 2 Apr 2024 23:42:37 +0100 Subject: [PATCH] feat(dl): Add *new* --workers to set download threads/workers The previously named --workers which is now --downloads specified how many tracks to download, not how many threads/workers are used per-download. It defaults to nothing, which each downloader then has their own defaults. All current downloaders though currently default to `min(32, (os.cpu_count() or 1) + 4)`, which is also the default for `ThreadPoolExecutor` in general. This also brings a side effect of changing DASH and HLS's forced max_workers of 16 to now a more appropriate default but more importantly actually configurable. You can set a default in your config under `dl.workers`. --- CONFIG.md | 7 +++++++ devine/commands/dl.py | 4 ++++ devine/core/manifests/dash.py | 3 ++- devine/core/manifests/hls.py | 3 ++- devine/core/tracks/subtitle.py | 3 ++- devine/core/tracks/track.py | 6 +++++- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 8b4d450..77c40b5 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -153,6 +153,13 @@ For example to set the default primary language to download to German, lang: de ``` +to set how many tracks to download concurrently to 4 and download threads to 16, + +```yaml +downloads: 4 +workers: 16 +``` + to set `--bitrate=CVBR` for the AMZN service, ```yaml diff --git a/devine/commands/dl.py b/devine/commands/dl.py index a9c4538..61ae360 100644 --- a/devine/commands/dl.py +++ b/devine/commands/dl.py @@ -131,6 +131,8 @@ class dl: help="Disable folder creation for TV Shows.") @click.option("--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path.") + @click.option("--workers", type=int, default=None, + help="Max workers/threads to download with per-track. Default depends on the downloader.") @click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.") @click.pass_context @@ -275,6 +277,7 @@ class dl: no_proxy: bool, no_folder: bool, no_source: bool, + workers: Optional[int], downloads: int, *_: Any, **__: Any @@ -528,6 +531,7 @@ class dl: vaults_only=vaults_only, export=export ), + max_workers=workers, progress=tracks_progress_callables[i] ) for i, track in enumerate(title.tracks) diff --git a/devine/core/manifests/dash.py b/devine/core/manifests/dash.py index ec3a731..c7eab18 100644 --- a/devine/core/manifests/dash.py +++ b/devine/core/manifests/dash.py @@ -236,6 +236,7 @@ class DASH: progress: partial, session: Optional[Session] = None, proxy: Optional[str] = None, + max_workers: Optional[int] = None, license_widevine: Optional[Callable] = None ): if not session: @@ -472,7 +473,7 @@ class DASH: headers=session.headers, cookies=session.cookies, proxy=proxy, - max_workers=16 + max_workers=max_workers ): file_downloaded = status_update.get("file_downloaded") if file_downloaded: diff --git a/devine/core/manifests/hls.py b/devine/core/manifests/hls.py index d371030..deb0967 100644 --- a/devine/core/manifests/hls.py +++ b/devine/core/manifests/hls.py @@ -197,6 +197,7 @@ class HLS: progress: partial, session: Optional[Session] = None, proxy: Optional[str] = None, + max_workers: Optional[int] = None, license_widevine: Optional[Callable] = None ) -> None: if not session: @@ -280,7 +281,7 @@ class HLS: headers=session.headers, cookies=session.cookies, proxy=proxy, - max_workers=16 + max_workers=max_workers ): file_downloaded = status_update.get("file_downloaded") if file_downloaded: diff --git a/devine/core/tracks/subtitle.py b/devine/core/tracks/subtitle.py index 7a64e43..bfeaa38 100644 --- a/devine/core/tracks/subtitle.py +++ b/devine/core/tracks/subtitle.py @@ -153,9 +153,10 @@ class Subtitle(Track): self, session: requests.Session, prepare_drm: partial, + max_workers: Optional[int] = None, progress: Optional[partial] = None ): - super().download(session, prepare_drm, progress) + super().download(session, prepare_drm, max_workers, progress) if not self.path: return diff --git a/devine/core/tracks/track.py b/devine/core/tracks/track.py index 7dadf3f..f91e678 100644 --- a/devine/core/tracks/track.py +++ b/devine/core/tracks/track.py @@ -135,6 +135,7 @@ class Track: self, session: Session, prepare_drm: partial, + max_workers: Optional[int] = None, progress: Optional[partial] = None ): """Download and optionally Decrypt this Track.""" @@ -191,6 +192,7 @@ class Track: progress=progress, session=session, proxy=proxy, + max_workers=max_workers, license_widevine=prepare_drm ) elif self.descriptor == self.Descriptor.DASH: @@ -201,6 +203,7 @@ class Track: progress=progress, session=session, proxy=proxy, + max_workers=max_workers, license_widevine=prepare_drm ) elif self.descriptor == self.Descriptor.URL: @@ -236,7 +239,8 @@ class Track: filename=save_path.name, headers=session.headers, cookies=session.cookies, - proxy=proxy + proxy=proxy, + max_workers=max_workers ): file_downloaded = status_update.get("file_downloaded") if not file_downloaded: