From ce53a1b636b6eb3bc29b1f29be061eddb98c50d3 Mon Sep 17 00:00:00 2001 From: rlaphoenix Date: Tue, 28 Feb 2023 08:18:30 +0000 Subject: [PATCH] Don't run aria2c under asyncio, further improve progress updates I've removed asyncio usage as it's generally unnecessary. If you want to run aria2c under a thread, run it under a thread. In the case for devine, this would take another thread, and would be another thread layer deep. Pointless. Would affect speed. With this change I've been able to improve the aria2c progress capture code quite a bit. --- devine/commands/dl.py | 13 +++-- devine/core/downloaders/aria2c.py | 88 +++++++++++++------------------ devine/core/manifests/dash.py | 13 +++-- devine/core/manifests/hls.py | 13 +++-- 4 files changed, 54 insertions(+), 73 deletions(-) diff --git a/devine/commands/dl.py b/devine/commands/dl.py index d34a6db..3462bb5 100644 --- a/devine/commands/dl.py +++ b/devine/commands/dl.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import html import logging import math @@ -756,13 +755,13 @@ class dl: # no else-if as DASH may convert the track to URL descriptor if track.descriptor == track.Descriptor.URL: - asyncio.run(aria2c( - track.url, - save_path, - service.session.headers, - proxy if track.needs_proxy else None, + aria2c( + uri=track.url, + out=save_path, + headers=service.session.headers, + proxy=proxy if track.needs_proxy else None, progress=progress - )) + ) track.path = save_path if not track.drm and isinstance(track, (Video, Audio)): diff --git a/devine/core/downloaders/aria2c.py b/devine/core/downloaders/aria2c.py index 486e6f7..d0886e3 100644 --- a/devine/core/downloaders/aria2c.py +++ b/devine/core/downloaders/aria2c.py @@ -1,6 +1,4 @@ -import asyncio import subprocess -from asyncio import IncompleteReadError from functools import partial from pathlib import Path from typing import Optional, Union @@ -12,7 +10,7 @@ from devine.core.console import console from devine.core.utilities import get_binary_path, start_pproxy -async def aria2c( +def aria2c( uri: Union[str, list[str]], out: Path, headers: Optional[dict] = None, @@ -81,68 +79,54 @@ async def aria2c( if proxy.lower().split(":")[0] != "http": # HTTPS proxies are not supported by aria2(c). # Proxy the proxy via pproxy to access it as an HTTP proxy. - async with start_pproxy(proxy) as pproxy_: - return await aria2c(uri, out, headers, pproxy_) + with start_pproxy(proxy) as pproxy_: + return aria2c(uri, out, headers, pproxy_) arguments += ["--all-proxy", proxy] - p = await asyncio.create_subprocess_exec( - executable, - *arguments, + p = subprocess.Popen( + [executable, *arguments], stdin=subprocess.PIPE, stderr=[None, subprocess.DEVNULL][silent], stdout=( subprocess.PIPE if progress else subprocess.DEVNULL if silent else None - ) + ), + universal_newlines=True ) - p.stdin.write(uri.encode()) - await p.stdin.drain() - p.stdin.close() + p._stdin_write(uri) # noqa if progress: - def update_progress_bar(data: str): - if "%" in data: - # id, dledMiB/totalMiB(x%), CN:xx, DL:xxMiB, ETA:Xs - # eta may not always be available - data_parts = data[1:-1].split() - perc_parts = data_parts[1].split("(") - if len(perc_parts) == 2: - # might otherwise be e.g., 0B/0B, with no % symbol provided - progress( - total=100, - completed=int(perc_parts[1][:-2]), - downloaded=f"{data_parts[3].split(':')[1]}/s" - ) + is_dl_summary = False + for line in iter(p.stdout.readline, ""): + line = line.strip() + if line: + if line.startswith("[") and line.endswith("]"): + if "%" in line: + # id, dledMiB/totalMiB(x%), CN:xx, DL:xxMiB, ETA:Xs + # eta may not always be available + data_parts = line[1:-1].split() + perc_parts = data_parts[1].split("(") + if len(perc_parts) == 2: + # might otherwise be e.g., 0B/0B, with no % symbol provided + progress( + total=100, + completed=int(perc_parts[1][:-2]), + downloaded=f"{data_parts[3].split(':')[1]}/s" + ) + elif line.startswith("Download Results"): + # we know it's 100% downloaded, but let's use the avg dl speed value + is_dl_summary = True + elif is_dl_summary and "OK" in line and "|" in line: + gid, status, avg_speed, path_or_uri = line.split("|") + progress(total=100, completed=100, downloaded=avg_speed.strip()) + elif not is_dl_summary: + buffer_msg = line.split(" ", maxsplit=2) + buffer_msg = f"[Aria2c]: {buffer_msg[-1].strip()}" + console.log(Text.from_ansi(buffer_msg)) - # I'm sorry for this shameful code, aria2(c) is annoying as f!!! - while not p.stdout.at_eof(): - try: - buffer = await p.stdout.readuntil(b"\r") - except IncompleteReadError as e: - buffer = e.partial - - buffer = buffer.decode().strip() - if buffer: - buffer_lines = buffer.splitlines() - is_dl_summary = False - for line in buffer_lines: - if line: - if line.startswith("[") and line.endswith("]"): - update_progress_bar(line) - elif line.startswith("Download Results"): - # we know it's 100% downloaded, but let's use the avg dl speed value - is_dl_summary = True - elif is_dl_summary and "OK" in line and "|" in line: - gid, status, avg_speed, path_or_uri = line.split("|") - progress(total=100, completed=100, downloaded=avg_speed.strip()) - elif not is_dl_summary: - buffer_msg = line.split(" ", maxsplit=2) - buffer_msg = f"[Aria2c]: {buffer_msg[-1].strip()}" - console.log(Text.from_ansi(buffer_msg)) - - await p.wait() + p.wait() if p.returncode != 0: raise subprocess.CalledProcessError(p.returncode, arguments) diff --git a/devine/core/manifests/dash.py b/devine/core/manifests/dash.py index d0bb47e..852d5a6 100644 --- a/devine/core/manifests/dash.py +++ b/devine/core/manifests/dash.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import base64 import logging import math @@ -469,13 +468,13 @@ class DASH: segment_save_path.parent.mkdir(parents=True, exist_ok=True) segment_save_path.write_bytes(res.content) else: - asyncio.run(aria2c( - segment_uri, - segment_save_path, - session.headers, - proxy, + aria2c( + uri=segment_uri, + out=segment_save_path, + headers=session.headers, + proxy=proxy, silent=True - )) + ) data_size = segment_save_path.stat().st_size diff --git a/devine/core/manifests/hls.py b/devine/core/manifests/hls.py index 8431188..eedc613 100644 --- a/devine/core/manifests/hls.py +++ b/devine/core/manifests/hls.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import logging import re import sys @@ -289,13 +288,13 @@ class HLS: segment_save_path.parent.mkdir(parents=True, exist_ok=True) segment_save_path.write_bytes(res.content) else: - asyncio.run(aria2c( - segment.uri, - segment_save_path, - session.headers, - proxy, + aria2c( + uri=segment.uri, + out=segment_save_path, + headers=session.headers, + proxy=proxy, silent=True - )) + ) data_size = segment_save_path.stat().st_size