ninjajiraiya
/
devine
forked from DRMTalks/devine
1
0
Fork 0
Code Pull Requests Activity

Remove --skip-dl and the DL_POOL_SKIP Event

This commit is contained in:
rlaphoenix 2024-01-09 11:07:13 +00:00
parent 172ab64017
commit 65bc93b600
3 changed files with 231 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

403
devine/commands/dl.py
View File

@ -40,7 +40,7 @@ from rich.tree import Tree
from devine.core.config import config from devine.core.config import config
from devine.core.console import console from devine.core.console import console
from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings from devine.core.constants import DOWNLOAD_CANCELLED, AnyTrack, context_settings
from devine.core.credential import Credential from devine.core.credential import Credential
from devine.core.downloaders import downloader from devine.core.downloaders import downloader
from devine.core.drm import DRM_T, Widevine from devine.core.drm import DRM_T, Widevine
@ -119,8 +119,6 @@ class dl:
help="Skip downloading and list available tracks and what tracks would have been downloaded.") help="Skip downloading and list available tracks and what tracks would have been downloaded.")
@click.option("--list-titles", is_flag=True, default=False, @click.option("--list-titles", is_flag=True, default=False,
help="Skip downloading, only list available titles that would have been downloaded.") help="Skip downloading, only list available titles that would have been downloaded.")
@click.option("--skip-dl", is_flag=True, default=False,
help="Skip downloading while still retrieving the decryption keys.")
@click.option("--export", type=Path, @click.option("--export", type=Path,
help="Export Decryption Keys as you obtain them to a JSON file.") help="Export Decryption Keys as you obtain them to a JSON file.")
@click.option("--cdm-only/--vaults-only", is_flag=True, default=None, @click.option("--cdm-only/--vaults-only", is_flag=True, default=None,
@ -272,7 +270,6 @@ class dl:
chapters_only: bool, chapters_only: bool,
slow: bool, list_: bool, slow: bool, list_: bool,
list_titles: bool, list_titles: bool,
skip_dl: bool,
export: Optional[Path], export: Optional[Path],
cdm_only: Optional[bool], cdm_only: Optional[bool],
no_proxy: bool, no_proxy: bool,
@ -462,9 +459,6 @@ class dl:
dl_start_time = time.time() dl_start_time = time.time()
if skip_dl:
DOWNLOAD_LICENCE_ONLY.set()
try: try:
with Live( with Live(
Padding( Padding(
@ -529,138 +523,204 @@ class dl:
)) ))
return return
if skip_dl: dl_time = time_elapsed_since(dl_start_time)
console.log("Skipped downloads as --skip-dl was used...") console.print(Padding(
else: f"Track downloads finished in [progress.elapsed]{dl_time}[/]",
dl_time = time_elapsed_since(dl_start_time) (0, 5)
console.print(Padding( ))
f"Track downloads finished in [progress.elapsed]{dl_time}[/]",
(0, 5)
))
video_track_n = 0 video_track_n = 0
while ( while (
not title.tracks.subtitles and not title.tracks.subtitles and
len(title.tracks.videos) > video_track_n and len(title.tracks.videos) > video_track_n and
any( any(
x.get("codec_name", "").startswith("eia_") x.get("codec_name", "").startswith("eia_")
for x in ffprobe(title.tracks.videos[video_track_n].path).get("streams", []) for x in ffprobe(title.tracks.videos[video_track_n].path).get("streams", [])
) )
): ):
with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."): with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."):
try: try:
# TODO: Figure out the real language, it might be different # TODO: Figure out the real language, it might be different
# EIA-CC tracks sadly don't carry language information :( # EIA-CC tracks sadly don't carry language information :(
# TODO: Figure out if the CC language is original lang or not. # TODO: Figure out if the CC language is original lang or not.
# Will need to figure out above first to do so. # Will need to figure out above first to do so.
video_track = title.tracks.videos[video_track_n] video_track = title.tracks.videos[video_track_n]
track_id = f"ccextractor-{video_track.id}" track_id = f"ccextractor-{video_track.id}"
cc_lang = title.language or video_track.language cc_lang = title.language or video_track.language
cc = video_track.ccextractor( cc = video_track.ccextractor(
track_id=track_id, track_id=track_id,
out_path=config.directories.temp / config.filenames.subtitle.format( out_path=config.directories.temp / config.filenames.subtitle.format(
id=track_id, id=track_id,
language=cc_lang language=cc_lang
), ),
language=cc_lang, language=cc_lang,
original=False original=False
)
if cc:
# will not appear in track listings as it's added after all times it lists
title.tracks.add(cc)
self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
else:
self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
except EnvironmentError:
self.log.error(
"Cannot extract Closed Captions as the ccextractor executable was not found..."
)
break
video_track_n += 1
with console.status(f"Converting Subtitles to {sub_format.name}..."):
for subtitle in title.tracks.subtitles:
if subtitle.codec != sub_format:
subtitle.convert(sub_format)
with console.status("Repackaging tracks with FFMPEG..."):
has_repacked = False
for track in title.tracks:
if track.needs_repack:
track.repackage()
has_repacked = True
if callable(track.OnRepacked):
track.OnRepacked(track)
if has_repacked:
# we don't want to fill up the log with "Repacked x track"
self.log.info("Repacked one or more tracks with FFMPEG")
muxed_paths = []
if isinstance(title, (Movie, Episode)):
progress = Progress(
TextColumn("[progress.description]{task.description}"),
SpinnerColumn(finished_text=""),
BarColumn(),
"•",
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
console=console
)
multi_jobs = len(title.tracks.videos) > 1
tasks = [
progress.add_task(
f"Multiplexing{f' {x.height}p' if multi_jobs else ''}...",
total=None,
start=False
) )
for x in title.tracks.videos or [None] if cc:
] # will not appear in track listings as it's added after all times it lists
with Live( title.tracks.add(cc)
Padding(progress, (0, 5, 1, 5)), self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
console=console else:
): self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
for task, video_track in zip_longest(tasks, title.tracks.videos, fillvalue=None): except EnvironmentError:
if video_track: self.log.error(
title.tracks.videos = [video_track] "Cannot extract Closed Captions as the ccextractor executable was not found..."
progress.start_task(task) # TODO: Needed? )
muxed_path, return_code = title.tracks.mux( break
str(title), video_track_n += 1
progress=partial(progress.update, task_id=task),
delete=False
)
muxed_paths.append(muxed_path)
if return_code == 1:
self.log.warning("mkvmerge had at least one warning, will continue anyway...")
elif return_code >= 2:
self.log.error(f"Failed to Mux video to Matroska file ({return_code})")
sys.exit(1)
if video_track:
video_track.delete()
for track in title.tracks:
track.delete()
else:
# dont mux
muxed_paths.append(title.tracks.audio[0].path)
for muxed_path in muxed_paths: with console.status(f"Converting Subtitles to {sub_format.name}..."):
media_info = MediaInfo.parse(muxed_path) for subtitle in title.tracks.subtitles:
final_dir = config.directories.downloads if subtitle.codec != sub_format:
final_filename = title.get_filename(media_info, show_service=not no_source) subtitle.convert(sub_format)
if not no_folder and isinstance(title, (Episode, Song)): with console.status("Repackaging tracks with FFMPEG..."):
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) has_repacked = False
for track in title.tracks:
if track.needs_repack:
track.repackage()
has_repacked = True
if callable(track.OnRepacked):
track.OnRepacked(track)
if has_repacked:
# we don't want to fill up the log with "Repacked x track"
self.log.info("Repacked one or more tracks with FFMPEG")
final_dir.mkdir(parents=True, exist_ok=True) muxed_paths = []
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
shutil.move(muxed_path, final_path) if isinstance(title, (Movie, Episode)):
progress = Progress(
TextColumn("[progress.description]{task.description}"),
SpinnerColumn(finished_text=""),
BarColumn(),
"•",
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
console=console
)
multi_jobs = len(title.tracks.videos) > 1
tasks = [
progress.add_task(
f"Multiplexing{f' {x.height}p' if multi_jobs else ''}...",
total=None,
start=False
)
if cc:
# will not appear in track listings as it's added after all times it lists
title.tracks.add(cc)
self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}")
else:
self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}")
except EnvironmentError:
self.log.error(
"Cannot extract Closed Captions as the ccextractor executable was not found..."
)
break
video_track_n += 1
title_dl_time = time_elapsed_since(dl_start_time) with console.status(f"Converting Subtitles to {sub_format.name}..."):
console.print(Padding( for subtitle in title.tracks.subtitles:
f":tada: Title downloaded in [progress.elapsed]{title_dl_time}[/]!", if subtitle.codec != sub_format:
(0, 5, 1, 5) writer = {
)) Subtitle.Codec.SubRip: pycaption.SRTWriter,
Subtitle.Codec.SubStationAlpha: None,
Subtitle.Codec.SubStationAlphav4: None,
Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter,
Subtitle.Codec.WebVTT: pycaption.WebVTTWriter,
# MPEG-DASH box-encapsulated subtitle formats
Subtitle.Codec.fTTML: None,
Subtitle.Codec.fVTT: None,
}[sub_format]
if writer is None:
self.log.error(f"Cannot yet convert {subtitle.codec} to {sub_format.name}...")
sys.exit(1)
caption_set = subtitle.parse(subtitle.path.read_bytes(), subtitle.codec)
subtitle.merge_same_cues(caption_set)
subtitle_text = writer().write(caption_set)
subtitle.path.write_text(subtitle_text, encoding="utf8")
subtitle.codec = sub_format
subtitle.move(subtitle.path.with_suffix(f".{sub_format.value.lower()}"))
with console.status("Repackaging tracks with FFMPEG..."):
has_repacked = False
for track in title.tracks:
if track.needs_repack:
track.repackage()
has_repacked = True
if callable(track.OnRepacked):
track.OnRepacked(track)
if has_repacked:
# we don't want to fill up the log with "Repacked x track"
self.log.info("Repacked one or more tracks with FFMPEG")
muxed_paths = []
if isinstance(title, (Movie, Episode)):
progress = Progress(
TextColumn("[progress.description]{task.description}"),
SpinnerColumn(finished_text=""),
BarColumn(),
"•",
TimeRemainingColumn(compact=True, elapsed_when_finished=True),
console=console
)
multi_jobs = len(title.tracks.videos) > 1
tasks = [
progress.add_task(
f"Multiplexing{f' {x.height}p' if multi_jobs else ''}...",
total=None,
start=False
)
for x in title.tracks.videos or [None]
]
with Live(
Padding(progress, (0, 5, 1, 5)),
console=console
):
for task, video_track in zip_longest(tasks, title.tracks.videos, fillvalue=None):
if video_track:
title.tracks.videos = [video_track]
progress.start_task(task) # TODO: Needed?
muxed_path, return_code = title.tracks.mux(
str(title),
progress=partial(progress.update, task_id=task),
delete=False
)
muxed_paths.append(muxed_path)
if return_code == 1:
self.log.warning("mkvmerge had at least one warning, will continue anyway...")
elif return_code >= 2:
self.log.error(f"Failed to Mux video to Matroska file ({return_code})")
sys.exit(1)
if video_track:
video_track.delete()
for track in title.tracks:
track.delete()
else:
# dont mux
muxed_paths.append(title.tracks.audio[0].path)
for muxed_path in muxed_paths:
media_info = MediaInfo.parse(muxed_path)
final_dir = config.directories.downloads
final_filename = title.get_filename(media_info, show_service=not no_source)
if not no_folder and isinstance(title, (Episode, Song)):
final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True)
final_dir.mkdir(parents=True, exist_ok=True)
final_path = final_dir / f"{final_filename}{muxed_path.suffix}"
shutil.move(muxed_path, final_path)
title_dl_time = time_elapsed_since(dl_start_time)
console.print(Padding(
f":tada: Title downloaded in [progress.elapsed]{title_dl_time}[/]!",
(0, 5, 1, 5)
))
# update cookies # update cookies
cookie_file = config.directories.cookies / service.__class__.__name__ / f"{self.profile}.txt" cookie_file = config.directories.cookies / service.__class__.__name__ / f"{self.profile}.txt"
@ -803,9 +863,6 @@ class dl:
prepare_drm: Callable, prepare_drm: Callable,
progress: partial progress: partial
): ):
if DOWNLOAD_LICENCE_ONLY.is_set():
progress(downloaded="[yellow]SKIPPING")
if DOWNLOAD_CANCELLED.is_set(): if DOWNLOAD_CANCELLED.is_set():
progress(downloaded="[yellow]CANCELLED") progress(downloaded="[yellow]CANCELLED")
return return
@ -829,18 +886,17 @@ class dl:
if save_dir.exists() and save_dir.name.endswith("_segments"): if save_dir.exists() and save_dir.name.endswith("_segments"):
shutil.rmtree(save_dir) shutil.rmtree(save_dir)
if not DOWNLOAD_LICENCE_ONLY.is_set(): if config.directories.temp.is_file():
if config.directories.temp.is_file(): self.log.error(f"Temp Directory '{config.directories.temp}' must be a Directory, not a file")
self.log.error(f"Temp Directory '{config.directories.temp}' must be a Directory, not a file") sys.exit(1)
sys.exit(1)
config.directories.temp.mkdir(parents=True, exist_ok=True) config.directories.temp.mkdir(parents=True, exist_ok=True)
# Delete any pre-existing temp files matching this track. # Delete any pre-existing temp files matching this track.
# We can't re-use or continue downloading these tracks as they do not use a # We can't re-use or continue downloading these tracks as they do not use a
# lock file. Or at least the majority don't. Even if they did I've encountered # lock file. Or at least the majority don't. Even if they did I've encountered
# corruptions caused by sudden interruptions to the lock file. # corruptions caused by sudden interruptions to the lock file.
cleanup() cleanup()
try: try:
if track.descriptor == track.Descriptor.M3U: if track.descriptor == track.Descriptor.M3U:
@ -888,35 +944,32 @@ class dl:
else: else:
drm = None drm = None
if DOWNLOAD_LICENCE_ONLY.is_set(): downloader(
progress(downloaded="[yellow]SKIPPED") uri=track.url,
else: out=save_path,
downloader( headers=service.session.headers,
uri=track.url, cookies=service.session.cookies,
out=save_path, proxy=proxy,
headers=service.session.headers, progress=progress
cookies=service.session.cookies, )
proxy=proxy,
progress=progress
)
track.path = save_path track.path = save_path
if drm: if drm:
progress(downloaded="Decrypting", completed=0, total=100) progress(downloaded="Decrypting", completed=0, total=100)
drm.decrypt(save_path) drm.decrypt(save_path)
track.drm = None track.drm = None
if callable(track.OnDecrypted): if callable(track.OnDecrypted):
track.OnDecrypted(track) track.OnDecrypted(track)
progress(downloaded="Decrypted", completed=100) progress(downloaded="Decrypted", completed=100)
if isinstance(track, Subtitle): if isinstance(track, Subtitle):
track_data = track.path.read_bytes() track_data = track.path.read_bytes()
track_data = try_ensure_utf8(track_data) track_data = try_ensure_utf8(track_data)
track_data = html.unescape(track_data.decode("utf8")).encode("utf8") track_data = html.unescape(track_data.decode("utf8")).encode("utf8")
track.path.write_bytes(track_data) track.path.write_bytes(track_data)
progress(downloaded="Downloaded") progress(downloaded="Downloaded")
except KeyboardInterrupt: except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set() DOWNLOAD_CANCELLED.set()
progress(downloaded="[yellow]CANCELLED") progress(downloaded="[yellow]CANCELLED")
@ -926,20 +979,18 @@ class dl:
progress(downloaded="[red]FAILED") progress(downloaded="[red]FAILED")
raise raise
except (Exception, KeyboardInterrupt): except (Exception, KeyboardInterrupt):
if not DOWNLOAD_LICENCE_ONLY.is_set(): cleanup()
cleanup()
raise raise
if DOWNLOAD_CANCELLED.is_set(): if DOWNLOAD_CANCELLED.is_set():
# we stopped during the download, let's exit # we stopped during the download, let's exit
return return
if not DOWNLOAD_LICENCE_ONLY.is_set(): if track.path.stat().st_size <= 3: # Empty UTF-8 BOM == 3 bytes
if track.path.stat().st_size <= 3: # Empty UTF-8 BOM == 3 bytes raise IOError("Download failed, the downloaded file is empty.")
raise IOError("Download failed, the downloaded file is empty.")
if callable(track.OnDownloaded): if callable(track.OnDownloaded):
track.OnDownloaded(track) track.OnDownloaded(track)
@staticmethod @staticmethod
def get_profile(service: str) -> Optional[str]: def get_profile(service: str) -> Optional[str]:

6
devine/core/manifests/dash.py
View File

@ -26,7 +26,7 @@ from requests import Session
from requests.cookies import RequestsCookieJar from requests.cookies import RequestsCookieJar
from rich import filesize from rich import filesize
from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from devine.core.constants import DOWNLOAD_CANCELLED, AnyTrack
from devine.core.downloaders import downloader from devine.core.downloaders import downloader
from devine.core.downloaders import requests as requests_downloader from devine.core.downloaders import requests as requests_downloader
from devine.core.drm import Widevine from devine.core.drm import Widevine
@ -405,10 +405,6 @@ class DASH:
else: else:
drm = None drm = None
if DOWNLOAD_LICENCE_ONLY.is_set():
progress(downloaded="[yellow]SKIPPED")
return
progress(total=len(segments)) progress(total=len(segments))
download_sizes = [] download_sizes = []

14
devine/core/manifests/hls.py
View File

@ -24,7 +24,7 @@ from pywidevine.pssh import PSSH
from requests import Session from requests import Session
from rich import filesize from rich import filesize
from devine.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from devine.core.constants import DOWNLOAD_CANCELLED, AnyTrack
from devine.core.downloaders import downloader from devine.core.downloaders import downloader
from devine.core.downloaders import requests as requests_downloader from devine.core.downloaders import requests as requests_downloader
from devine.core.drm import DRM_T, ClearKey, Widevine from devine.core.drm import DRM_T, ClearKey, Widevine
@ -290,10 +290,6 @@ class HLS:
# it successfully downloaded, and it was not cancelled # it successfully downloaded, and it was not cancelled
progress(advance=1) progress(advance=1)
if download_size == -1: # skipped for --skip-dl
progress(downloaded="[yellow]SKIPPING")
continue
now = time.time() now = time.time()
time_since = now - last_speed_refresh time_since = now - last_speed_refresh
@ -307,9 +303,6 @@ class HLS:
last_speed_refresh = now last_speed_refresh = now
download_sizes.clear() download_sizes.clear()
if DOWNLOAD_LICENCE_ONLY.is_set():
return
with open(save_path, "wb") as f: with open(save_path, "wb") as f:
for segment_file in sorted(save_dir.iterdir()): for segment_file in sorted(save_dir.iterdir()):
segment_data = segment_file.read_bytes() segment_data = segment_file.read_bytes()
@ -362,6 +355,8 @@ class HLS:
if the Segment's DRM uses Widevine. if the Segment's DRM uses Widevine.
proxy: Proxy URI to use when downloading the Segment file. proxy: Proxy URI to use when downloading the Segment file.
session: Python-Requests Session used when requesting init data. session: Python-Requests Session used when requesting init data.
stop_event: Prematurely stop the Download from beginning. Useful if ran from
a Thread Pool. It will raise a KeyboardInterrupt if set.
Returns the file size of the downloaded Segment in bytes. Returns the file size of the downloaded Segment in bytes.
""" """
@ -422,9 +417,6 @@ class HLS:
finally: finally:
segment_key.put(newest_segment_key) segment_key.put(newest_segment_key)
if DOWNLOAD_LICENCE_ONLY.is_set():
return -1
headers_ = session.headers headers_ = session.headers
if segment.byterange: if segment.byterange:
# aria2(c) doesn't support byte ranges, use python-requests # aria2(c) doesn't support byte ranges, use python-requests