diff --git a/devine/core/downloaders/curl_impersonate.py b/devine/core/downloaders/curl_impersonate.py index 3bb6405..9f724e3 100644 --- a/devine/core/downloaders/curl_impersonate.py +++ b/devine/core/downloaders/curl_impersonate.py @@ -6,6 +6,7 @@ from http.cookiejar import CookieJar from pathlib import Path from typing import Any, Generator, MutableMapping, Optional, Union +from curl_cffi import CurlOpt from curl_cffi.requests import Session from rich import filesize @@ -23,7 +24,7 @@ BROWSER = config.curl_impersonate.get("browser", "chrome120") def download( url: str, save_path: Path, - session: Optional[Session] = None, + session: Session, **kwargs: Any ) -> Generator[dict[str, Any], None, None]: """ @@ -52,8 +53,10 @@ def download( for one-time request changes like a header, cookie, or proxy. For example, to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`. """ - if not session: - session = Session(impersonate=BROWSER) + # https://github.com/yifeikong/curl_cffi/issues/6#issuecomment-2028518677 + # must be applied here since the `session.curl` is thread-localized + # noinspection PyProtectedMember + session.curl.setopt(CurlOpt.PROXY_CAINFO, session.curl._cacert) save_dir = save_path.parent control_file = save_path.with_name(f"{save_path.name}.!dev") @@ -224,10 +227,7 @@ def curl_impersonate( if cookies: session.cookies.update(cookies) if proxy: - session.proxies.update({ - "http": proxy.replace("https://", "http://"), - "https": proxy.replace("https://", "http://") - }) + session.proxies.update({"all": proxy}) yield dict(total=len(urls))