DRMTalks
/
devine
mirror of https://github.com/devine-dl/devine.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

fix(curl-impersonate): Set Cert-Authority Bundle for HTTPS Proxies 

For some reason curl-impersonate (curl_cffi project) does not set the certificate-authority bundle for proxies, which to be fair is for some reason seperated into two curl-options.

Doing this change as well as removing the https->http scheme enforcement on proxies, fixes HTTPS proxies on the curl-impersonate downloaders. I also simplified the seperate http and https proxy definitions to the `all` definition which was not originally supported but does seem to be supported as of v0.6.2.

I tested this on NordVPN proxies which are explicitly HTTPS-only and it does work.
This commit is contained in:
rlaphoenix 2024-04-01 14:59:53 +01:00
parent 45ccc129ce
commit f25d2419cf
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
devine/core/downloaders/curl_impersonate.py
View File

@ -6,6 +6,7 @@ from http.cookiejar import CookieJar
from pathlib import Path from pathlib import Path
from typing import Any, Generator, MutableMapping, Optional, Union from typing import Any, Generator, MutableMapping, Optional, Union
from curl_cffi import CurlOpt
from curl_cffi.requests import Session from curl_cffi.requests import Session
from rich import filesize from rich import filesize
@ -23,7 +24,7 @@ BROWSER = config.curl_impersonate.get("browser", "chrome120")
def download( def download(
url: str, url: str,
save_path: Path, save_path: Path,
session: Optional[Session] = None, session: Session,
**kwargs: Any **kwargs: Any
) -> Generator[dict[str, Any], None, None]: ) -> Generator[dict[str, Any], None, None]:
""" """
@ -52,8 +53,10 @@ def download(
for one-time request changes like a header, cookie, or proxy. For example, for one-time request changes like a header, cookie, or proxy. For example,
to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`. to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
""" """
if not session: # https://github.com/yifeikong/curl_cffi/issues/6#issuecomment-2028518677
session = Session(impersonate=BROWSER) # must be applied here since the `session.curl` is thread-localized
# noinspection PyProtectedMember
session.curl.setopt(CurlOpt.PROXY_CAINFO, session.curl._cacert)
save_dir = save_path.parent save_dir = save_path.parent
control_file = save_path.with_name(f"{save_path.name}.!dev") control_file = save_path.with_name(f"{save_path.name}.!dev")
@ -224,10 +227,7 @@ def curl_impersonate(
if cookies: if cookies:
session.cookies.update(cookies) session.cookies.update(cookies)
if proxy: if proxy:
session.proxies.update({ session.proxies.update({"all": proxy})
"http": proxy.replace("https://", "http://"),
"https": proxy.replace("https://", "http://")
})
yield dict(total=len(urls)) yield dict(total=len(urls))