From 9bd13fe5bbe1df6bb01d4edb68f2c63a4812bf94 Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Tue, 30 Aug 2022 16:54:46 +0000 Subject: [PATCH] [cookies] Support firefox container in `--cookies-from-browser` (#4753) Authored by: bashonly --- README.md | 11 ++++++----- yt_dlp/YoutubeDL.py | 5 +++-- yt_dlp/__init__.py | 6 +++++- yt_dlp/cookies.py | 45 ++++++++++++++++++++++++++++++++++++--------- yt_dlp/options.py | 8 ++++---- 5 files changed, 54 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 8957711dd..c101048d5 100644 --- a/README.md +++ b/README.md @@ -706,13 +706,14 @@ You can also fork the project on github and run your fork's [build workflow](.gi and dump cookie jar in --no-cookies Do not read/dump cookies from/to file (default) - --cookies-from-browser BROWSER[+KEYRING][:PROFILE] + --cookies-from-browser BROWSER[+KEYRING][:PROFILE[:CONTAINER]] The name of the browser and (optionally) the name/path of the profile to load cookies - from, separated by a ":". Currently - supported browsers are: brave, chrome, - chromium, edge, firefox, opera, safari, - vivaldi. By default, the most recently + from (and container name if Firefox) + separated by a ":". Currently supported + browsers are: brave, chrome, chromium, edge, + firefox, opera, safari, vivaldi. By default, + the default container of the most recently accessed profile is used. The keyring used for decrypting Chromium cookies on Linux can be (optionally) specified after the browser diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 491e02dec..10c17ea00 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -304,8 +304,9 @@ class YoutubeDL: should act on each input URL as opposed to for the entire queue cookiefile: File name or text stream from where cookies should be read and dumped to cookiesfrombrowser: A tuple containing the name of the browser, the profile - name/path from where cookies are loaded, and the name of the - keyring, e.g. ('chrome', ) or ('vivaldi', 'default', 'BASICTEXT') + name/path from where cookies are loaded, the name of the keyring, + and the container name, e.g. ('chrome', ) or + ('vivaldi', 'default', 'BASICTEXT') or ('firefox', 'default', None, 'Meta') legacyserverconnect: Explicitly allow HTTPS connection to servers that do not support RFC 5746 secure renegotiation nocheckcertificate: Do not verify SSL certificates diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 3dc9b6e56..f4a2086ce 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -346,6 +346,7 @@ def validate_options(opts): # Cookies from browser if opts.cookiesfrombrowser: + container = None mobj = re.match(r'(?P[^+:]+)(\s*\+\s*(?P[^:]+))?(\s*:(?P.+))?', opts.cookiesfrombrowser) if mobj is None: raise ValueError(f'invalid cookies from browser arguments: {opts.cookiesfrombrowser}') @@ -354,12 +355,15 @@ def validate_options(opts): if browser_name not in SUPPORTED_BROWSERS: raise ValueError(f'unsupported browser specified for cookies: "{browser_name}". ' f'Supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}') + elif profile and browser_name == 'firefox': + if ':' in profile and not os.path.exists(profile): + profile, container = profile.split(':', 1) if keyring is not None: keyring = keyring.upper() if keyring not in SUPPORTED_KEYRINGS: raise ValueError(f'unsupported keyring specified for cookies: "{keyring}". ' f'Supported keyrings are: {", ".join(sorted(SUPPORTED_KEYRINGS))}') - opts.cookiesfrombrowser = (browser_name, profile, keyring) + opts.cookiesfrombrowser = (browser_name, profile, keyring, container) # MetadataParser def metadataparser_actions(f): diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index 1a164bb31..c5fb5ab68 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -3,6 +3,7 @@ import contextlib import http.cookiejar import json import os +import re import shutil import struct import subprocess @@ -24,7 +25,7 @@ from .dependencies import ( sqlite3, ) from .minicurses import MultilinePrinter, QuietMultilinePrinter -from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path +from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path, try_call CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'} SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'} @@ -85,8 +86,9 @@ def _create_progress_bar(logger): def load_cookies(cookie_file, browser_specification, ydl): cookie_jars = [] if browser_specification is not None: - browser_name, profile, keyring = _parse_browser_specification(*browser_specification) - cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring)) + browser_name, profile, keyring, container = _parse_browser_specification(*browser_specification) + cookie_jars.append( + extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl), keyring=keyring, container=container)) if cookie_file is not None: is_filename = YoutubeDLCookieJar.is_path(cookie_file) @@ -101,9 +103,9 @@ def load_cookies(cookie_file, browser_specification, ydl): return _merge_cookie_jars(cookie_jars) -def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None): +def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), *, keyring=None, container=None): if browser_name == 'firefox': - return _extract_firefox_cookies(profile, logger) + return _extract_firefox_cookies(profile, container, logger) elif browser_name == 'safari': return _extract_safari_cookies(profile, logger) elif browser_name in CHROMIUM_BASED_BROWSERS: @@ -112,7 +114,7 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger(), raise ValueError(f'unknown browser: {browser_name}') -def _extract_firefox_cookies(profile, logger): +def _extract_firefox_cookies(profile, container, logger): logger.info('Extracting cookies from firefox') if not sqlite3: logger.warning('Cannot extract cookies from firefox without sqlite3 support. ' @@ -126,6 +128,20 @@ def _extract_firefox_cookies(profile, logger): else: search_root = os.path.join(_firefox_browser_dir(), profile) + container_id = None + if container is not None: + containers_path = os.path.join(search_root, 'containers.json') + if not os.path.isfile(containers_path) or not os.access(containers_path, os.R_OK): + raise FileNotFoundError(f'could not read containers.json in {search_root}') + with open(containers_path, 'r') as containers: + identities = json.load(containers).get('identities', []) + container_id = next((context.get('userContextId') for context in identities if container in ( + context.get('name'), + try_call(lambda: re.fullmatch(r'userContext([^\.]+)\.label', context['l10nID']).group()) + )), None) + if not isinstance(container_id, int): + raise ValueError(f'could not find firefox container "{container}" in containers.json') + cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger) if cookie_database_path is None: raise FileNotFoundError(f'could not find firefox cookies database in {search_root}') @@ -135,7 +151,18 @@ def _extract_firefox_cookies(profile, logger): cursor = None try: cursor = _open_database_copy(cookie_database_path, tmpdir) - cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies') + origin_attributes = '' + if isinstance(container_id, int): + origin_attributes = f'^userContextId={container_id}' + logger.debug( + f'Only loading cookies from firefox container "{container}", ID {container_id}') + try: + cursor.execute( + 'SELECT host, name, value, path, expiry, isSecure FROM moz_cookies WHERE originAttributes=?', + (origin_attributes, )) + except sqlite3.OperationalError: + logger.debug('Database exception, loading all cookies') + cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies') jar = YoutubeDLCookieJar() with _create_progress_bar(logger) as progress_bar: table = cursor.fetchall() @@ -948,11 +975,11 @@ def _is_path(value): return os.path.sep in value -def _parse_browser_specification(browser_name, profile=None, keyring=None): +def _parse_browser_specification(browser_name, profile=None, keyring=None, container=None): if browser_name not in SUPPORTED_BROWSERS: raise ValueError(f'unsupported browser: "{browser_name}"') if keyring not in (None, *SUPPORTED_KEYRINGS): raise ValueError(f'unsupported keyring: "{keyring}"') if profile is not None and _is_path(profile): profile = os.path.expanduser(profile) - return browser_name, profile, keyring + return browser_name, profile, keyring, container diff --git a/yt_dlp/options.py b/yt_dlp/options.py index e66738448..e50ecc579 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1400,12 +1400,12 @@ def create_parser(): help='Do not read/dump cookies from/to file (default)') filesystem.add_option( '--cookies-from-browser', - dest='cookiesfrombrowser', metavar='BROWSER[+KEYRING][:PROFILE]', + dest='cookiesfrombrowser', metavar='BROWSER[+KEYRING][:PROFILE[:CONTAINER]]', help=( - 'The name of the browser and (optionally) the name/path of ' - 'the profile to load cookies from, separated by a ":". ' + 'The name of the browser and (optionally) the name/path of the profile to load cookies from ' + '(and container name if Firefox) separated by a ":". ' f'Currently supported browsers are: {", ".join(sorted(SUPPORTED_BROWSERS))}. ' - 'By default, the most recently accessed profile is used. ' + 'By default, the default container of the most recently accessed profile is used. ' 'The keyring used for decrypting Chromium cookies on Linux can be ' '(optionally) specified after the browser name separated by a "+". ' f'Currently supported keyrings are: {", ".join(map(str.lower, sorted(SUPPORTED_KEYRINGS)))}'))