diff --git a/README.md b/README.md index 671d4c7e..8605d350 100644 --- a/README.md +++ b/README.md @@ -426,7 +426,7 @@ For more discussion on third-party hosting options see here: Path | None: - """Get the default Chrome user data directory for the current platform.""" - system = platform.system() - home = Path.home() - - if system == "Darwin": # macOS - candidates = [ - home / "Library" / "Application Support" / "Google" / "Chrome", - home / "Library" / "Application Support" / "Chromium", - ] - elif system == "Linux": - candidates = [ - home / ".config" / "google-chrome", - home / ".config" / "chromium", - home / ".config" / "chrome", - home / "snap" / "chromium" / "common" / "chromium", - ] - elif system == "Windows": - local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local")) - candidates = [ - local_app_data / "Google" / "Chrome" / "User Data", - local_app_data / "Chromium" / "User Data", - ] - else: - candidates = [] - - for candidate in candidates: - if candidate.exists() and (candidate / "Default").exists(): - return candidate - - return None - - -def get_brave_user_data_dir() -> Path | None: - """Get the default Brave user data directory for the current platform.""" - system = platform.system() - home = Path.home() - - if system == "Darwin": - candidates = [ - home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser", - ] - elif system == "Linux": - candidates = [ - home / ".config" / "BraveSoftware" / "Brave-Browser", - ] - elif system == "Windows": - local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local")) - candidates = [ - local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data", - ] - else: - candidates = [] - - for candidate in candidates: - if candidate.exists() and (candidate / "Default").exists(): - return candidate - - return None - - -def get_edge_user_data_dir() -> Path | None: - """Get the default Edge user data directory for the current platform.""" - system = platform.system() - home = Path.home() - - if system == "Darwin": - candidates = [ - home / "Library" / "Application Support" / "Microsoft Edge", - ] - elif system == "Linux": - candidates = [ - home / ".config" / "microsoft-edge", - home / ".config" / "microsoft-edge-beta", - home / ".config" / "microsoft-edge-dev", - ] - elif system == "Windows": - local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local")) - candidates = [ - local_app_data / "Microsoft" / "Edge" / "User Data", - ] - else: - candidates = [] - - for candidate in candidates: - if candidate.exists() and (candidate / "Default").exists(): - return candidate - - return None - - -BROWSER_PROFILE_FINDERS = { - "chrome": get_chrome_user_data_dir, - "chromium": get_chrome_user_data_dir, # Same locations - "brave": get_brave_user_data_dir, - "edge": get_edge_user_data_dir, -} - -CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"} - - -# ============================================================================= -# Cookie Extraction via CDP -# ============================================================================= - # ============================================================================= # Validation Helpers # ============================================================================= @@ -218,6 +107,8 @@ def create_personas( names: Iterable[str], import_from: str | None = None, profile: str | None = None, + source: str | None = None, + browser_binary: str | None = None, ) -> int: """ Create Personas from names. @@ -239,25 +130,25 @@ def create_personas( rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr) return 1 - # Validate import source if specified - source_profile_dir = None + import_source = None + if source and not import_from: + rprint("[red]--source requires --import (the source browser name).[/red]", file=sys.stderr) + return 1 if import_from: - import_from = import_from.lower() - if import_from not in BROWSER_PROFILE_FINDERS: - rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr) - rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr) + try: + if source: + import_source = persona_importers.resolve_custom_import_source(source, profile_dir=profile) + import_source = replace(import_source, browser=import_from.lower(), browser_binary=browser_binary) + elif import_from.startswith(("http://", "https://", "ws://", "wss://")): + import_source = persona_importers.resolve_custom_import_source(import_from) + else: + import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile) + if browser_binary: + import_source = replace(import_source, browser_binary=browser_binary) + except ValueError as err: + rprint(f"[red]{err}[/red]", file=sys.stderr) return 1 - source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]() - if not source_profile_dir: - rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr) - return 1 - - rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr) - - if profile is None and (source_profile_dir / "Default").exists(): - profile = "Default" - created_count = 0 for name in name_list: name = name.strip() @@ -275,16 +166,16 @@ def create_personas( if created: persona.ensure_dirs() created_count += 1 - rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr) + else: rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr) cookies_file = Path(persona.path) / "cookies.txt" # Import browser profile if requested - if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None: + if import_source is not None: try: - import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile) + rprint(f"[dim]Importing {import_source.display_label}; copying settings and exporting cookies...[/dim]", file=sys.stderr) import_result = persona_importers.import_persona_from_source( persona, import_source, @@ -292,20 +183,25 @@ def create_personas( import_cookies=True, capture_storage=False, ) - except Exception as e: + except (Exception, KeyboardInterrupt) as e: + if created: + shutil.rmtree(persona.path, ignore_errors=True) + persona.delete() rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr) return 1 if import_result.profile_copied: rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr) if import_result.cookies_imported: - rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr) + rprint(f"[green]Extracted {import_result.cookie_count} cookies to {cookies_file}[/green]", file=sys.stderr) elif not import_result.profile_copied: rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr) for warning in import_result.warnings: rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr) + if created: + rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr) if not is_tty: write_record( { @@ -544,11 +440,17 @@ def main(): @main.command("create") @click.argument("names", nargs=-1) -@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)") +@click.option("--import", "import_from", help="Import from chrome, chromium, brave, edge, or a live CDP URL") +@click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)") +@click.option( + "--browser-binary", + type=click.Path(exists=True, dir_okay=False), + help="Source Chromium browser executable; required for other Chromium-based browsers", +) @click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)") -def create_cmd(names: tuple, import_from: str | None, profile: str | None): +def create_cmd(names: tuple, import_from: str | None, profile: str | None, source: str | None, browser_binary: str | None): """Create Personas, optionally importing from a browser profile.""" - sys.exit(create_personas(names, import_from=import_from, profile=profile)) + sys.exit(create_personas(names, import_from=import_from, profile=profile, source=source, browser_binary=browser_binary)) @main.command("list") diff --git a/archivebox/config/common.py b/archivebox/config/common.py index d3f41d0b..47e06c9b 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -461,7 +461,8 @@ class ArchivingConfig(BaseConfigSet): USER_AGENT: str = Field( default=f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)", ) - COOKIES_FILE: Path | None = Field(default=None) + COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION}) + AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION}) URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST") URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST") diff --git a/archivebox/personas/export_browser_state.js b/archivebox/personas/export_browser_state.js index 0ad7b1f6..e6fe7606 100644 --- a/archivebox/personas/export_browser_state.js +++ b/archivebox/personas/export_browser_state.js @@ -13,7 +13,6 @@ */ const fs = require('fs'); -const os = require('os'); const path = require('path'); const pluginsDir = process.env.ARCHIVEBOX_ABX_PLUGINS_DIR || process.env.ABX_PLUGINS_DIR; @@ -26,7 +25,7 @@ const baseUtils = require(path.join(pluginsDir, 'base', 'utils.js')); baseUtils.ensureNodeModuleResolution(module); const chromeUtils = require(path.join(pluginsDir, 'chrome', 'chrome_utils.js')); -const puppeteer = require('puppeteer-core'); +const puppeteer = require('puppeteer'); function cookieToNetscape(cookie) { let domain = cookie.domain; @@ -115,38 +114,25 @@ async function openBrowser() { throw new Error(`User data directory does not exist: ${userDataDir}`); } - const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'abx-browser-state-')); const binary = process.env.CHROME_BINARY; if (!binary || !path.isAbsolute(binary) || !fs.existsSync(binary)) { throw new Error('CHROME_BINARY must be an absolute executable path resolved by abxpkg'); } - const launched = await chromeUtils.launchChromium({ - binary, - outputDir, + // The ordinary archiving launcher uses a mock keychain. An import must + // instead let the original signed browser decrypt its own profile cookies. + const browser = await puppeteer.launch({ + executablePath: binary, userDataDir, headless: true, - killZombies: false, + ignoreDefaultArgs: ['--use-mock-keychain', '--password-store=basic'], + args: ['--no-first-run', '--disable-sync', '--disable-background-networking', + '--profile-directory=Default', ...(process.platform === 'linux' && process.getuid?.() === 0 ? ['--no-sandbox'] : [])], }); - - if (!launched.success) { - throw new Error(launched.error || 'Chrome launch failed'); - } - - const browser = await chromeUtils.connectToBrowserEndpoint(puppeteer, launched.cdpUrl, { defaultViewport: null }); - return { browser, async cleanup() { - try { - await browser.disconnect(); - } catch (error) {} - try { - await chromeUtils.killChrome(launched.pid, outputDir); - } catch (error) {} - try { - fs.rmSync(outputDir, { recursive: true, force: true }); - } catch (error) {} + await browser.close(); }, sourceDescription: userDataDir, }; diff --git a/archivebox/personas/importers.py b/archivebox/personas/importers.py index 2b172493..dd96a109 100644 --- a/archivebox/personas/importers.py +++ b/archivebox/personas/importers.py @@ -11,6 +11,9 @@ import json import os import platform import shutil +import sqlite3 +import time +from http.cookiejar import MozillaCookieJar import subprocess import sys import tempfile @@ -42,25 +45,27 @@ BROWSER_PROFILE_DIR_NAMES = ( ) VOLATILE_PROFILE_COPY_PATTERNS = ( + # Chromium/macOS atomic-write staging files (e.g. + # .com.brave.Browser.TransportSecurity.Ze7UBU) disappear after rename. + ".*.??????", "Cache", "Code Cache", "GPUCache", "ShaderCache", - "Service Worker", - "GCM Store", - "chrome-extension_*", - "DNR Extension Rules", - "Extension Rules", - "Extension Scripts", - "Extension State", - "Local Extension Settings", - "*.log", "Crashpad", "BrowserMetrics", "BrowserMetrics-spare.pma", + "RunningChromeVersion", + "DevToolsActivePort", "SingletonLock", "SingletonSocket", "SingletonCookie", + "Sessions", + "Sessions_Encrypted", + "Current Session", + "Current Tabs", + "Last Session", + "Last Tabs", ) PERSONA_PROFILE_DIR_CANDIDATES = ( @@ -201,6 +206,7 @@ class PersonaImportResult: source: PersonaImportSource profile_copied: bool = False cookies_imported: bool = False + cookie_count: int = 0 storage_captured: bool = False user_agent_imported: bool = False warnings: list[str] = field(default_factory=list) @@ -419,7 +425,10 @@ def resolve_browser_import_source(browser: str, profile_dir: str | None = None) user_data_dir = BROWSER_PROFILE_FINDERS[browser]() if not user_data_dir: - raise ValueError(f"Could not find {browser} profile directory") + raise ValueError( + f"Could not find {browser} profile directory. Use --source /path/to/browser-data. " + "For Docker on macOS/Windows, import on the host into the shared data directory first.", + ) chosen_profile = profile_dir or pick_default_profile_dir(user_data_dir) if not chosen_profile: @@ -444,6 +453,8 @@ def resolve_browser_profile_source( resolved_root = resolved_root.resolve() if not resolved_root.exists(): raise ValueError(f"Profile root does not exist: {resolved_root}") + if Path(profile_dir).name != profile_dir or profile_dir in {".", ".."}: + raise ValueError("Profile must be a directory name within the source browser root.") if not profile_dir.strip(): raise ValueError("Profile directory name cannot be empty.") @@ -508,6 +519,46 @@ def pick_default_profile_dir(user_data_dir: Path) -> str | None: return profiles[0] +def resolve_source_browser_binary(source: PersonaImportSource) -> str: + """Use the originating browser so OS-protected cookies use the correct keychain.""" + if source.browser_binary: + return str(Path(source.browser_binary).expanduser().resolve()) + names = { + "chrome": ("Google Chrome", "google-chrome"), + "chromium": ("Chromium", "chromium"), + "brave": ("Brave Browser", "brave-browser"), + "edge": ("Microsoft Edge", "microsoft-edge"), + } + app, executable = names.get(source.browser, ("", "")) + candidates = [] + if platform.system() == "Darwin" and app: + candidates = [ + Path("/Applications") / f"{app}.app/Contents/MacOS/{app}", + Path.home() / f"Applications/{app}.app/Contents/MacOS/{app}", + ] + elif platform.system() == "Windows": + relative = { + "chrome": "Google/Chrome/Application/chrome.exe", + "edge": "Microsoft/Edge/Application/msedge.exe", + "brave": "BraveSoftware/Brave-Browser/Application/brave.exe", + "chromium": "Chromium/Application/chrome.exe", + }.get(source.browser) + if relative: + candidates = [ + Path(os.environ[root]) / relative for root in ("LOCALAPPDATA", "PROGRAMFILES", "PROGRAMFILES(X86)") if os.environ.get(root) + ] + for candidate in candidates: + if candidate.is_file() and os.access(candidate, os.X_OK): + return str(candidate) + if executable and (found := shutil.which(executable)): + return found + raise ValueError( + f"Source {source.browser} browser executable not found. Use --browser-binary with the originating Chromium browser. " + "For Docker, run persona create --import on the browser's host into the shared data directory first: " + "the container cannot decrypt cookies using the host's macOS Keychain or Windows credentials.", + ) + + def import_persona_from_source( persona: Persona, source: PersonaImportSource, @@ -516,67 +567,171 @@ def import_persona_from_source( import_cookies: bool = True, capture_storage: bool = False, ) -> PersonaImportResult: + # Stage all work before replacing an existing, working identity. persona.ensure_dirs() result = PersonaImportResult(source=source) + expected_cookies = 0 + browser_binary = None + native_cookie_browsers = {"chrome", "chromium", "brave", "edge", "vivaldi", "opera", "opera_gx"} + # browser-cookie3 needs a desktop D-Bus session on Linux. In a headless + # environment, let the originating browser decode its own profile instead. + has_native_cookie_decoder = platform.system() != "Linux" or bool(os.environ.get("DBUS_SESSION_BUS_ADDRESS")) + native_cookie_import = source.kind == "browser-profile" and source.browser in native_cookie_browsers and has_native_cookie_decoder + if source.kind == "browser-profile" and (import_cookies or capture_storage): + if not native_cookie_import: + browser_binary = resolve_source_browser_binary(source) + for cookie_db in (source.profile_path / "Network" / "Cookies", source.profile_path / "Cookies"): + if cookie_db.is_file(): + with sqlite3.connect(f"{cookie_db.as_uri()}?mode=ro", uri=True) as conn: + expected_cookies = conn.execute( + "SELECT COUNT(*) FROM cookies WHERE expires_utc = 0 OR expires_utc > ?", + (int((time.time() + 11644473600) * 1_000_000),), + ).fetchone()[0] + break + with tempfile.TemporaryDirectory(prefix=".import-", dir=persona.path.parent) as tmp: + stage = Path(tmp) + staged_profile = stage / "chrome_profile" + if source.kind == "browser-profile": + assert source.user_data_dir and source.profile_path + if copy_profile or import_cookies or capture_storage: + staged_profile.mkdir() + # Only the selected profile belongs to this persona. Normalize its + # name so downstream Chromium launches always select it. + copy_browser_user_data_dir(source.profile_path, staged_profile / "Default") + local_state = source.user_data_dir / "Local State" + if local_state.exists(): + state = json.loads(local_state.read_text()) + profile_state = state.setdefault("profile", {}) + profile_state["last_used"] = "Default" + profile_state["last_active_profiles"] = ["Default"] + info = profile_state.get("info_cache", {}).get(source.profile_dir) + profile_state["info_cache"] = {"Default": info} if info else {} + (staged_profile / "Local State").write_text(json.dumps(state)) + persona.cleanup_chrome_profile(staged_profile) + result.profile_copied = copy_profile + elif copy_profile: + result.warnings.append( + "CDP imports capture cookies and open-tab storage, not browser settings. Use --source for a full profile import.", + ) - persona_chrome_dir = Path(persona.CHROME_USER_DATA_DIR) - cookies_file = persona.path / "cookies.txt" - auth_file = persona.path / "auth.json" - - launch_user_data_dir: Path | None = None - - if source.kind == "browser-profile": - if copy_profile and source.user_data_dir: - resolved_source_root = source.user_data_dir.resolve() - resolved_persona_root = persona_chrome_dir.resolve() - if resolved_source_root == resolved_persona_root: - result.warnings.append( - "Skipped profile copy because the selected source is already this persona's chrome_profile directory.", - ) + if import_cookies or capture_storage: + if native_cookie_import: + auth_payload = export_profile_cookies(source, staged_profile, stage) + success, message = True, "" else: - copy_browser_user_data_dir(resolved_source_root, resolved_persona_root) - persona.cleanup_chrome_profile(resolved_persona_root) - result.profile_copied = True - launch_user_data_dir = resolved_persona_root - else: - launch_user_data_dir = source.user_data_dir - elif copy_profile: - result.warnings.append( - "Profile copying is only available for local Chromium profile paths. CDP imports can only pull cookies and open-tab storage.", - ) - - if not import_cookies and not capture_storage: - return result - - if source.kind == "cdp": - export_success, auth_payload, export_message = export_browser_state( - cdp_url=source.cdp_url, - cookies_output_file=cookies_file if import_cookies else None, - auth_output_file=auth_file if capture_storage else None, - ) - else: - export_success, auth_payload, export_message = export_browser_state( - user_data_dir=launch_user_data_dir, - profile_dir=source.profile_dir, - chrome_binary=source.browser_binary, - cookies_output_file=cookies_file if import_cookies else None, - auth_output_file=auth_file if capture_storage else None, - ) - - if not export_success: - result.warnings.append(export_message or "Browser import failed.") - return result - - if import_cookies and cookies_file.exists(): - result.cookies_imported = True - if capture_storage and auth_file.exists(): - result.storage_captured = True - if _apply_imported_user_agent(persona, auth_payload): - result.user_agent_imported = True + # Custom browsers and headless Linux use the originating browser. + # Desktop imports use their OS decoder without launching or probing. + launch_profile = stage / "export_profile" + if source.kind == "browser-profile": + copy_browser_user_data_dir(staged_profile, launch_profile) + success, auth_payload, message = export_browser_state( + user_data_dir=launch_profile if source.kind == "browser-profile" else None, + cdp_url=source.cdp_url, + profile_dir="Default" if source.kind == "browser-profile" else None, + chrome_binary=browser_binary, + cookies_output_file=stage / "cookies.txt" if import_cookies else None, + auth_output_file=stage / "auth.json", + ) + if not success: + raise ValueError(message or "Browser state export failed; persona was not replaced.") + if expected_cookies and not (auth_payload or {}).get("cookies"): + raise ValueError( + "The source has cookies but the browser exported none. Run the import on the source host " + "with the original browser and unlock its OS keychain; the existing persona was not replaced.", + ) + if import_cookies: + result.cookies_imported = True + result.cookie_count = len((auth_payload or {}).get("cookies", [])) + if capture_storage: + result.storage_captured = True + # Keep full CDP cookie attributes (sameSite, httpOnly, partition keys) + # as well as cookies.txt for non-browser extractors. + result.user_agent_imported = _apply_imported_user_agent(persona, auth_payload) + if result.profile_copied: + target = Path(persona.CHROME_USER_DATA_DIR) + if target.exists(): + shutil.rmtree(target) + staged_profile.rename(target) + for filename in ("cookies.txt", "auth.json"): + exported = stage / filename + if exported.exists(): + exported.chmod(0o600) + exported.replace(persona.path / filename) return result +def export_profile_cookies(source: PersonaImportSource, staged_profile: Path, output: Path) -> dict: + """Decode with the host keychain, never by starting the user's browser.""" + import browser_cookie3 + + payload = {"TYPE": "auth", "cookies": [], "localStorage": {}, "sessionStorage": {}} + jar = MozillaCookieJar(str(output / "cookies.txt")) + assert source.profile_path + for relative in (Path("Network/Cookies"), Path("Cookies")): + original = source.profile_path / relative + if not original.is_file(): + continue + database = staged_profile / "Default" / relative + for suffix in ("", "-wal", "-shm"): + database.with_name(database.name + suffix).unlink(missing_ok=True) + # SQLite backup includes committed WAL data even while the browser is open. + with sqlite3.connect(f"{original.as_uri()}?mode=ro", uri=True) as reader: + with sqlite3.connect(database) as writer: + reader.backup(writer) + try: + cookies = getattr(browser_cookie3, source.browser)( + cookie_file=str(database), + key_file=str(staged_profile / "Local State"), + ) + except Exception as err: + raise ValueError( + "Could not decrypt browser cookies. Run the import as your desktop user on the browser host " + "with its OS keychain unlocked. Mounting macOS/Windows browser files into Linux does not provide " + "the decryption keys. The existing persona was not replaced.", + ) from err + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + columns = {row[1] for row in connection.execute("PRAGMA table_info(cookies)")} + metadata_columns = [ + name + for name in ("host_key", "path", "name", "samesite", "top_frame_site_key", "has_cross_site_ancestor") + if name in columns + ] + rows = { + (row["host_key"], row["path"], row["name"]): dict(row) + for row in connection.execute("SELECT " + ", ".join(metadata_columns) + " FROM cookies") + } + for cookie in cookies: + if cookie.is_expired(): + continue + jar.set_cookie(cookie) + item = { + "name": cookie.name, + "value": cookie.value, + "domain": cookie.domain, + "path": cookie.path, + "secure": bool(cookie.secure), + "httpOnly": cookie.has_nonstandard_attr("HTTPOnly"), + } + if cookie.expires: + item["expires"] = cookie.expires + row = rows.get((cookie.domain, cookie.path, cookie.name), {}) + same_site = {0: "None", 1: "Lax", 2: "Strict"}.get(row.get("samesite")) + if same_site: + item["sameSite"] = same_site + if row.get("top_frame_site_key"): + item["partitionKey"] = { + "topLevelSite": row["top_frame_site_key"], + "hasCrossSiteAncestor": bool(row.get("has_cross_site_ancestor", False)), + } + payload["cookies"].append(item) + break + jar.save(ignore_discard=True, ignore_expires=False) + (output / "auth.json").write_text(json.dumps(payload) + "\n") + return payload + + def copy_browser_user_data_dir(source_dir: Path, destination_dir: Path) -> None: destination_dir.parent.mkdir(parents=True, exist_ok=True) shutil.rmtree(destination_dir, ignore_errors=True) @@ -611,21 +766,27 @@ def export_browser_state( chrome_config = chrome_plugin_dir / "chrome" / "config.json" env = os.environ.copy() - if chrome_binary: - env["CHROME_BINARY"] = str(chrome_binary) - dependency_env = subprocess.run( - [ - str(Path(sys.executable).with_name("abxpkg")), - "env", - "--install", - "--json", - f"--lib={get_config().ABXPKG_LIB_DIR}", - f"--deps-from={chrome_config}:required_binaries", - ], - capture_output=True, - text=True, - env=env, - ) + dependency_config = json.loads(chrome_config.read_text()) + dependency_config["required_binaries"] = [dep for dep in dependency_config["required_binaries"] if dep["name"] != "{CHROME_BINARY}"] + dependency_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + with dependency_file: + json.dump(dependency_config, dependency_file) + try: + dependency_env = subprocess.run( + [ + str(Path(sys.executable).with_name("abxpkg")), + "env", + "--install", + "--json", + f"--lib={get_config().ABXPKG_LIB_DIR}", + f"--deps-from={dependency_file.name}:required_binaries", + ], + capture_output=True, + text=True, + env=env, + ) + finally: + Path(dependency_file.name).unlink() if dependency_env.returncode != 0: return False, None, dependency_env.stderr.strip() or "abxpkg could not resolve browser export dependencies." try: @@ -639,6 +800,9 @@ def export_browser_state( if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK): return False, None, f"abxpkg did not resolve Node.js into {node_projection}." env.update({str(key): str(value) for key, value in resolved_env.items()}) + if chrome_binary: + env["CHROME_BINARY"] = chrome_binary + env["NODE_MODULES_DIR"] = str(abxpkg_lib_dir / "pnpm" / "packages" / "chrome" / "node_modules") env["NODE_BINARY"] = str(node_projection) env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(chrome_plugin_dir) @@ -865,7 +1029,7 @@ def _apply_imported_user_agent(persona: Persona, auth_payload: dict | None) -> b if not auth_payload: return False - user_agent = str(auth_payload.get("user_agent") or "").strip() + user_agent = str(auth_payload.get("user_agent") or "").strip().replace("HeadlessChrome/", "Chrome/") if not user_agent: return False diff --git a/archivebox/personas/models.py b/archivebox/personas/models.py index 8e00842f..25c4aea1 100644 --- a/archivebox/personas/models.py +++ b/archivebox/personas/models.py @@ -45,17 +45,23 @@ VOLATILE_PROFILE_DIR_NAMES = { "Code Cache", "GPUCache", "ShaderCache", - "Service Worker", - "GCM Store", "Crashpad", "BrowserMetrics", + "Sessions", + "Sessions_Encrypted", } VOLATILE_PROFILE_FILE_NAMES = { "BrowserMetrics-spare.pma", + "RunningChromeVersion", + "DevToolsActivePort", "SingletonCookie", "SingletonLock", "SingletonSocket", + "Current Session", + "Current Tabs", + "Last Session", + "Last Tabs", } @@ -211,7 +217,7 @@ class Persona(ModelWithConfig): cleaned = True for path in profile_paths(): - if not path.match("*.log"): + if path.name not in {"chrome.log", "chrome_debug.log"}: continue try: path.unlink() @@ -297,6 +303,10 @@ class Persona(ModelWithConfig): else: runtime_profile_dir.mkdir(parents=True, exist_ok=True) + for filename in ("cookies.txt", "auth.json"): + source = self.path / filename + if source.is_file(): + shutil.copy2(source, runtime_root / filename) runtime_downloads_dir.mkdir(parents=True, exist_ok=True) self.cleanup_chrome_profile(runtime_profile_dir) @@ -313,6 +323,11 @@ class Persona(ModelWithConfig): # derivation centralized in the Chrome plugin helpers. "PERSONAS_DIR": str(runtime_root.parent), "ACTIVE_PERSONA": self.name, + **{ + key: str(runtime_root / filename) + for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json")) + if (runtime_root / filename).is_file() + }, } def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]: @@ -329,6 +344,12 @@ class Persona(ModelWithConfig): else: runtime_profile_dir.mkdir(parents=True, exist_ok=True) + for filename in ("cookies.txt", "auth.json"): + source = self.runtime_root_for_crawl(snapshot.crawl) / filename + if not source.is_file(): + source = self.path / filename + if source.is_file(): + shutil.copy2(source, runtime_root / filename) runtime_downloads_dir.mkdir(parents=True, exist_ok=True) self.cleanup_chrome_profile(runtime_profile_dir) @@ -343,6 +364,11 @@ class Persona(ModelWithConfig): # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way. "PERSONAS_DIR": str(runtime_root.parent), "ACTIVE_PERSONA": self.name, + **{ + key: str(runtime_root / filename) + for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json")) + if (runtime_root / filename).is_file() + }, } def cleanup_runtime_for_crawl(self, crawl) -> None: diff --git a/archivebox/tests/test_cli_persona.py b/archivebox/tests/test_cli_persona.py index 6379facc..6d5a7359 100644 --- a/archivebox/tests/test_cli_persona.py +++ b/archivebox/tests/test_cli_persona.py @@ -14,3 +14,129 @@ def test_persona_help_runs_successfully(tmp_path): assert result.returncode == 0 assert "persona" in result.stdout.lower() assert "list" in result.stdout + + +def test_persona_import_missing_source_is_actionable_and_does_not_create(initialized_archive, tmp_path): + result = run_archivebox_cmd( + ["persona", "create", "--import=brave", "--source", str(tmp_path / "missing"), "broken"], + cwd=initialized_archive, + ) + assert result.returncode == 1 + assert "does not exist" in result.stderr + listed = run_archivebox_cmd(["persona", "list", "--name=broken"], cwd=initialized_archive) + assert listed.returncode == 0 + assert not listed.stdout.strip() + assert not (initialized_archive / "personas" / "broken").exists() + + +def test_persona_import_real_browser_session(initialized_archive, tmp_path, httpserver): + """Import an actual Chromium-created session and archive its private page.""" + import hashlib + import json + import subprocess + + from werkzeug.wrappers import Response + from archivebox.tests.conftest import resolve_abxpkg_chrome_env + + browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib") + source = tmp_path / "browser" + httpserver.expect_request("/login").respond_with_data( + 'Signed inSigned in', + content_type="text/html", + headers={"Set-Cookie": "persona_session=real-session; Path=/; Max-Age=3600; HttpOnly; SameSite=Lax"}, + ) + + def private_page(request): + authenticated = request.cookies.get("persona_session") == "real-session" + return Response( + "Private notebook

Authenticated persona notebook

" + '' + if authenticated + else "Please log in", + content_type="text/html", + ) + + httpserver.expect_request("/private").respond_with_handler(private_page) + seeded = subprocess.run( + [ + browser_env["NODE_BINARY"], + "-e", + """ +const puppeteer = require(process.argv[1]); +(async () => { + const browser = await puppeteer.launch({ + executablePath: process.argv[2], userDataDir: process.argv[3], headless: true, + ignoreDefaultArgs: ['--use-mock-keychain', '--password-store=basic'], + args: ['--no-sandbox', '--profile-directory=Profile 2'], + }); + try { + const page = await browser.newPage(); + await page.goto(process.argv[4]); + console.log(await page.title()); + } finally { await browser.close(); } +})().catch(error => { console.error(error); process.exit(1); }); +""", + str(tmp_path / "lib/pnpm/packages/chrome/node_modules/puppeteer"), + browser_env["CHROME_BINARY"], + str(source), + httpserver.url_for("/login"), + ], + capture_output=True, + text=True, + timeout=60, + ) + assert seeded.returncode == 0, seeded.stderr + assert "Signed in" in seeded.stdout + preferences = source / "Profile 2" / "Preferences" + original_preferences = preferences.read_bytes() + + imported = run_archivebox_cmd( + [ + "persona", + "create", + "--import=chromium", + "--source", + str(source), + "--browser-binary", + browser_env["CHROME_BINARY"], + "--profile=Profile 2", + "session", + ], + cwd=initialized_archive, + timeout=120, + ) + assert imported.returncode == 0, imported.stderr + persona = initialized_archive / "personas" / "session" + assert (persona / "chrome_profile" / "Default" / "Preferences").read_bytes() == original_preferences + assert preferences.read_bytes() == original_preferences + auth = json.loads((persona / "auth.json").read_text()) + cookie = next(cookie for cookie in auth["cookies"] if cookie["name"] == "persona_session") + assert cookie["value"] == "real-session" + assert cookie["httpOnly"] is True + assert cookie["sameSite"] == "Lax" + before = { + path.relative_to(persona): hashlib.sha256(path.read_bytes()).hexdigest() + for path in (persona / "cookies.txt", persona / "auth.json", persona / "chrome_profile/Default/Preferences") + } + failed = run_archivebox_cmd( + ["persona", "create", "--import=unsupported-browser", "--source", str(source), "--profile=Profile 2", "session"], + cwd=initialized_archive, + timeout=60, + ) + assert failed.returncode == 1 + assert "browser executable not found" in " ".join(failed.stderr.split()) + assert all(hashlib.sha256((persona / relative).read_bytes()).hexdigest() == digest for relative, digest in before.items()) + + archived = run_archivebox_cmd( + ["add", "--persona=session", "--plugins=dom", httpserver.url_for("/private")], + cwd=initialized_archive, + timeout=180, + env={**browser_env, "CHROME_SANDBOX": "false"}, + ) + assert archived.returncode == 0, archived.stderr + outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html")) + assert len(outputs) == 1 + html = outputs[0].read_text() + assert "Authenticated persona notebook" in html + assert "forest-green" in html + assert "Please log in" not in html diff --git a/docs/Personas.md b/docs/Personas.md new file mode 100644 index 00000000..b6bfefa4 --- /dev/null +++ b/docs/Personas.md @@ -0,0 +1,78 @@ +# Personas: import browser logins and settings + +A persona holds a copy of one Chromium browser profile, its cookies, and its browser settings. ArchiveBox uses that identity when you pass `--persona=personal` to `add`. Chrome, Chromium, Brave, and Edge are detected automatically. Other Chromium browsers work with an explicit profile path and browser executable. + +## Import on the computer where you use the browser + +Run the import as your normal desktop user, with your OS keychain unlocked. For Chrome, Chromium, Brave, Edge, Vivaldi, and Opera on desktop systems, ArchiveBox reads a consistent copy of the cookie database and decrypts it through the host OS keychain without launching the browser. On headless Linux without a desktop D-Bus session, it exports through a temporary profile copy using the originating browser instead. It does not change your original profile. Close the source browser first if you want a consistent copy of all settings and site storage. + +```bash +uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev' +mkdir -p ~/archivebox/data +cd ~/archivebox/data +archivebox init +archivebox persona create --import=brave personal +archivebox add --persona=personal 'https://example.com/private-page' +``` + +Use `--import=chrome`, `--import=chromium`, or `--import=edge` for those browsers. Your OS may ask to allow keychain access for the import. Cookie values are never printed by the command. + +If you have multiple profiles, select one explicitly: + +```bash +archivebox persona create --import=chrome --profile='Profile 1' work +``` + +The command reports success only after profile copying and cookie export finish. If import fails, it exits nonzero, removes a newly created persona, and preserves an existing persona's imported files. Re-importing replaces the selected persona's profile and exported state with the new source state. + +## Docker on macOS or Windows + +**Import on the host, then archive in Docker using the same data directory.** Mounting the raw browser directory alone does not give Linux access to macOS Keychain or Windows credential protection. An encrypted cookie database is not a portable login session. + +```bash +# On your host, after installing ArchiveBox with uv as above: +mkdir -p ~/archivebox/data +cd ~/archivebox/data +archivebox init +archivebox persona create --import=brave personal + +# Now use the resulting collection in Docker: +docker pull archivebox/archivebox:dev +docker run --rm -v "$PWD:/data" archivebox/archivebox:dev \ + add --persona=personal 'https://example.com/private-page' +``` + +Both commands use the public CLI. No copying files into persona directories, manual database changes, cookie extensions, or credentials pasted into Docker are needed. Refresh expired sessions by re-running the host import command. Stop archiving while replacing an existing persona. + +## Explicit source paths and other Chromium browsers + +`--source` accepts either a browser user-data directory or the exact profile directory. Known browsers select their own OS cookie decoder. For other Chromium browsers, also pass `--browser-binary` to export through a temporary copy launched with the originating browser: + +```bash +archivebox persona create --import=brave \ + --source="$HOME/Library/Application Support/BraveSoftware/Brave-Browser" \ + --profile=Default personal + +# Example for another Chromium browser on macOS: +archivebox persona create --import=vivaldi \ + --source="$HOME/Library/Application Support/Vivaldi" \ + personal +``` + +In Linux containers, `--source=/browser` can select a read-only mounted browser directory, but the originating browser and its decryption credentials must also be available. Prefer importing on the desktop host to avoid keyring and browser-installation setup inside Docker. + +## A running browser with CDP + +For any Chromium browser already exposing an authorized Chrome DevTools Protocol endpoint: + +```bash +archivebox persona create --import=http://127.0.0.1:9222 personal +``` + +This exports cookies and accessible open-tab storage. It cannot copy browser preferences or extensions; use the profile import above when settings are needed. Keep debugging endpoints private. + +## What is preserved + +The selected profile is normalized to `Default` within the persona. Preferences, bookmarks, extensions and their settings, local storage, IndexedDB, and service-worker storage are copied. Cache, temporary writes, runtime locks, and saved tab sessions are discarded so archiving does not reopen your personal tabs. Other browser profiles are not imported. `auth.json` retains exported cookie attributes such as HttpOnly and SameSite; `cookies.txt` supplies the formats that wget/curl and other non-browser extractors accept. + +Browser versions and products can differ in which settings and extensions they support. Windows app-bound cookie encryption may prevent direct cookie export; use an authorized live CDP endpoint if the browser refuses to decrypt the clone. A website can expire or reject a session, so verify the saved screenshot or HTML contains the private content. A successful HTTP response or extractor exit alone does not prove authentication. diff --git a/docs/Usage.md b/docs/Usage.md index 585ceba4..3f8cb494 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -131,10 +131,10 @@ To archive logged-in sites, you can import cookies from your browser into a pers archivebox persona create --import=chrome personal # supported: chrome/chromium/brave/edge (Chromium-based only) # use --profile to target a specific profile (e.g. Default, Profile 1) -# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data +# re-running import replaces this persona with the selected profile and exported cookies ``` -If cookie extraction fails, you can still export a Netscape-format `cookies.txt` using a browser extension and place it at `data/personas//cookies.txt`. +For Docker on macOS/Windows, run the import on the browser host into the shared data directory first. The container cannot decrypt the host's encrypted cookies. See [Personas](Personas.md) for the complete Docker workflow, explicit source paths, other Chromium browsers, and import troubleshooting.
diff --git a/pyproject.toml b/pyproject.toml index 6a2ea0b7..269b9399 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ dependencies = [ "abx-dl==1.12.269", # shared ArchiveBox downloader package ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 + "browser-cookie3>=0.20.1", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index ef164765..9a7fccc4 100644 --- a/uv.lock +++ b/uv.lock @@ -13,11 +13,11 @@ supported-markers = [ ] [options] -exclude-newer = "2026-09-02T09:18:47.302936861Z" +exclude-newer = "2026-09-06T20:29:42.802472Z" exclude-newer-span = "P5D" [options.exclude-newer-package] -abxbus = { timestamp = "2026-09-07T09:18:46.302952054Z", span = "PT1S" } +abxbus = { timestamp = "2026-09-11T20:29:41.802584Z", span = "PT1S" } abx-plugins = "2100-01-01T00:00:00Z" abx-dl = "2100-01-01T00:00:00Z" abxpkg = "2100-01-01T00:00:00Z" @@ -38,9 +38,9 @@ dependencies = [ { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/2b/3a4ac36610d02f4f69bb336dba6f281f3949377447905a26fb72b51deb62/abx_dl-1.12.269.tar.gz", hash = "sha256:63cbfff8d44dcbe13fe58a6c25dd18df00c392da07a58bcc3f915417e5b1a3c5", size = 86798 } +sdist = { url = "https://files.pythonhosted.org/packages/ea/2b/3a4ac36610d02f4f69bb336dba6f281f3949377447905a26fb72b51deb62/abx_dl-1.12.269.tar.gz", hash = "sha256:63cbfff8d44dcbe13fe58a6c25dd18df00c392da07a58bcc3f915417e5b1a3c5", size = 86798, upload-time = "2026-09-07T09:16:52.132Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/50/2f9ef17e208df4e98385eb72be98eaeda5e2850891a9d0e048716d2d80f3/abx_dl-1.12.269-py3-none-any.whl", hash = "sha256:a83b10fe5a38dd71ce0780ad66ff6c0e3aed8a4181412bc34448509a90a6500c", size = 92074 }, + { url = "https://files.pythonhosted.org/packages/c4/50/2f9ef17e208df4e98385eb72be98eaeda5e2850891a9d0e048716d2d80f3/abx_dl-1.12.269-py3-none-any.whl", hash = "sha256:a83b10fe5a38dd71ce0780ad66ff6c0e3aed8a4181412bc34448509a90a6500c", size = 92074, upload-time = "2026-09-07T09:16:51.001Z" }, ] [[package]] @@ -137,6 +137,7 @@ dependencies = [ { name = "atomicwrites", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "base32-crockford", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "bleach", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "browser-cookie3", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "croniter", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "daphne", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -233,6 +234,7 @@ requires-dist = [ { name = "atomicwrites", specifier = "==1.4.1" }, { name = "base32-crockford", specifier = ">=0.3.0" }, { name = "bleach", specifier = ">=6.2.0" }, + { name = "browser-cookie3", specifier = ">=0.20.1" }, { name = "click", specifier = ">=8.3.1" }, { name = "croniter", specifier = ">=6.0.0" }, { name = "daphne", specifier = ">=4.2.1" }, @@ -423,6 +425,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/f6/b55ec74cfe68c6584163faa311503c20b0da4c09883a41e8e00d6726c954/bottle-0.13.4-py2.py3-none-any.whl", hash = "sha256:045684fbd2764eac9cdeb824861d1551d113e8b683d8d26e296898d3dd99a12e", size = 103807, upload-time = "2025-06-15T10:08:57.691Z" }, ] +[[package]] +name = "browser-cookie3" +version = "0.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jeepney", marker = "sys_platform == 'linux' or (sys_platform == 'darwin' and 'bsd' in sys_platform)" }, + { name = "lz4", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pycryptodomex", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/e1/652adea0ce25948e613ef78294c8ceaf4b32844aae00680d3a1712dde444/browser_cookie3-0.20.1.tar.gz", hash = "sha256:6d8d0744bf42a5327c951bdbcf77741db3455b8b4e840e18bab266d598368a12", size = 22665, upload-time = "2024-12-20T00:31:30.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/57/2a716f4ecf6c50b2dbe27439507c480bb7ca5725edef82349ecdcfcdd084/browser_cookie3-0.20.1-py3-none-any.whl", hash = "sha256:4b38bf669d386250733c8339f0036e1cf09c3d8e4d326fd507b9afb84def13d6", size = 17229, upload-time = "2025-01-04T14:46:14.753Z" }, +] + [[package]] name = "bumpver" version = "2026.1132" @@ -1085,6 +1101,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, ] +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1178,6 +1203,29 @@ django = [ { name = "opentelemetry-instrumentation-django", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c", size = 207171, upload-time = "2025-11-03T13:01:56.595Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a", size = 207163, upload-time = "2025-11-03T13:01:57.721Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d", size = 1292136, upload-time = "2025-11-03T13:02:00.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/d1/f1d259352227bb1c185288dd694121ea303e43404aa77560b879c90e7073/lz4-4.4.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:533298d208b58b651662dd972f52d807d48915176e5b032fb4f8c3b6f5fe535c", size = 1279639, upload-time = "2025-11-03T13:02:01.649Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fb/ba9256c48266a09012ed1d9b0253b9aa4fe9cdff094f8febf5b26a4aa2a2/lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451039b609b9a88a934800b5fc6ee401c89ad9c175abf2f4d9f8b2e4ef1afc64", size = 1368257, upload-time = "2025-11-03T13:02:03.35Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/6a5c2952971af73f15ed4ebfdd69774b454bd0dc905b289082ca8664fba1/lz4-4.4.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67531da3b62f49c939e09d56492baf397175ff39926d0bd5bd2d191ac2bff95f", size = 207348, upload-time = "2025-11-03T13:02:08.117Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d7/fd62cbdbdccc35341e83aabdb3f6d5c19be2687d0a4eaf6457ddf53bba64/lz4-4.4.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a1acbbba9edbcbb982bc2cac5e7108f0f553aebac1040fbec67a011a45afa1ba", size = 207340, upload-time = "2025-11-03T13:02:09.152Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/225ffadaacb4b0e0eb5fd263541edd938f16cd21fe1eae3cd6d5b6a259dc/lz4-4.4.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a482eecc0b7829c89b498fda883dbd50e98153a116de612ee7c111c8bcf82d1d", size = 1293398, upload-time = "2025-11-03T13:02:10.272Z" }, + { url = "https://files.pythonhosted.org/packages/c6/9e/2ce59ba4a21ea5dc43460cba6f34584e187328019abc0e66698f2b66c881/lz4-4.4.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e099ddfaa88f59dd8d36c8a3c66bd982b4984edf127eb18e30bb49bdba68ce67", size = 1281209, upload-time = "2025-11-03T13:02:12.091Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/4d946bd1624ec229b386a3bc8e7a85fa9a963d67d0a62043f0af0978d3da/lz4-4.4.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2af2897333b421360fdcce895c6f6281dc3fab018d19d341cf64d043fc8d90d", size = 1369406, upload-time = "2025-11-03T13:02:13.683Z" }, + { url = "https://files.pythonhosted.org/packages/63/9c/70bdbdb9f54053a308b200b4678afd13efd0eafb6ddcbb7f00077213c2e5/lz4-4.4.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c216b6d5275fc060c6280936bb3bb0e0be6126afb08abccde27eed23dead135f", size = 207586, upload-time = "2025-11-03T13:02:18.263Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cb/bfead8f437741ce51e14b3c7d404e3a1f6b409c440bad9b8f3945d4c40a7/lz4-4.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8e71b14938082ebaf78144f3b3917ac715f72d14c076f384a4c062df96f9df6", size = 207161, upload-time = "2025-11-03T13:02:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/e7/18/b192b2ce465dfbeabc4fc957ece7a1d34aded0d95a588862f1c8a86ac448/lz4-4.4.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b5e6abca8df9f9bdc5c3085f33ff32cdc86ed04c65e0355506d46a5ac19b6e9", size = 1292415, upload-time = "2025-11-03T13:02:20.829Z" }, + { url = "https://files.pythonhosted.org/packages/67/79/a4e91872ab60f5e89bfad3e996ea7dc74a30f27253faf95865771225ccba/lz4-4.4.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b84a42da86e8ad8537aabef062e7f661f4a877d1c74d65606c49d835d36d668", size = 1279920, upload-time = "2025-11-03T13:02:22.013Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/d52c7b11eaa286d49dae619c0eec4aabc0bf3cda7a7467eb77c62c4471f3/lz4-4.4.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bba042ec5a61fa77c7e380351a61cb768277801240249841defd2ff0a10742f", size = 1368661, upload-time = "2025-11-03T13:02:23.208Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -1725,6 +1773,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pycryptodomex" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/85/e24bf90972a30b0fcd16c73009add1d7d7cd9140c2498a68252028899e41/pycryptodomex-3.23.0.tar.gz", hash = "sha256:71909758f010c82bc99b0abf4ea12012c98962fbf0583c2164f8b84533c2e4da", size = 4922157, upload-time = "2025-05-17T17:23:41.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/00/10edb04777069a42490a38c137099d4b17ba6e36a4e6e28bdc7470e9e853/pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:7b37e08e3871efe2187bc1fd9320cc81d87caf19816c648f24443483005ff886", size = 2498764, upload-time = "2025-05-17T17:22:21.453Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3f/2872a9c2d3a27eac094f9ceaa5a8a483b774ae69018040ea3240d5b11154/pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:91979028227543010d7b2ba2471cf1d1e398b3f183cb105ac584df0c36dac28d", size = 1643012, upload-time = "2025-05-17T17:22:23.702Z" }, + { url = "https://files.pythonhosted.org/packages/70/af/774c2e2b4f6570fbf6a4972161adbb183aeeaa1863bde31e8706f123bf92/pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b8962204c47464d5c1c4038abeadd4514a133b28748bcd9fa5b6d62e3cec6fa", size = 2187643, upload-time = "2025-05-17T17:22:26.37Z" }, + { url = "https://files.pythonhosted.org/packages/de/a3/71065b24cb889d537954cedc3ae5466af00a2cabcff8e29b73be047e9a19/pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a33986a0066860f7fcf7c7bd2bc804fa90e434183645595ae7b33d01f3c91ed8", size = 2273762, upload-time = "2025-05-17T17:22:28.313Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0b/ff6f43b7fbef4d302c8b981fe58467b8871902cdc3eb28896b52421422cc/pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7947ab8d589e3178da3d7cdeabe14f841b391e17046954f2fbcd941705762b5", size = 2313012, upload-time = "2025-05-17T17:22:30.57Z" }, + { url = "https://files.pythonhosted.org/packages/02/de/9d4772c0506ab6da10b41159493657105d3f8bb5c53615d19452afc6b315/pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c25e30a20e1b426e1f0fa00131c516f16e474204eee1139d1603e132acffc314", size = 2186856, upload-time = "2025-05-17T17:22:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/28/ad/8b30efcd6341707a234e5eba5493700a17852ca1ac7a75daa7945fcf6427/pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:da4fa650cef02db88c2b98acc5434461e027dce0ae8c22dd5a69013eaf510006", size = 2347523, upload-time = "2025-05-17T17:22:35.386Z" }, + { url = "https://files.pythonhosted.org/packages/0f/02/16868e9f655b7670dbb0ac4f2844145cbc42251f916fc35c414ad2359849/pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:58b851b9effd0d072d4ca2e4542bf2a4abcf13c82a29fd2c93ce27ee2a2e9462", size = 2272825, upload-time = "2025-05-17T17:22:37.632Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9c/1a8f35daa39784ed8adf93a694e7e5dc15c23c741bbda06e1d45f8979e9e/pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:06698f957fe1ab229a99ba2defeeae1c09af185baa909a31a5d1f9d42b1aaed6", size = 2499240, upload-time = "2025-05-17T17:22:46.953Z" }, + { url = "https://files.pythonhosted.org/packages/7a/62/f5221a191a97157d240cf6643747558759126c76ee92f29a3f4aee3197a5/pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2c2537863eccef2d41061e82a881dcabb04944c5c06c5aa7110b577cc487545", size = 1644042, upload-time = "2025-05-17T17:22:49.098Z" }, + { url = "https://files.pythonhosted.org/packages/8c/fd/5a054543c8988d4ed7b612721d7e78a4b9bf36bc3c5ad45ef45c22d0060e/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43c446e2ba8df8889e0e16f02211c25b4934898384c1ec1ec04d7889c0333587", size = 2186227, upload-time = "2025-05-17T17:22:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a9/8862616a85cf450d2822dbd4fff1fcaba90877907a6ff5bc2672cafe42f8/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f489c4765093fb60e2edafdf223397bc716491b2b69fe74367b70d6999257a5c", size = 2272578, upload-time = "2025-05-17T17:22:53.676Z" }, + { url = "https://files.pythonhosted.org/packages/46/9f/bda9c49a7c1842820de674ab36c79f4fbeeee03f8ff0e4f3546c3889076b/pycryptodomex-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdc69d0d3d989a1029df0eed67cc5e8e5d968f3724f4519bd03e0ec68df7543c", size = 2312166, upload-time = "2025-05-17T17:22:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/03/cc/870b9bf8ca92866ca0186534801cf8d20554ad2a76ca959538041b7a7cf4/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bbcb1dd0f646484939e142462d9e532482bc74475cecf9c4903d4e1cd21f003", size = 2185467, upload-time = "2025-05-17T17:22:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/96/e3/ce9348236d8e669fea5dd82a90e86be48b9c341210f44e25443162aba187/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:8a4fcd42ccb04c31268d1efeecfccfd1249612b4de6374205376b8f280321744", size = 2346104, upload-time = "2025-05-17T17:23:02.112Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e869bcee87beb89040263c416a8a50204f7f7a83ac11897646c9e71e0daf/pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd", size = 2271038, upload-time = "2025-05-17T17:23:04.872Z" }, +] + [[package]] name = "pydantic" version = "2.13.4"