mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
481 lines
33 KiB
HTML
481 lines
33 KiB
HTML
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<meta name="description" content="ArchiveBox is open-source self-hosted web archiving for preserving websites, bookmarks, RSS feeds, social posts, media, evidence, research sources, and institutional records in durable HTML, PDF, PNG, WARC, JSON, SQLite, and filesystem formats.">
|
|
<meta name="keywords" content="ArchiveBox, self-hosted web archiving, open source web archive, website preservation, archive bookmarks, RSS archiving, WARC, SingleFile, yt-dlp, Chrome headless, Docker Compose archive, digital preservation">
|
|
<meta name="robots" content="index,follow,max-image-preview:large">
|
|
<link rel="canonical" href="https://archivebox.io/">
|
|
<meta property="og:type" content="website">
|
|
<meta property="og:url" content="https://archivebox.io/">
|
|
<meta property="og:title" content="ArchiveBox - Open-source self-hosted web archiving">
|
|
<meta property="og:description" content="Preserve websites, media, bookmarks, feeds, source code, evidence, and research material in durable files you control.">
|
|
<meta property="og:image" content="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2">
|
|
<meta name="twitter:card" content="summary_large_image">
|
|
<title>ArchiveBox - Open-source self-hosted web archiving</title>
|
|
<link rel="icon" href="./icon.png">
|
|
<link rel="apple-touch-icon" href="./icon.png">
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
|
<link rel="stylesheet" href="./shadcn-theme.css">
|
|
<link rel="stylesheet" href="./styles.css">
|
|
<script type="application/ld+json">
|
|
{
|
|
"@context": "https://schema.org",
|
|
"@type": "SoftwareApplication",
|
|
"name": "ArchiveBox",
|
|
"applicationCategory": "WebApplication",
|
|
"operatingSystem": "Linux, macOS, BSD, Windows with Docker",
|
|
"description": "Open-source self-hosted web archiving for saving websites, bookmarks, RSS feeds, social media, media files, source code, and research material in durable formats.",
|
|
"url": "https://archivebox.io/",
|
|
"codeRepository": "https://github.com/ArchiveBox/ArchiveBox",
|
|
"license": "https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE",
|
|
"softwareHelp": "https://github.com/ArchiveBox/ArchiveBox/wiki",
|
|
"downloadUrl": "https://github.com/ArchiveBox/ArchiveBox/releases",
|
|
"programmingLanguage": "Python",
|
|
"offers": {
|
|
"@type": "Offer",
|
|
"price": "0",
|
|
"priceCurrency": "USD"
|
|
}
|
|
}
|
|
</script>
|
|
</head>
|
|
<body>
|
|
<header class="site-header">
|
|
<a class="brand" href="#top" aria-label="ArchiveBox home">
|
|
<img src="./icon.png" alt="" width="42" height="42">
|
|
<span>ArchiveBox</span>
|
|
</a>
|
|
<nav aria-label="Primary navigation">
|
|
<a href="#install"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 3v11"/><path d="m7 10 5 5 5-5"/><path d="M5 21h14"/></svg><span>Install</span></a>
|
|
<a href="#use-cases"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><path d="M22 21v-2a4 4 0 0 0-3-3.87"/><path d="M16 3.13a4 4 0 0 1 0 7.75"/></svg><span>Use cases</span></a>
|
|
<a href="#configuration"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 21v-7"/><path d="M4 10V3"/><path d="M12 21v-9"/><path d="M12 8V3"/><path d="M20 21v-5"/><path d="M20 12V3"/><path d="M2 14h4"/><path d="M10 8h4"/><path d="M18 16h4"/></svg><span>Configuration</span></a>
|
|
<a href="#layout-security"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10Z"/><path d="m9 12 2 2 4-4"/></svg><span>Layout & security</span></a>
|
|
<a href="#background"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 8v4l3 3"/><circle cx="12" cy="12" r="9"/></svg><span>Backstory</span></a>
|
|
<a href="#ecosystem"><svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="14" width="7" height="7" rx="1.5"/><path d="M10 6.5h4"/><path d="M17.5 10v4"/><path d="M10 17.5h4"/></svg><span>Ecosystem</span></a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"/><path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2Z"/></svg><span>Docs</span></a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M15 22v-4a4.8 4.8 0 0 0-.09-.93"/><path d="M9 22v-4a4.8 4.8 0 0 1 .09-.93"/><path d="M17.8 20.2c1.4-.5 2.2-1.7 2.2-3.7V12.7c0-.7-.2-1.4-.7-2 .1-.3.3-1.7-.1-3.2 0 0-1.1-.4-3.4.9a11.5 11.5 0 0 0-6 0C7.5 7.1 6.4 7.5 6.4 7.5c-.4 1.5-.2 2.9-.1 3.2-.5.6-.7 1.3-.7 2v3.8c0 2 1 3.2 2.4 3.7"/><path d="M9 18c-3 1.3-5 0-6-2"/></svg><span>GitHub</span></a>
|
|
<a href="https://pypi.org/project/archivebox/"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16.5 9.4 7.5 4.2"/><path d="M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16Z"/><path d="M3.3 7 12 12l8.7-5"/><path d="M12 22V12"/></svg><span>PyPI</span></a>
|
|
</nav>
|
|
</header>
|
|
|
|
<main id="top">
|
|
<section class="hero section">
|
|
<div class="hero-copy">
|
|
<p class="eyebrow">Open-source self-hosted web archiving</p>
|
|
<h1><span class="hero-mainline">Preserve the web.</span><span class="hero-subline">On infrastructure you control.</span></h1>
|
|
<p class="lede">ArchiveBox saves websites, bookmarks, RSS feeds, social posts, media, source code, and research material in durable files like HTML, PDF, PNG, TXT, JSON, WARC, MP4, and SQLite.</p>
|
|
<div class="hero-actions" aria-label="Primary actions">
|
|
<a class="button primary" href="#install">Run with Docker Compose</a>
|
|
<a class="button secondary" href="./screenshots/">View screenshots</a>
|
|
</div>
|
|
<br/>
|
|
<hr/>
|
|
<br/>
|
|
<div class="hero-shields" aria-label="Project stats and package links">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green" alt="Open source MIT badge"></a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active" alt="Active development badge"></a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue" alt="GitHub stars badge"></a>
|
|
<a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls" alt="Docker pulls badge"></a>
|
|
<a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae" alt="PyPI installs badge"></a>
|
|
<a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8" alt="Chrome store users badge"></a>
|
|
</div>
|
|
<div class="hero-links" aria-label="Project links">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Documentation</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community">Community</a>
|
|
</div>
|
|
</div>
|
|
<div class="hero-panel" aria-label="ArchiveBox setup preview">
|
|
<div class="ab-terminal">
|
|
<div class="ab-terminal__bar">
|
|
<div class="ab-terminal__dots"><span class="ab-terminal__dot" aria-hidden="true"> </span><span class="ab-terminal__dot" aria-hidden="true"> </span><span class="ab-terminal__dot" aria-hidden="true"> </span></div>
|
|
<span>zsh · ~/archivebox · docker-compose</span>
|
|
</div>
|
|
<div class="ab-terminal__body">
|
|
<pre><span class="ab-dim"># Docker Compose is the recommended setup</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">mkdir -p ~/archivebox/data && cd ~/archivebox</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up -d --wait</span>
|
|
<span class="ab-out">-> initialized ./data/index.sqlite3</span>
|
|
<span class="ab-ok">ok listening on http://127.0.0.1:8000</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cursor" aria-hidden="true"> </span></pre>
|
|
</div>
|
|
</div>
|
|
<img class="product-shot" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2" alt="ArchiveBox snapshot list in the admin web interface">
|
|
</div>
|
|
</section>
|
|
|
|
<section class="trust-strip" aria-label="ArchiveBox capabilities">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">CLI</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Web UI</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication#rest-api">REST API</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/pull/1418">Webhooks</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#browser-extension-usage">Browser extension</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox#archive-layout">Filesystem access</a>
|
|
</section>
|
|
|
|
<section class="section split">
|
|
<div>
|
|
<p class="eyebrow">Why ArchiveBox</p>
|
|
<h2>Designed for useful archives, not proprietary silos.</h2>
|
|
</div>
|
|
<div class="feature-grid">
|
|
<article>
|
|
<h3>Readable for decades</h3>
|
|
<p>Snapshots are stored as ordinary files and folders, with metadata in SQLite and JSON. You can browse the collection without depending on a hosted service.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Multiple captures per URL</h3>
|
|
<p>ArchiveBox can save rendered HTML, screenshots, PDFs, WARC files, article text, headers, favicons, media, subtitles, and source repositories.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Bring your sources</h3>
|
|
<p>Import one URL, pipe text into the CLI, upload exported bookmarks, or schedule recurring pulls from RSS feeds and other text-based source lists.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Self-hosted by default</h3>
|
|
<p>Run it as a Docker web app, use one-off CLI commands, or automate with APIs while keeping private and public material under your own policy.</p>
|
|
</article>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="use-cases" class="section">
|
|
<div class="section-heading">
|
|
<p class="eyebrow">Who it is for</p>
|
|
<h2>Archiving workflows for individuals, professionals, and institutions.</h2>
|
|
</div>
|
|
<div class="audience-grid">
|
|
<article class="audience-card">
|
|
<h3>Personal archivists and self-hosters</h3>
|
|
<p>Save bookmarks, browser history, RSS feeds, social media, form content, videos, podcasts, music, photos, and personal knowledge collections.</p>
|
|
<ul>
|
|
<li>Own your data and keep it on local or remote storage you control.</li>
|
|
<li>Use the browser extension, CLI, Web UI, and scheduled imports together.</li>
|
|
<li>Export static HTML or browse the filesystem directly.</li>
|
|
</ul>
|
|
</article>
|
|
<article class="audience-card">
|
|
<h3>Lawyers and journalists</h3>
|
|
<p>Capture web pages, articles, source material, and public records while preserving reviewable copies outside of volatile platforms.</p>
|
|
<ul>
|
|
<li>Store screenshots, PDFs, headers, WARC files, and text extraction.</li>
|
|
<li>Tag and review sources through the self-hosted web interface.</li>
|
|
<li>Tune privacy, authentication, and publishing settings for casework.</li>
|
|
</ul>
|
|
</article>
|
|
<article class="audience-card">
|
|
<h3>Researchers and institutions</h3>
|
|
<p>Support OSINT, social media research, AI-powered research agents, libraries, governments, and collection-building teams.</p>
|
|
<ul>
|
|
<li>Automate imports through the CLI, REST API, webhooks, and schedules.</li>
|
|
<li>Keep machine-readable metadata in JSON and SQLite.</li>
|
|
<li>Extend extraction pipelines through the ArchiveBox ecosystem.</li>
|
|
</ul>
|
|
</article>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="install" class="section install-section">
|
|
<div class="install-copy">
|
|
<p class="eyebrow">Recommended install</p>
|
|
<h2>Start with Docker Compose.</h2>
|
|
<p>Docker Compose is the recommended ArchiveBox setup for the easiest install and update path, better isolation, and bundled archiving dependencies like Chrome, wget, yt-dlp, SingleFile, and readability tools.</p>
|
|
<div class="mini-links">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Docker install guide</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker usage wiki</a>
|
|
<a href="https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/docker-compose.yml">docker-compose.yml</a>
|
|
</div>
|
|
</div>
|
|
<div class="command-card">
|
|
<div class="step">
|
|
<span>1</span>
|
|
<div>
|
|
<h3>Create a data directory</h3>
|
|
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">mkdir -p ~/archivebox/data && cd ~/archivebox</span></pre></div></div>
|
|
</div>
|
|
</div>
|
|
<div class="step">
|
|
<span>2</span>
|
|
<div>
|
|
<h3>Download Docker Compose config</h3>
|
|
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml</span></pre></div></div>
|
|
</div>
|
|
</div>
|
|
<div class="step">
|
|
<span>3</span>
|
|
<div>
|
|
<h3>Start ArchiveBox (initializes automatically)</h3>
|
|
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up -d --wait</span></pre></div></div>
|
|
</div>
|
|
</div>
|
|
<div class="step">
|
|
<span>4</span>
|
|
<div>
|
|
<h3>Add your first URL</h3>
|
|
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose exec archivebox archivebox add 'https://example.com'</span></pre></div></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<section class="section compact-docs" aria-labelledby="other-installs-title">
|
|
<h2 id="other-installs-title">Other install paths</h2>
|
|
<p>ArchiveBox can also run with plain Docker, pip, brew, deb packages, or the optional setup script. Docker Compose remains the default recommendation, especially when you want bundled dependencies and clean upgrades.</p>
|
|
<div class="doc-link-grid">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Bare metal install</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">README quickstart</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">CLI usage</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="configuration" class="section deep-section">
|
|
<div class="section-heading">
|
|
<p class="eyebrow">Configuration</p>
|
|
<h2>Configure ArchiveBox with powerful options.</h2>
|
|
<p>ArchiveBox can be configured with environment variables, the <code>archivebox config</code> CLI, or by editing <code>./ArchiveBox.conf</code>. The same configuration model works in Docker, Docker Compose, bare-metal installs, scheduled jobs, and one-off CLI runs.</p>
|
|
</div>
|
|
<div class="prose-grid">
|
|
<article>
|
|
<h3>Common settings people search for</h3>
|
|
<p>Use <code>TIMEOUT</code> for slow networks, <code>CHECK_SSL_VALIDITY</code> for sites with broken certificates, <code>PUBLIC_INDEX</code>, <code>PUBLIC_SNAPSHOTS</code>, and <code>PUBLIC_ADD_VIEW</code> for publishing policy, and browser user-agent settings for sites that block obvious bots.</p>
|
|
<p>For authenticated or difficult sites, review <code>CHROME_USER_DATA_DIR</code>, <code>COOKIES_FILE</code>, <code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, and <code>CURL_USER_AGENT</code>. Public archive operators should also configure instance branding and contact details with settings like <code>FOOTER_INFO</code> and <code>CUSTOM_TEMPLATES_DIR</code>.</p>
|
|
<div class="ab-terminal">
|
|
<div class="ab-terminal__bar">
|
|
<div class="ab-terminal__dots"><span class="ab-terminal__dot" aria-hidden="true"> </span><span class="ab-terminal__dot" aria-hidden="true"> </span><span class="ab-terminal__dot" aria-hidden="true"> </span></div>
|
|
<span>archivebox config</span>
|
|
</div>
|
|
<div class="ab-terminal__body">
|
|
<pre><span class="ab-prompt">$</span> <span class="ab-cmd">archivebox config</span> <span class="ab-dim"># view full config</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">archivebox config --get CHROME_BINARY</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">archivebox config --set TIMEOUT=240</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">archivebox config --set PUBLIC_INDEX=False</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">env CHROME_BINARY=chromium archivebox add 'https://example.com'</span></pre>
|
|
</div>
|
|
</div>
|
|
</article>
|
|
<article>
|
|
<h3>Dependencies and extractors</h3>
|
|
<p>ArchiveBox uses standard tools like Chrome or Chromium, wget, curl, yt-dlp, git, SingleFile, Readability, and article parsers. Docker bundles these dependencies for easier upgrades and better isolation; non-Docker installs can run <code>archivebox install</code> and <code>archivebox --version</code> to check what is available.</p>
|
|
<ul class="readme-list">
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration wiki</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker configuration</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Install dependencies</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Chromium install options</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies and logged-in browser profiles</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Troubleshooting installs</a></li>
|
|
</ul>
|
|
</article>
|
|
</div>
|
|
</section>
|
|
|
|
<section class="section data-section">
|
|
<div class="section-heading">
|
|
<p class="eyebrow">Inputs and outputs</p>
|
|
<h2>Feed it URLs from any source. Save redundant, portable captures.</h2>
|
|
</div>
|
|
<div class="data-grid">
|
|
<article>
|
|
<h3>Inputs</h3>
|
|
<p>Archive one URL at a time or schedule imports from bookmarks, browser history, RSS, JSON, CSV, TXT, SQL, HTML, Markdown, Pocket, Pinboard, Instapaper, Shaarli, Wallabag, and more.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Outputs</h3>
|
|
<p>Each snapshot can include original HTML, rendered single-file HTML, PDF, screenshot PNG, WARC, title, article text, favicon, headers, media, subtitles, metadata, thumbnails, and git clones.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Access</h3>
|
|
<p>Manage the same collection through the Web UI, CLI, REST API, Python API, SQLite, or the data folder itself. The tools are complementary, not separate products.</p>
|
|
</article>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="layout-security" class="section deep-section">
|
|
<div class="section-heading">
|
|
<p class="eyebrow">Disk layout, exporting, and security</p>
|
|
<h2>A normal folder, a SQLite index, and files you can inspect without ArchiveBox.</h2>
|
|
<p>All ArchiveBox state for a collection lives in one data folder: the SQLite database, configuration, logs, and archived content. Snapshots are organized on disk as ordinary files and folders so they can be backed up, searched, published carefully, or browsed by hand.</p>
|
|
</div>
|
|
<div class="prose-grid prose-grid--wide">
|
|
<article>
|
|
<h3>Archive layout</h3>
|
|
<div class="ab-terminal">
|
|
<div class="ab-terminal__bar">
|
|
<div class="ab-terminal__dots"><span class="ab-terminal__dot" aria-hidden="true"> </span><span class="ab-terminal__dot" aria-hidden="true"> </span><span class="ab-terminal__dot" aria-hidden="true"> </span></div>
|
|
<span>~/archivebox/data</span>
|
|
</div>
|
|
<div class="ab-terminal__body">
|
|
<pre><span class="ab-out">data/</span>
|
|
<span class="ab-out"> index.sqlite3</span> <span class="ab-dim"># main metadata database</span>
|
|
<span class="ab-out"> ArchiveBox.conf</span> <span class="ab-dim"># collection configuration</span>
|
|
<span class="ab-out"> archive/</span>
|
|
<span class="ab-out"> 1617687755/</span>
|
|
<span class="ab-out"> index.html</span>
|
|
<span class="ab-out"> index.json</span>
|
|
<span class="ab-out"> screenshot.png</span>
|
|
<span class="ab-out"> output.pdf</span>
|
|
<span class="ab-out"> warc/1617687755.warc.gz</span>
|
|
<span class="ab-out"> media/some_video.mp4</span>
|
|
<span class="ab-out"> git/somerepo.git</span></pre>
|
|
</div>
|
|
</div>
|
|
<p>Each snapshot folder includes static <code>index.html</code> and <code>index.json</code> metadata plus extractor outputs such as HTML, PDFs, screenshots, WARC files, media, subtitles, headers, favicons, article text, and git repositories.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Static exports and publishing</h3>
|
|
<p>You can export the archive index as static HTML, JSON, or CSV with <code>archivebox list</code> so collections can be reviewed without running the web server. Keep generated exports next to the <code>archive/</code> folder so relative snapshot paths continue to work.</p>
|
|
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --html --with-headers > index.html</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --json --with-headers > index.json</span>
|
|
<span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --csv=timestamp,url,title > index.csv</span></pre></div></div>
|
|
<ul class="readme-list">
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Disk layout docs</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Storage backends: SMB, NFS, S3, B2, Google Drive</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publishing your archive</a></li>
|
|
<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Upgrading or merging archives</a></li>
|
|
</ul>
|
|
</article>
|
|
</div>
|
|
|
|
<div class="notice-grid">
|
|
<article>
|
|
<h3>Private content</h3>
|
|
<p>If you archive private URLs, paywalled pages, unlisted media, Google Docs, or browser sessions, assume snapshot viewers may see private URLs, cookies, session tokens, headers, and page content unless you configure access carefully.</p>
|
|
<p>Restrict public access with <code>PUBLIC_INDEX=False</code>, <code>PUBLIC_SNAPSHOTS=False</code>, and <code>PUBLIC_ADD_VIEW=False</code>, then create authenticated users with <code>archivebox manage createsuperuser</code>.</p>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Read the security overview</a>
|
|
</article>
|
|
<article>
|
|
<h3>Archived JavaScript</h3>
|
|
<p>Archived pages that execute JavaScript can make requests from the same domain as the web UI when viewed. If this risk matters for your collection, review the publishing guidance and consider disabling extractor outputs that execute archived JS when opened.</p>
|
|
<p><code>archivebox config --set SAVE_WGET=False SAVE_DOM=False</code></p>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/issues/239">Read Issue #239</a>
|
|
</article>
|
|
<article>
|
|
<h3>Storage requirements</h3>
|
|
<p>ArchiveBox can use roughly 1 GB to 50 GB per 1,000 snapshots depending on media, video, audio, and extractor settings. Tune <code>YTDLP_ENABLED</code> and <code>YTDLP_MAX_SIZE</code> for media-heavy collections.</p>
|
|
<p>Keep <code>index.sqlite3</code> on local storage or SSD when possible. Large <code>archive/</code> folders can live on NFS, SMB, FUSE, S3-backed, or HDD storage, but Docker and fileshare setups may need ownership and root-squash adjustments. Avoid older filesystems like EXT3 or FAT for very large archives.</p>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Plan for large archives</a>
|
|
</article>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="ecosystem" class="section ecosystem">
|
|
<div class="section-heading">
|
|
<p class="eyebrow">Ecosystem</p>
|
|
<h2>Core projects that make ArchiveBox extensible.</h2>
|
|
</div>
|
|
<div class="project-grid">
|
|
<article>
|
|
<h3>abx-plugins</h3>
|
|
<p>Extractor and plugin packages used to preserve more types of content.</p>
|
|
<div>
|
|
<a href="https://github.com/ArchiveBox/abx-plugins">GitHub</a>
|
|
<a href="https://archivebox.github.io/abx-plugins/">Docs</a>
|
|
</div>
|
|
</article>
|
|
<article>
|
|
<h3>abx-dl</h3>
|
|
<p>Download and extraction tooling for saving web content and media.</p>
|
|
<div>
|
|
<a href="https://github.com/ArchiveBox/abx-dl">GitHub</a>
|
|
</div>
|
|
</article>
|
|
<article>
|
|
<h3>abxpkg</h3>
|
|
<p>Runtime package management for dependencies used by ArchiveBox and plugins.</p>
|
|
<div>
|
|
<a href="https://github.com/ArchiveBox/abxpkg">GitHub</a>
|
|
<a href="https://archivebox.github.io/abxpkg/">Docs</a>
|
|
</div>
|
|
</article>
|
|
<article>
|
|
<h3>abxbus</h3>
|
|
<p>Event bus infrastructure for ArchiveBox automation and integrations.</p>
|
|
<div>
|
|
<a href="https://github.com/ArchiveBox/abxbus">GitHub</a>
|
|
<a href="https://abxbus.archivebox.io">Site</a>
|
|
</div>
|
|
</article>
|
|
</div>
|
|
</section>
|
|
|
|
<section id="background" class="section deep-section">
|
|
<div class="section-heading">
|
|
<p class="eyebrow">Background and motivation</p>
|
|
<h2>Built for decentralized preservation.</h2>
|
|
<p>ArchiveBox exists because link rot, platform churn, censorship, and disappearing media routinely erase useful knowledge. The project aims to make the web content you care about viewable with common software in 50 to 100 years, without requiring ArchiveBox or a hosted replay service to understand your files.</p>
|
|
</div>
|
|
<div class="prose-grid">
|
|
<article>
|
|
<h3>Why self-host?</h3>
|
|
<p>Centralized public archives are essential, but not every page belongs in a global public service. ArchiveBox lets individuals and organizations save public or private material they can access, keep it locally or within their institution, and decide what to publish case by case.</p>
|
|
<p>The project differentiates itself by combining a comprehensive CLI, a self-hosted web UI, API surfaces, scheduled imports, standard output formats, and a simple on-disk data layout that remains useful without the app running.</p>
|
|
</article>
|
|
<article>
|
|
<h3>Ethical and legal context</h3>
|
|
<p>Responsible archiving depends on your jurisdiction, your use case, and your publishing policy. ArchiveBox is a tool; operators remain responsible for handling private data, copyright, DMCA or GDPR requests, and local legal requirements.</p>
|
|
<p>Public instances should publish contact and removal information, avoid monetizing copied content, and review the security and publishing documentation before exposing snapshots to anonymous viewers.</p>
|
|
</article>
|
|
</div>
|
|
<div class="alternatives-band">
|
|
<h3>Related web archiving tools and communities</h3>
|
|
<p>ArchiveBox is a broad, self-hosted archive manager. For specialized crawling, replay, bookmark management, or research workflows, the wider web archiving ecosystem is worth knowing.</p>
|
|
<div class="doc-link-grid">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects">Web archiving projects</a>
|
|
<a href="https://archiveweb.page">ArchiveWeb.page</a>
|
|
<a href="https://replayweb.page">ReplayWeb.page</a>
|
|
<a href="https://github.com/webrecorder/browsertrix-crawler">Browsertrix</a>
|
|
<a href="https://scrapy.org/">Scrapy</a>
|
|
<a href="https://github.com/hoarder-app/hoarder">Hoarder</a>
|
|
<a href="https://github.com/linkwarden/linkwarden">Linkwarden</a>
|
|
<a href="https://github.com/iipc/awesome-web-archiving">Awesome Web Archiving</a>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<section class="section docs-band">
|
|
<div>
|
|
<p class="eyebrow">Documentation</p>
|
|
<h2>Further Reading.</h2>
|
|
<p>The README and wiki cover supported sources, outputs, scheduled archiving, storage backends, authentication, security, publishing, Chrome setup, upgrades, troubleshooting, API usage, development, and the broader web archiving community.</p>
|
|
</div>
|
|
<div class="doc-link-grid">
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart">Quickstart</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site">Supported outputs</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving">Scheduled archiving</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Storage">Storage backends</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication">Authentication</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security overview</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Chromium setup</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting">Troubleshooting</a>
|
|
<a href="https://docs.archivebox.io">API docs</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/issues">Issue tracker</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap">Roadmap</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/releases">Changelog</a>
|
|
</div>
|
|
</section>
|
|
</main>
|
|
|
|
<footer class="site-footer">
|
|
<div class="brand">
|
|
<img src="./icon.png" alt="" width="34" height="34">
|
|
<span>ArchiveBox</span>
|
|
</div>
|
|
<p>Open-source self-hosted web archiving. MIT licensed, nonprofit-supported, and built for durable collections.</p>
|
|
<div>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE">License</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/releases">Changelog</a>
|
|
<a href="https://zulip.archivebox.io">Community chat</a>
|
|
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations">Donations</a>
|
|
</div>
|
|
</footer>
|
|
</body>
|
|
</html>
|