The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity
7167c5dd8a
ArchiveBox/pyproject.toml
Claude 7167c5dd8a
Add native PostgreSQL support alongside SQLite 
Add DATABASE_ENGINE=postgres (plus DATABASE_HOST/PORT/USER/PASSWORD/NAME)
config and centralize all sqlite-vs-postgres branching in
archivebox.misc.db:

- get_database_settings() builds DATABASES for either backend; the sqlite
  path is unchanged (custom lock-retry backend, same PRAGMAs).
- database_exists()/ensure_database_ready() replace index.sqlite3 file
  checks; init auto-creates the postgres database when missing.
- approximate_row_counts() serves admin index counts from sqlite_stat1 or
  pg_class.reltuples; missing-table detection covers both vendors.
- rebuild_models_from_migration_state() lets historical sqlite-only raw
  SQL migrations resync postgres schema from Django migration state at
  every divergence point (postgres can never hold legacy data, so
  affected tables are empty when these run). All raw-DDL and PRAGMA
  migrations are now vendor-gated with sqlite behavior byte-for-byte
  unchanged.
- A pre_save clamp truncates CharField values to max_length: sqlite
  never enforced varchar(n) but postgres does (e.g. long crawl labels).
- Collation-sensitive URL range scans branch to escaped LIKE on postgres
  (with a text_pattern_ops index) since linguistic collations break
  bytewise range tricks; the crawl-config JSON search wave gets a
  jsonb-text implementation.

Verified on real PostgreSQL 16: fresh init applies the entire migration
graph, schema matches models exactly (column-level parity check +
makemigrations --check), and add/run/list/search/status/remove all work
end-to-end. New test_postgres_backend.py suite boots a real throwaway
postgres cluster (initdb + pg_ctl); CI workflows install postgres server
binaries.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019YHSjZM6TstSAMN2PhgfUg
2026-07-23 17:08:47 +00:00

374 lines
13 KiB
TOML
Executable File
Raw Blame History

[project]
name = "archivebox"
version = "0.9.35rc138"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
license = {text = "MIT"}
readme = "README.md"
keywords = ["internet archiving", "web archiving", "digipres", "warc", "preservation", "backups", "archiving", "web", "bookmarks", "puppeteer", "browser", "download"]
classifiers = [
"Development Status :: 4 - Beta",
"Environment :: Console",
"Environment :: Web Environment",
"Framework :: Django",
"Intended Audience :: Developers",
"Intended Audience :: Education",
"Intended Audience :: End Users/Desktop",
"Intended Audience :: Information Technology",
"Intended Audience :: Legal Industry",
"Intended Audience :: System Administrators",
"License :: OSI Approved :: MIT License",
"Natural Language :: English",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
"Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
"Topic :: Sociology :: History",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: System :: Archiving",
"Topic :: System :: Archiving :: Backup",
"Topic :: System :: Recovery Tools",
"Topic :: Utilities",
"Typing :: Typed",
]
dependencies = [
### Django libraries
"setuptools>=74.1.0", # for: django 5 on python >=3.12, distutils is no longer in stdlib but django 5.1 expects distutils (TODO: check if this can be removed eventually)
"django>=6.0",
"psycopg[binary]>=3.2", # for: PostgreSQL database backend (ARCHIVEBOX_DATABASE_ENGINE=postgres)
"daphne>=4.2.1", # ASGI server for Django (no channels needed - websockets not used)
"django-ninja>=1.5.1",
"django-extensions>=3.2.3",
"django-signal-webhooks>=0.3.0",
"django-admin-data-views>=0.4.1",
"django-object-actions>=4.3.0",
"bleach>=6.2.0", # for: stripping unsafe HTML from user-editable titles, notes, labels, tags
### State Management
"python-statemachine[diagrams]>=2.3.6",
### CLI / Logging
"click>=8.3.1", # for: nicer CLI command + argument definitions
"rich>=14.2.0", # for: pretty CLI output
"rich-click>=1.9.5", # for: pretty CLI command help text & output
"ipython>=8.27.0", # for: archivebox shell (TODO: replace with bpython?)
"toml>=0.10.2", # for: archivebox config output and TOML migration helpers
### Host OS / System
"supervisor>=4.2.5", # for: archivebox server starting daphne and workers
"psutil>=6.0.0", # for: monitoring orchestractor,actors,workers,etc. and machine.models.Process
"platformdirs>=4.3.6", # for: finding a xdg-config dir to store tmp/lib files in
"py-machineid>=0.6.0", # for: machine/detect.py calculating unique machine guid
"atomicwrites==1.4.1", # for: config file writes, index.json file writes, etc. (TODO: remove this deprecated lib in favor of archivebox.filestore.util/os.rename/os.replace)
### Base Types
"pydantic>=2.8.0", # for: archivebox.api (django-ninja), archivebox.config (pydantic-settings), and archivebox.index.schema (pydantic)
"pydantic-settings>=2.5.2", # for: archivebox.config
"base32-crockford>=0.3.0", # for: encoding UUIDs in base32
### Static Typing
"django-stubs>=5.0.4", # for: vscode type hints on models and common django APIs
### API clients
"requests>=2.32.3", # for: fetching title, static files, headers (TODO: replace with httpx?)
"sonic-client>=1.0.0",
### Parsers
"dateparser>=1.2.0", # for: parsing pocket/pinboard/etc. RSS/bookmark import dates
"croniter>=6.0.0", # for: validating and computing crawl schedules
"tzdata>=2024.2", # needed for dateparser {TZ: UTC} on some systems: https://github.com/ArchiveBox/ArchiveBox/issues/1553
"w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags
### Extractor dependencies (runtime binaries resolved through abxpkg)
### Binary/Package Management
"abxbus==2.5.40", # EventBus API
"abxpkg==1.11.288", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins==1.11.311", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl==1.11.268", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]
[project.optional-dependencies]
sonic = [
# sonic client lib now included by default, sonic group is now a no-op:
# "sonic-client>=1.0.0",
# to use sonic make sure you have a sonic server running in docker (archivebox/sonic) or locally:
# echo "deb [signed-by=/usr/share/keyrings/valeriansaliou_sonic.gpg] https://packagecloud.io/valeriansaliou/sonic/debian/ bookworm main" > /etc/apt/sources.list.d/valeriansaliou_sonic.list
# curl -fsSL https://packagecloud.io/valeriansaliou/sonic/gpgkey | gpg --dearmor -o /usr/share/keyrings/valeriansaliou_sonic.gpg
# apt install sonic
]
ldap = [
# python-ldap depends on the openldap bindings which provide no prebuilt wheels because they link against tons of other system packages
# apt install build-essential python3-dev python3-ldap libsasl2-dev libldap2-dev libssl-dev
"python-ldap>=3.4.3",
"django-auth-ldap>=4.1.0",
]
debug = [
# packages needed for running with DEBUG=True
"django-debug-toolbar>=4.4.6",
"djdt_flamegraph>=0.2.13",
"ipdb>=0.13.13",
"requests-tracker>=0.3.3",
"django-autotyping>=0.5.1",
]
all = [
"archivebox[sonic,ldap,debug]"
]
[dependency-groups]
dev = [
### BUILD
"uv>=0.11.3",
"bumpver>=2023.1129",
### DOCS
"recommonmark>=0.7.1",
"sphinx>=8.1.3",
"sphinx-rtd-theme>=2.0.0",
"sphinxcontrib-mermaid>=1.0.0",
"myst-parser>=4.0.0",
"sphinx-autodoc2>=0.5.0",
"linkify-it-py>=2.0.3",
### DEBUGGING
"django-debug-toolbar>=4.4.6",
"requests-tracker>=0.3.3",
"djdt_flamegraph>=0.2.13",
"ipdb>=0.13.13",
"logfire[django]>=0.51.0",
"opentelemetry-instrumentation-django>=0.47b0",
"opentelemetry-instrumentation-sqlite3>=0.47b0",
"viztracer>=0.17.0", # usage: viztracer ../.venv/bin/archivebox manage check
# "snakeviz", # usage: python -m cProfile -o flamegraph.prof ../.venv/bin/archivebox manage check
### TESTING
"pytest-django>=4.11.1",
"pytest>=8.3.3,<9", # pytest-codeblocks 0.17.0 still uses the pre-pytest-9 collect hook signature.
"pytest-codeblocks>=0.17.0",
"pytest-cov>=6.0.0",
"pytest-github-actions-annotate-failures>=0.4.0",
"pytest-httpserver>=1.1.0",
"pytest-sugar>=1.0.0",
"pytest-timeout>=2.3.1",
"coverage[toml]>=7.6.0",
"bottle>=0.13.1",
### LINTING
"prek>=0.3.6",
"ruff>=0.6.6",
"pyright>=1.1.406",
"ty>=0.0.1a19",
]
[tool.uv]
environments = ["sys_platform == 'darwin'", "sys_platform == 'linux'"]
package = true
exclude-newer = "5 days"
exclude-newer-package = { abx-plugins = "2100-01-01T00:00:00Z", abx-dl = "2100-01-01T00:00:00Z", abxpkg = "2100-01-01T00:00:00Z", abxbus = "1 second" }
# compile-bytecode = true
[build-system]
requires = ["pdm-backend"]
build-backend = "pdm.backend"
# https://github.com/astral-sh/uv/issues/3957
[tool.pdm.build]
includes = ["archivebox/"]
source-includes = [
".git/HEAD",
".git/refs/heads/dev",
]
excludes = [
".cache",
".mypy_cache",
".pdm-build",
".pytest_cache",
".ruff_cache",
".tmp",
".uv-cache",
".venv",
"ArchiveBox.conf",
"archive",
"archivebox/.DS_Store",
"archivebox/*/.DS_Store",
"archivebox/.flake8",
"archivebox/__pycache__",
"archivebox/index.sqlite3",
"archivebox/index.sqlite3-*",
"archivebox/machine/tests",
"archivebox/mcp/README.md",
"archivebox/tests/",
"archivebox/workers/tests",
"build",
"cache",
"caches",
"chrome_profile",
"chromeprofile",
"crawls",
"data",
"data*",
"deb_dist",
"dist",
"index.sqlite3",
"index.sqlite3-*",
"lib",
"logs",
"node_modules",
"out",
"output",
"personas",
"pip_dist",
"queue.sqlite3",
"queue.sqlite3-*",
"snapshots",
"sources",
"tests",
"tmp",
"users",
"venv",
]
[tool.setuptools]
packages = ["archivebox"]
package-dir = {"archivebox" = "archivebox"}
[tool.ruff]
line-length = 140
target-version = "py313"
src = ["archivebox"]
force-exclude = true
exclude = ["*.pyi", "*.html", "**/*.html", "typings/", "migrations/", "archivebox/tests/data/"]
# https://docs.astral.sh/ruff/rules/
[tool.ruff.lint]
ignore = ["E731", "E303", "E266", "E241", "E222"]
[tool.codespell]
ignore-words-list = "abx,archivebox,adminsnapshots,bu,wit,dont,cant,wont,havent,thats,shouldnt,doesnt,doenst,succeded,re-use,re-used,re-using,re-usable,generat"
skip = "*.json,*.min.js,*.min.css,uv.lock,old/*,publicsite/*,docs/apidocs/*"
[tool.pytest.ini_options]
testpaths = [ "archivebox/tests", "README.md", "AGENTS.md", "skills", "docs" ]
addopts = ["--codeblocks"]
norecursedirs = ["archivebox/tests/data"]
timeout = 600
timeout_method = "thread"
# Note: Plugin tests under abx_plugins/plugins/ must NOT load Django
# They use a conftest.py to disable Django automatically
[tool.coverage.run]
# Enable branch coverage (tracks if/else branches)
branch = true
# What to measure
source = ["archivebox"]
# Support parallel execution (for integration tests, dev server, etc.)
parallel = true
# Store data in .coverage instead of .coverage.<pid>
data_file = ".coverage"
# What to exclude
omit = [
"*/tests/*",
"*/test_*.py",
"*/migrations/*",
"*/typings/*",
"*/__pycache__/*",
"*/node_modules/*",
"*/.venv/*",
"*/manage.py",
]
[tool.coverage.report]
# Show lines missing coverage
show_missing = true
# Skip files with no executable code
skip_empty = true
# Fail if coverage below this (set to 0 for now)
fail_under = 0
# Exclude patterns (regex)
exclude_lines = [
# Standard pragma
"pragma: no cover",
# Don't complain about missing debug code
"def __repr__",
"if self.debug",
# Don't complain if tests don't cover defensive assertion code
"raise AssertionError",
"raise NotImplementedError",
# Don't complain if non-runnable code isn't run
"if 0:",
"if False:",
"if __name__ == .__main__.:",
# Type checking blocks
"if TYPE_CHECKING:",
# Abstract methods
"@(abc\\.)?abstractmethod",
]
[tool.coverage.html]
directory = "htmlcov"
[tool.coverage.json]
output = "coverage.json"
show_contexts = true
[tool.pyright]
include = [
"archivebox",
]
exclude = [
".venv",
"**/*.pyi",
"**/__init__.pyi",
"**/node_modules",
"**/__pycache__",
"**/migrations",
"archivebox/tests/data",
"archivebox/tests/data/**",
]
stubPath = "./typings"
venvPath = "."
venv = ".venv"
reportMissingImports = true
reportMissingTypeStubs = false
pythonVersion = "3.13"
pythonPlatform = "Linux"
[tool.ty]
environment = { python-version = "3.13", python-platform = "linux" }
src = { include = ["archivebox"], exclude = [".venv", "**/*.pyi", "**/__init__.pyi", "**/node_modules", "**/__pycache__", "**/migrations", "archivebox/tests/data", "archivebox/tests/data/**"] }
[project.scripts]
archivebox = "archivebox.cli:main"
[project.urls]
Homepage = "https://github.com/ArchiveBox/ArchiveBox"
Source = "https://github.com/ArchiveBox/ArchiveBox"
Documentation = "https://github.com/ArchiveBox/ArchiveBox/wiki"
"Bug Tracker" = "https://github.com/ArchiveBox/ArchiveBox/issues"
Changelog = "https://github.com/ArchiveBox/ArchiveBox/releases"
Roadmap = "https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap"
Community = "https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community"
Demo = "https://demo.archivebox.io"
Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
[tool.bumpver]
current_version = "v0.9.35rc138"
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
commit_message = "bump version {old_version} -> {new_version}"
tag_message = "{new_version}"
tag_scope = "default"
pre_commit_hook = "bin/bumpver_update_lock.sh"
post_commit_hook = ""
commit = true
tag = true
push = true
[tool.bumpver.file_patterns]
"pyproject.toml" = [
'current_version = "{version}"',
'version = "{pep440_version}"',
]
"etc/package.json" = [
'"version": "{pep440_version}"',
]
Reference in New Issue View Git Blame Copy Permalink