The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: v0.9.33rc53

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-31 01:41:43 -07:00
parent c1e792fee6
commit fe421fdb79
No known key found for this signature in database
7 changed files with 1100 additions and 1136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
archivebox/core/models.py
View File

@ -756,22 +756,40 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
crawl = self.crawl
if not crawl.url_passes_filters(self.url, snapshot=self):
return
# Sibling snapshots in the same crawl race to append into the
# same ``crawl.urls`` text field; reload + retry on CAS miss
# so concurrent appends don't clobber each other.
for _ in range(16):
# TEMP INSTRUMENTATION: count retries to verify root cause
import logging as _logging
import os
for attempt in range(16):
crawl.refresh_from_db()
existing_urls = {url for _raw_line, url in crawl._iter_url_lines() if url}
if self.url in existing_urls:
if attempt > 0:
_logging.getLogger("archivebox.crawls").warning(
"CAS_RETRY[%d] another_writer_won pid=%d snap=%s crawl=%s url=%s",
attempt,
os.getpid(),
self.id,
crawl.id,
self.url,
)
return
urls = f"{crawl.urls}\n{self.url}"
if crawl.safe_update({"urls": urls, "modified_at": timezone.now()}, refresh=False):
if attempt > 0:
_logging.getLogger("archivebox.crawls").warning(
"CAS_RETRY[%d] succeeded pid=%d snap=%s crawl=%s url=%s",
attempt,
os.getpid(),
self.id,
crawl.id,
self.url,
)
crawl.urls = urls
return
import logging as _logging
_logging.getLogger("archivebox.crawls").error(
"Snapshot %s could not append its URL to crawl %s.urls: %s",
"CAS_RETRY exhausted pid=%d snap=%s crawl=%s url=%s",
os.getpid(),
self.id,
crawl.id,
self.url,

15
archivebox/workers/models.py
View File

@ -228,9 +228,21 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
current = type(self).objects.filter(pk=self.pk).values("modified_at", self.state_field_name).first()
current_modified_at = current.get("modified_at") if current else "<deleted>"
current_status = current.get(self.state_field_name) if current else "<deleted>"
# TEMP INSTRUMENTATION: capture caller stack so we can identify
# who's racing on the same row.
import traceback as _tb
caller_frame = "<unknown>"
try:
stack = _tb.extract_stack(limit=12)[:-1]
relevant = [f"{Path(s.filename).name}:{s.lineno}:{s.name}" for s in stack if "site-packages" not in s.filename][-6:]
caller_frame = " <- ".join(relevant)
except Exception:
pass
logger.error(
"\nSafeUpdateCASMiss: %s row %s was modified by another process while writing; "
"loaded modified_at=%s loaded %s=%s current modified_at=%s current %s=%s update_fields=%s extra_filter=%s\n",
"loaded modified_at=%s loaded %s=%s current modified_at=%s current %s=%s "
"update_fields=%s extra_filter=%s caller=%s\n",
type(self).__name__,
self.pk,
self.modified_at,
@ -241,6 +253,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
current_status,
sorted(values),
extra_filter or {},
caller_frame,
)
if refresh:
try:

2055
docs/Configuration.md
View File

File diff suppressed because it is too large Load Diff

34
docs/apidocs/archivebox/archivebox.config.configset.md
View File

@ -27,6 +27,18 @@
```
````
### Functions
````{list-table}
:class: autosummary longtable
:align: left
* - {py:obj}`_read_ini_config_cached <archivebox.config.configset._read_ini_config_cached>`
- ```{autodoc2-docstring} archivebox.config.configset._read_ini_config_cached
:summary:
```
````
### Data
````{list-table}
@ -37,6 +49,10 @@
- ```{autodoc2-docstring} archivebox.config.configset.COMPUTED_CONFIG_KEYS
:summary:
```
* - {py:obj}`_INI_CACHE <archivebox.config.configset._INI_CACHE>`
- ```{autodoc2-docstring} archivebox.config.configset._INI_CACHE
:summary:
```
````
### API
@ -66,6 +82,24 @@ Bases: {py:obj}`configparser.ConfigParser`
`````
````{py:data} _INI_CACHE
:canonical: archivebox.config.configset._INI_CACHE
:type: dict[tuple[str, float], dict[str, typing.Any]]
:value: >
None
```{autodoc2-docstring} archivebox.config.configset._INI_CACHE
```
````
````{py:function} _read_ini_config_cached(config_path_str: str) -> dict[str, typing.Any]
:canonical: archivebox.config.configset._read_ini_config_cached
```{autodoc2-docstring} archivebox.config.configset._read_ini_config_cached
```
````
`````{py:class} IniConfigSettingsSource(settings_cls: type[pydantic_settings.main.BaseSettings])
:canonical: archivebox.config.configset.IniConfigSettingsSource

92
docs/apidocs/archivebox/archivebox.core.templatetags.core_tags.md
View File

@ -91,8 +91,12 @@
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.result_list_tag
:summary:
```
* - {py:obj}`security_mode_banner <archivebox.core.templatetags.core_tags.security_mode_banner>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.security_mode_banner
* - {py:obj}`_machine_health_stats <archivebox.core.templatetags.core_tags._machine_health_stats>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._machine_health_stats
:summary:
```
* - {py:obj}`system_warnings_banner <archivebox.core.templatetags.core_tags.system_warnings_banner>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.system_warnings_banner
:summary:
```
* - {py:obj}`_unconfigured_banner_context <archivebox.core.templatetags.core_tags._unconfigured_banner_context>`
@ -187,6 +191,26 @@
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._MEDIA_FILE_EXTS
:summary:
```
* - {py:obj}`_LOW_DISK_THRESHOLD_GB <archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB
:summary:
```
* - {py:obj}`_HIGH_MEMORY_THRESHOLD_PCT <archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT
:summary:
```
* - {py:obj}`_HIGH_LOAD_MULTIPLE <archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE
:summary:
```
* - {py:obj}`_HEALTH_CHECK_INTERVAL_SECONDS <archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS
:summary:
```
* - {py:obj}`_health_cache <archivebox.core.templatetags.core_tags._health_cache>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags._health_cache
:summary:
```
````
### API
@ -384,10 +408,68 @@
```
````
````{py:function} security_mode_banner(context)
:canonical: archivebox.core.templatetags.core_tags.security_mode_banner
````{py:data} _LOW_DISK_THRESHOLD_GB
:canonical: archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB
:value: >
1.0
```{autodoc2-docstring} archivebox.core.templatetags.core_tags.security_mode_banner
```{autodoc2-docstring} archivebox.core.templatetags.core_tags._LOW_DISK_THRESHOLD_GB
```
````
````{py:data} _HIGH_MEMORY_THRESHOLD_PCT
:canonical: archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT
:value: >
95.0
```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_MEMORY_THRESHOLD_PCT
```
````
````{py:data} _HIGH_LOAD_MULTIPLE
:canonical: archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE
:value: >
3
```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HIGH_LOAD_MULTIPLE
```
````
````{py:data} _HEALTH_CHECK_INTERVAL_SECONDS
:canonical: archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS
:value: >
30
```{autodoc2-docstring} archivebox.core.templatetags.core_tags._HEALTH_CHECK_INTERVAL_SECONDS
```
````
````{py:data} _health_cache
:canonical: archivebox.core.templatetags.core_tags._health_cache
:type: dict
:value: >
None
```{autodoc2-docstring} archivebox.core.templatetags.core_tags._health_cache
```
````
````{py:function} _machine_health_stats() -> dict
:canonical: archivebox.core.templatetags.core_tags._machine_health_stats
```{autodoc2-docstring} archivebox.core.templatetags.core_tags._machine_health_stats
```
````
````{py:function} system_warnings_banner(context)
:canonical: archivebox.core.templatetags.core_tags.system_warnings_banner
```{autodoc2-docstring} archivebox.core.templatetags.core_tags.system_warnings_banner
```
````

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.33rc52",
"version": "0.9.33rc53",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

6
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.33rc52"
version = "0.9.33rc53"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -80,8 +80,8 @@ dependencies = [
### Binary/Package Management
"abxbus==2.5.8", # EventBus API
"abxpkg>=1.11.81", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.87", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.87", # shared ArchiveBox downloader package with blocking install preflight
"abx-plugins>=1.11.88", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.88", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]