mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
release: v0.9.32rc24
This commit is contained in:
Nick Sweeting
parent
066f79a029
commit
89a3119f2a
@ -47,7 +47,7 @@ class CrawlSchema(Schema):
|
||||
|
||||
@staticmethod
|
||||
def resolve_crawl_max_concurrent_snapshots(obj):
|
||||
return int((obj.config or {}).get("CRAWL_MAX_CONCURRENT_SNAPSHOTS") or get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS)
|
||||
return int(get_config(crawl=obj).CRAWL_MAX_CONCURRENT_SNAPSHOTS)
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
@ -119,15 +119,13 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
|
||||
raise HttpError(400, "crawl_max_size must be >= 0")
|
||||
if data.snapshot_max_size < 0:
|
||||
raise HttpError(400, "snapshot_max_size must be >= 0")
|
||||
crawl_max_concurrent_snapshots = data.crawl_max_concurrent_snapshots
|
||||
if crawl_max_concurrent_snapshots is None:
|
||||
crawl_max_concurrent_snapshots = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
|
||||
if crawl_max_concurrent_snapshots < 1:
|
||||
if data.crawl_max_concurrent_snapshots is not None and data.crawl_max_concurrent_snapshots < 1:
|
||||
raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
|
||||
|
||||
tags = normalize_tag_list(data.tags, data.tags_str)
|
||||
config = dict(data.config or {})
|
||||
config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
|
||||
if data.crawl_max_concurrent_snapshots is not None:
|
||||
config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = data.crawl_max_concurrent_snapshots
|
||||
crawl = Crawl.objects.create(
|
||||
urls="\n".join(urls),
|
||||
max_depth=data.max_depth,
|
||||
|
||||
@ -85,6 +85,7 @@ def add(
|
||||
crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
|
||||
snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
|
||||
config = get_config()
|
||||
crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
|
||||
if crawl_max_concurrent_snapshots is None:
|
||||
crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
|
||||
crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
|
||||
@ -141,9 +142,26 @@ def add(
|
||||
# Read URLs directly into crawl
|
||||
urls_content = sources_file.read_text()
|
||||
persona_name = (persona or "Default").strip() or "Default"
|
||||
plugins = plugins or str(config.get("PLUGINS") or "")
|
||||
plugins = plugins or ""
|
||||
persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
|
||||
persona_obj.ensure_dirs()
|
||||
effective_persona_config = get_config(persona=persona_obj)
|
||||
|
||||
crawl_config = {
|
||||
**({"ONLY_NEW": not update} if not update else {}),
|
||||
**({"INDEX_ONLY": True} if index_only else {}),
|
||||
**({"OVERWRITE": True} if overwrite else {}),
|
||||
**({"PLUGINS": plugins} if plugins else {}),
|
||||
**(
|
||||
{"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
|
||||
if crawl_max_concurrent_snapshots_override
|
||||
and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
|
||||
else {}
|
||||
),
|
||||
**({"PARSER": parser} if parser != "auto" else {}),
|
||||
**({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
|
||||
**({"URL_DENYLIST": url_denylist} if url_denylist else {}),
|
||||
}
|
||||
|
||||
crawl = Crawl.objects.create(
|
||||
urls=urls_content,
|
||||
@ -157,17 +175,7 @@ def add(
|
||||
created_by_id=created_by_id,
|
||||
status=Crawl.StatusChoices.QUEUED if bg or index_only else Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now() if bg else None,
|
||||
config={
|
||||
"ONLY_NEW": not update,
|
||||
"INDEX_ONLY": index_only,
|
||||
"OVERWRITE": overwrite,
|
||||
"PLUGINS": plugins,
|
||||
"DEFAULT_PERSONA": persona_name,
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
|
||||
"PARSER": parser,
|
||||
**({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
|
||||
**({"URL_DENYLIST": url_denylist} if url_denylist else {}),
|
||||
},
|
||||
config=crawl_config,
|
||||
)
|
||||
|
||||
print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
|
||||
|
||||
@ -1097,7 +1097,7 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
if not isinstance(plugin_config, dict):
|
||||
plugin_config = {}
|
||||
custom_config = self._get_custom_config_overrides(form)
|
||||
persona_name = persona.name if persona else "Default"
|
||||
custom_config.pop("DEFAULT_PERSONA", None)
|
||||
if persona:
|
||||
persona.ensure_dirs()
|
||||
|
||||
@ -1121,19 +1121,20 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
# 2. create a new Crawl with the URLs from the file
|
||||
timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
|
||||
urls_content = sources_file.read_text()
|
||||
# Build complete config
|
||||
config = {
|
||||
"INDEX_ONLY": index_only,
|
||||
"DEPTH": depth,
|
||||
"PLUGINS": plugins or "",
|
||||
"DEFAULT_PERSONA": persona_name,
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
|
||||
}
|
||||
# Store only explicit crawl-scoped overrides. Persona/machine/plugin
|
||||
# defaults are resolved at hook runtime via get_config(...).
|
||||
config = {}
|
||||
if index_only:
|
||||
config["INDEX_ONLY"] = True
|
||||
if plugins:
|
||||
config["PLUGINS"] = plugins
|
||||
effective_config = get_config(persona=persona) if persona else get_config()
|
||||
if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
|
||||
config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
|
||||
|
||||
# Merge custom config overrides
|
||||
config.update(plugin_config)
|
||||
config.update(custom_config)
|
||||
config["DEFAULT_PERSONA"] = persona_name
|
||||
if url_filters.get("allowlist"):
|
||||
config["URL_ALLOWLIST"] = url_filters["allowlist"]
|
||||
if url_filters.get("denylist"):
|
||||
@ -1527,14 +1528,22 @@ def live_progress_view(request):
|
||||
key=lambda crawl: crawl.modified_at,
|
||||
reverse=True,
|
||||
)[:10]
|
||||
persona_names_by_id: dict[str, str] = {}
|
||||
persona_details_by_id: dict[str, dict[str, str]] = {}
|
||||
persona_details_by_name: dict[str, dict[str, str]] = {}
|
||||
persona_ids = {crawl.persona_id for crawl in active_crawls_list if crawl.persona_id}
|
||||
if persona_ids:
|
||||
persona_names = {
|
||||
str((crawl.config or {}).get("DEFAULT_PERSONA") or "Default") for crawl in active_crawls_list if not crawl.persona_id
|
||||
}
|
||||
if persona_ids or persona_names:
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
persona_names_by_id = {
|
||||
str(persona_id): name for persona_id, name in Persona.objects.filter(id__in=persona_ids).values_list("id", "name")
|
||||
}
|
||||
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name"):
|
||||
persona_details = {
|
||||
"name": persona.name,
|
||||
"admin_url": f"/admin/personas/persona/{persona.pk}/change/",
|
||||
}
|
||||
persona_details_by_id[str(persona.id)] = persona_details
|
||||
persona_details_by_name[persona.name] = persona_details
|
||||
active_crawl_ids = [crawl.id for crawl in active_crawls_list]
|
||||
snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
|
||||
cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
@ -1929,8 +1938,9 @@ def live_progress_view(request):
|
||||
can_start = bool(crawl.urls)
|
||||
urls_preview = crawl.urls[:60] if crawl.urls else None
|
||||
crawl_tags = [tag.strip() for tag in (crawl.tags_str or "").replace("\n", ",").split(",") if tag.strip()]
|
||||
persona_name = persona_names_by_id.get(str(crawl.persona_id)) if crawl.persona_id else None
|
||||
persona_name = persona_name or str((crawl.config or {}).get("DEFAULT_PERSONA") or "Default")
|
||||
persona_details = persona_details_by_id.get(str(crawl.persona_id)) if crawl.persona_id else None
|
||||
persona_name = persona_details["name"] if persona_details else str((crawl.config or {}).get("DEFAULT_PERSONA") or "Default")
|
||||
persona_details = persona_details or persona_details_by_name.get(persona_name)
|
||||
crawl_output_size = crawl_output_sizes_by_crawl.get(str(crawl.id), 0)
|
||||
avg_snapshot_size = int(crawl_output_size / total_snapshots) if total_snapshots else 0
|
||||
|
||||
@ -1958,6 +1968,7 @@ def live_progress_view(request):
|
||||
"progress": crawl_progress,
|
||||
"created_by": crawl.created_by.username,
|
||||
"persona": persona_name,
|
||||
"persona_admin_url": persona_details["admin_url"] if persona_details else None,
|
||||
"max_depth": crawl.max_depth,
|
||||
"max_urls": crawl.max_urls,
|
||||
"max_crawl_size": crawl.crawl_max_size,
|
||||
|
||||
@ -6,6 +6,7 @@ from io import StringIO
|
||||
import uuid
|
||||
import json
|
||||
import re
|
||||
from itertools import islice
|
||||
from datetime import timedelta
|
||||
from archivebox.uuid_compat import uuid7
|
||||
from pathlib import Path
|
||||
@ -180,6 +181,13 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
return f"[...{short_id}] {first_url[:120]}"
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
update_fields = kwargs.get("update_fields")
|
||||
sync_tags = update_fields is None or "tags_str" in update_fields
|
||||
previous_tag_names = set()
|
||||
if sync_tags and self.pk:
|
||||
previous_tags_str = type(self).objects.filter(pk=self.pk).values_list("tags_str", flat=True).first()
|
||||
previous_tag_names = set(self.parse_tag_names(previous_tags_str or ""))
|
||||
|
||||
config = dict(self.config or {})
|
||||
if self.max_urls > 0:
|
||||
config["CRAWL_MAX_URLS"] = self.max_urls
|
||||
@ -210,6 +218,20 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
|
||||
|
||||
super().save(*args, **kwargs)
|
||||
if sync_tags:
|
||||
next_tag_names = set(self.parse_tag_names(self.tags_str or ""))
|
||||
added_tag_names = next_tag_names - previous_tag_names
|
||||
removed_tag_names = previous_tag_names - next_tag_names
|
||||
if added_tag_names or removed_tag_names:
|
||||
# Keep the SQLite write phase short: the Crawl row is already
|
||||
# saved, and the potentially large snapshot tag fanout runs in
|
||||
# chunked ORM writes after any caller atomic() exits.
|
||||
transaction.on_commit(
|
||||
lambda: self.apply_snapshot_tag_diff(
|
||||
added_tag_names=added_tag_names,
|
||||
removed_tag_names=removed_tag_names,
|
||||
),
|
||||
)
|
||||
# if is_new:
|
||||
# from archivebox.misc.logging_util import log_worker_event
|
||||
# first_url = self.get_urls_list()[0] if self.get_urls_list() else ''
|
||||
@ -229,6 +251,68 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
def api_url(self) -> str:
|
||||
return str(reverse_lazy("api-1:get_crawl", args=[self.id]))
|
||||
|
||||
@staticmethod
|
||||
def parse_tag_names(tags: Iterable[str] | str, *, pattern: str = r",") -> list[str]:
|
||||
raw_tags = re.split(pattern, tags) if isinstance(tags, str) else tags
|
||||
tag_names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw_tag in raw_tags:
|
||||
tag_name = str(raw_tag or "").strip()
|
||||
if not tag_name:
|
||||
continue
|
||||
lowered = tag_name.lower()
|
||||
if lowered in seen:
|
||||
continue
|
||||
seen.add(lowered)
|
||||
tag_names.append(tag_name)
|
||||
return tag_names
|
||||
|
||||
def current_tag_names(self) -> list[str]:
|
||||
current_tags_str = type(self).objects.filter(pk=self.pk).values_list("tags_str", flat=True).first() if self.pk else self.tags_str
|
||||
if current_tags_str is not None:
|
||||
self.tags_str = current_tags_str
|
||||
return self.parse_tag_names(self.tags_str or "")
|
||||
|
||||
def apply_snapshot_tag_diff(self, *, added_tag_names: Iterable[str], removed_tag_names: Iterable[str]) -> None:
|
||||
from archivebox.core.models import Snapshot, SnapshotTag, Tag
|
||||
|
||||
added_names = self.parse_tag_names(added_tag_names)
|
||||
removed_names = self.parse_tag_names(removed_tag_names)
|
||||
if not added_names and not removed_names:
|
||||
return
|
||||
|
||||
if added_names:
|
||||
tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
|
||||
missing_tags = [Tag(name=name) for name in added_names if name not in tags_by_name]
|
||||
if missing_tags:
|
||||
# One small write for missing tag rows, followed by chunked
|
||||
# M2M fanout below; avoid per-snapshot get_or_create loops.
|
||||
Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
|
||||
tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
|
||||
|
||||
tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
|
||||
snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
|
||||
while True:
|
||||
batch_snapshot_ids = list(islice(snapshot_ids, 5000))
|
||||
if not batch_snapshot_ids:
|
||||
break
|
||||
for tag_id in tag_ids:
|
||||
# Chunked bulk_create keeps memory bounded and uses the
|
||||
# SnapshotTag uniqueness constraint instead of row-by-row
|
||||
# existence checks.
|
||||
SnapshotTag.objects.bulk_create(
|
||||
[SnapshotTag(snapshot_id=snapshot_id, tag_id=tag_id) for snapshot_id in batch_snapshot_ids],
|
||||
ignore_conflicts=True,
|
||||
batch_size=5000,
|
||||
)
|
||||
|
||||
if removed_names:
|
||||
removed_tag_ids = list(Tag.objects.filter(name__in=removed_names).values_list("pk", flat=True))
|
||||
if removed_tag_ids:
|
||||
# One DELETE with a subquery keeps the tag removal transaction
|
||||
# bounded to the M2M rows touched by this crawl only.
|
||||
SnapshotTag.objects.filter(snapshot__crawl=self, tag_id__in=removed_tag_ids).delete()
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""
|
||||
Convert Crawl model instance to a JSON-serializable dict.
|
||||
@ -653,13 +737,15 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
Returns:
|
||||
List of newly created Snapshot objects
|
||||
"""
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.core.models import Snapshot, Tag
|
||||
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
|
||||
|
||||
if self.status == self.StatusChoices.SEALED:
|
||||
return []
|
||||
|
||||
created_snapshots = []
|
||||
crawl_tag_names = self.current_tag_names()
|
||||
tags_by_name: dict[str, Tag] = {}
|
||||
|
||||
for line in self.urls.splitlines():
|
||||
if not line.strip():
|
||||
@ -673,14 +759,14 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
depth = entry.get("depth", 0)
|
||||
title = entry.get("title")
|
||||
timestamp = entry.get("timestamp")
|
||||
tags = entry.get("tags", "")
|
||||
tag_names = [*crawl_tag_names, *self.parse_tag_names(entry.get("tags", ""))]
|
||||
except json.JSONDecodeError:
|
||||
snapshot_id = None
|
||||
url = sanitize_extracted_url(fix_url_from_markdown(line.strip()))
|
||||
depth = 0
|
||||
title = None
|
||||
timestamp = None
|
||||
tags = self.tags_str
|
||||
tag_names = crawl_tag_names
|
||||
|
||||
if not url:
|
||||
continue
|
||||
@ -747,8 +833,17 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
|
||||
if created:
|
||||
created_snapshots.append(snapshot)
|
||||
if tags:
|
||||
snapshot.save_tags(tags.split(","))
|
||||
if tag_names:
|
||||
missing_names = [tag_name for tag_name in tag_names if tag_name not in tags_by_name]
|
||||
if missing_names:
|
||||
tags_by_name.update({tag.name: tag for tag in Tag.objects.filter(name__in=missing_names)})
|
||||
missing_tags = [Tag(name=tag_name) for tag_name in missing_names if tag_name not in tags_by_name]
|
||||
if missing_tags:
|
||||
# Create tag rows in bulk, then attach through the M2M
|
||||
# relation without clearing any non-crawl snapshot tags.
|
||||
Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
|
||||
tags_by_name.update({tag.name: tag for tag in Tag.objects.filter(name__in=missing_names)})
|
||||
snapshot.tags.add(*[tag.pk for tag_name in tag_names if (tag := tags_by_name.get(tag_name))])
|
||||
|
||||
# Symlink creation touches the filesystem and can be slow on remote disks.
|
||||
# Defer it until after any active DB transaction commits so SQLite does
|
||||
@ -797,6 +892,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
return []
|
||||
|
||||
config = get_config(crawl=self, snapshot=parent_snapshot)
|
||||
crawl_tag_names = self.current_tag_names()
|
||||
allowlist = self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
|
||||
denylist = self.split_filter_patterns(config.get("URL_DENYLIST", ""))
|
||||
protected_subdomains = {"admin", "web", "api", "public"}
|
||||
@ -894,9 +990,12 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
|
||||
|
||||
tag_names_by_url: dict[str, set[str]] = {}
|
||||
for snapshot in created_snapshots:
|
||||
tags = str(deduped_records[snapshot.url].get("tags") or "").strip()
|
||||
if tags:
|
||||
tag_names_by_url[snapshot.url] = {tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags) if tag.strip()}
|
||||
tag_names = {
|
||||
*crawl_tag_names,
|
||||
*self.parse_tag_names(str(deduped_records[snapshot.url].get("tags") or ""), pattern=config.TAG_SEPARATOR_PATTERN),
|
||||
}
|
||||
if tag_names:
|
||||
tag_names_by_url[snapshot.url] = tag_names
|
||||
# Same transaction rule as create_snapshots_from_urls(): bulk_create()
|
||||
# only writes DB rows; symlink creation waits until after commit.
|
||||
transaction.on_commit(lambda snapshot=snapshot: snapshot.ensure_crawl_symlink())
|
||||
|
||||
@ -561,8 +561,7 @@ class CrawlRunner:
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
|
||||
config = get_config(crawl=self.crawl, snapshot=snapshot, include_machine=False)
|
||||
config.update(self.base_config)
|
||||
config = get_config(crawl=snapshot.crawl, snapshot=snapshot, include_machine=False)
|
||||
config["CRAWL_DIR"] = self.crawl_output_dir
|
||||
config["SNAP_DIR"] = str(snapshot.output_dir)
|
||||
extra_context: dict[str, Any] = {}
|
||||
|
||||
@ -244,10 +244,30 @@
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor a.crawl-badge {
|
||||
cursor: pointer;
|
||||
text-decoration: none !important;
|
||||
}
|
||||
#progress-monitor a.crawl-badge:hover {
|
||||
border-color: currentColor;
|
||||
filter: brightness(1.14);
|
||||
}
|
||||
#progress-monitor .crawl-badge strong {
|
||||
color: #8b949e;
|
||||
font-weight: 600;
|
||||
}
|
||||
#progress-monitor .crawl-title-link {
|
||||
display: inline-block;
|
||||
max-width: 100%;
|
||||
color: inherit;
|
||||
text-decoration: none !important;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
#progress-monitor .crawl-title-link:hover {
|
||||
color: #79c0ff;
|
||||
}
|
||||
#progress-monitor .crawl-badge.persona {
|
||||
color: #d2a8ff;
|
||||
background: rgba(163, 113, 247, 0.16);
|
||||
@ -560,6 +580,21 @@
|
||||
background: #21262d;
|
||||
color: #6e7681;
|
||||
}
|
||||
#progress-monitor .duration-badge {
|
||||
flex-shrink: 0;
|
||||
min-width: 28px;
|
||||
padding: 2px 6px;
|
||||
border-radius: 10px;
|
||||
background: rgba(88, 166, 255, 0.12);
|
||||
color: #a5d6ff;
|
||||
border: 1px solid rgba(88, 166, 255, 0.2);
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
line-height: 1.2;
|
||||
text-align: center;
|
||||
font-variant-numeric: tabular-nums;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* Thumbnail Strip */
|
||||
#progress-monitor .thumbnail-strip {
|
||||
@ -749,7 +784,7 @@
|
||||
return headers;
|
||||
}
|
||||
function escapeHtml(value) {
|
||||
return String(value || '').replace(/[&<>"']/g, char => ({
|
||||
return String(value ?? '').replace(/[&<>"']/g, char => ({
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
@ -802,6 +837,35 @@
|
||||
function renderThumbnail(thumb, isNew) { return null; }
|
||||
function updateThumbnails(thumbnails) {}
|
||||
|
||||
function formatDuration(seconds) {
|
||||
seconds = Math.max(0, Math.floor(seconds || 0));
|
||||
const hours = Math.floor(seconds / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
const secs = seconds % 60;
|
||||
if (hours > 0) return `${hours}hr ${minutes}m ${secs}s`;
|
||||
if (minutes > 0) return `${minutes}m ${secs}s`;
|
||||
return `${secs}s`;
|
||||
}
|
||||
|
||||
function durationText(startedAt) {
|
||||
const startedMs = Date.parse(startedAt || '');
|
||||
if (!Number.isFinite(startedMs)) return '';
|
||||
return formatDuration((Date.now() - startedMs) / 1000);
|
||||
}
|
||||
|
||||
function renderDurationBadge(startedAt) {
|
||||
const text = durationText(startedAt);
|
||||
if (!text) return '';
|
||||
return `<span class="duration-badge" data-started-at="${escapeAttr(startedAt)}" title="Duration">${escapeHtml(text)}</span>`;
|
||||
}
|
||||
|
||||
function updateDurationBadges() {
|
||||
document.querySelectorAll('#progress-monitor .duration-badge[data-started-at]').forEach((badge) => {
|
||||
const text = durationText(badge.dataset.startedAt);
|
||||
if (text) badge.textContent = text;
|
||||
});
|
||||
}
|
||||
|
||||
function renderExtractor(extractor) {
|
||||
const icon = extractor.status === 'started' ? '▶' :
|
||||
extractor.status === 'succeeded' ? '✓' :
|
||||
@ -841,6 +905,7 @@
|
||||
? `<button class="cancel-item-btn" data-cancel-type="snapshot" data-snapshot-id="${snapshot.id}" data-label="✕" title="Cancel snapshot">✕</button>`
|
||||
: '';
|
||||
const snapshotPidHtml = snapshot.worker_pid ? `<span class="pid-label compact">pid ${snapshot.worker_pid}</span>` : '';
|
||||
const snapshotDurationHtml = renderDurationBadge(snapshot.started);
|
||||
const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
|
||||
const urlText = snapshot.full_url || snapshot.url || '';
|
||||
const faviconHtml = snapshot.favicon_url
|
||||
@ -902,6 +967,7 @@
|
||||
</div>
|
||||
</div>
|
||||
${snapshotPidHtml}
|
||||
${snapshotDurationHtml}
|
||||
<span class="status-badge ${snapshot.status || 'unknown'}">${snapshot.status || 'unknown'}</span>
|
||||
</a>
|
||||
${cancelBtn}
|
||||
@ -919,6 +985,13 @@
|
||||
|
||||
function renderCrawl(crawl) {
|
||||
const adminUrl = `/admin/crawls/crawl/${crawl.id || 'unknown'}/change/`;
|
||||
const adminFieldUrl = (fieldName) => `${adminUrl}#id_${fieldName}`;
|
||||
const crawlBadge = (className, label, value, href, title) => {
|
||||
const tag = href ? 'a' : 'span';
|
||||
const hrefAttr = href ? ` href="${escapeAttr(href)}"` : '';
|
||||
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
|
||||
return `<${tag} class="crawl-badge ${className}"${hrefAttr}${titleAttr}><strong>${escapeHtml(label)}</strong>${escapeHtml(value)}</${tag}>`;
|
||||
};
|
||||
const crawlId = (crawl.id || 'unknown').toString();
|
||||
const crawlShortId = crawlId === 'unknown' ? 'unknown' : crawlId.slice(-8);
|
||||
const startedDate = crawl.started ? crawl.started.slice(0, 10) : 'unknown date';
|
||||
@ -927,6 +1000,7 @@
|
||||
? `<button class="cancel-item-btn" data-cancel-type="crawl" data-crawl-id="${crawl.id}" data-label="✕" title="Cancel crawl">✕</button>`
|
||||
: '';
|
||||
const crawlPidHtml = crawl.worker_pid ? `<span class="pid-label compact">pid ${crawl.worker_pid}</span>` : '';
|
||||
const crawlDurationHtml = renderDurationBadge(crawl.started);
|
||||
|
||||
let snapshotsHtml = '';
|
||||
if (crawl.active_snapshots && crawl.active_snapshots.length > 0) {
|
||||
@ -1005,12 +1079,12 @@
|
||||
const crawlSizeLimitText = `${crawl.crawl_output_size_display || '0 B'} / ${crawl.max_crawl_size_display || 'unlimited'}`;
|
||||
const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`;
|
||||
const crawlBadges = [
|
||||
`<span class="crawl-badge persona"><strong>persona</strong>${escapeHtml(crawl.persona || 'Default')}</span>`,
|
||||
`<span class="crawl-badge limit"><strong>depth</strong>${crawl.max_depth || 0}</span>`,
|
||||
`<span class="crawl-badge limit"><strong>urls</strong>${escapeHtml(urlLimitText)}</span>`,
|
||||
`<span class="crawl-badge size"><strong>crawl size</strong>${escapeHtml(crawlSizeLimitText)}</span>`,
|
||||
`<span class="crawl-badge size"><strong>avg snap</strong>${escapeHtml(snapshotSizeLimitText)}</span>`,
|
||||
...(crawl.tags || []).map(tag => `<span class="crawl-badge tag">#${escapeHtml(tag)}</span>`),
|
||||
crawlBadge('persona', 'Persona Config', crawl.persona || 'Default', crawl.persona_admin_url, 'Edit persona config'),
|
||||
crawlBadge('limit', 'depth', crawl.max_depth || 0, null, ''),
|
||||
crawlBadge('limit', 'urls', urlLimitText, adminFieldUrl('max_urls'), 'Edit max URLs'),
|
||||
crawlBadge('size', 'crawl size', crawlSizeLimitText, adminFieldUrl('crawl_max_size'), 'Edit max crawl size'),
|
||||
crawlBadge('size', 'avg snap', snapshotSizeLimitText, adminFieldUrl('snapshot_max_size'), 'Edit max snapshot size'),
|
||||
...(crawl.tags || []).map(tag => `<a class="crawl-badge tag" href="${escapeAttr(adminFieldUrl('tags_editor'))}" title="Edit crawl tags">#${escapeHtml(tag)}</a>`),
|
||||
].join('');
|
||||
const statsHtml = [
|
||||
`<span class="crawl-badge count"><strong>done</strong>${crawl.completed_snapshots || 0}</span>`,
|
||||
@ -1022,17 +1096,18 @@
|
||||
return `
|
||||
<div class="crawl-item" data-crawl-id="${crawl.id || 'unknown'}">
|
||||
<div class="crawl-header">
|
||||
<a class="crawl-header-link" href="${adminUrl}">
|
||||
<div class="crawl-header-link">
|
||||
<div class="crawl-info">
|
||||
<div class="crawl-label">Crawl #${escapeHtml(crawlShortId)} ${escapeHtml(startedDate)} started by ${escapeHtml(crawl.created_by || 'unknown')}</div>
|
||||
<a class="crawl-label crawl-title-link" href="${escapeAttr(adminUrl)}">Crawl #${escapeHtml(crawlShortId)} ${escapeHtml(startedDate)} started by ${escapeHtml(crawl.created_by || 'unknown')}</a>
|
||||
<div class="crawl-badges">${crawlBadges}</div>
|
||||
</div>
|
||||
<div class="crawl-stats">
|
||||
${statsHtml}
|
||||
</div>
|
||||
${crawlPidHtml}
|
||||
${crawlDurationHtml}
|
||||
<span class="status-badge ${crawl.status || 'unknown'}">${crawl.status || 'unknown'}</span>
|
||||
</a>
|
||||
</div>
|
||||
${cancelBtn}
|
||||
</div>
|
||||
<div class="crawl-progress">
|
||||
@ -1139,6 +1214,7 @@
|
||||
}
|
||||
|
||||
// Recent thumbnails removed
|
||||
updateDurationBadges();
|
||||
}
|
||||
|
||||
function fetchProgress() {
|
||||
@ -1277,6 +1353,7 @@
|
||||
|
||||
// Start polling when page loads
|
||||
startPolling();
|
||||
setInterval(updateDurationBadges, 1000);
|
||||
|
||||
// Pause polling when tab is hidden
|
||||
document.addEventListener('visibilitychange', function() {
|
||||
|
||||
@ -17,9 +17,13 @@ pytest_plugins = ["archivebox.tests.fixtures"]
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
|
||||
SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
|
||||
SESSION_DATA_DIR = Path(
|
||||
os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
|
||||
).resolve()
|
||||
# Force ArchiveBox imports to see a temp DATA_DIR during test collection.
|
||||
os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
|
||||
os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
|
||||
os.environ.pop("ARCHIVE_DIR", None)
|
||||
os.environ.pop("USERS_DIR", None)
|
||||
os.environ.pop("CRAWL_DIR", None)
|
||||
@ -129,10 +133,9 @@ def isolate_test_runtime(tmp_path, monkeypatch):
|
||||
contract is that every test starts in its own temp directory and any
|
||||
in-process ``os.environ`` edits are rolled back afterwards.
|
||||
|
||||
We intentionally clear ``DATA_DIR`` for the body of each test so subprocess
|
||||
tests that rely on cwd keep working. During collection/import time we still
|
||||
seed a separate session-scoped temp ``DATA_DIR`` above so any ArchiveBox
|
||||
config imported before this fixture runs never points at the repo root.
|
||||
Each in-process test gets an explicit temp ``DATA_DIR`` so ArchiveBox code
|
||||
never falls back to the repo cwd. Subprocess helpers that intentionally test
|
||||
cwd-based behavior remove ``DATA_DIR`` for the child process themselves.
|
||||
"""
|
||||
_assert_not_repo_path(tmp_path, label="tmp_path")
|
||||
original_cwd = Path.cwd()
|
||||
@ -140,11 +143,6 @@ def isolate_test_runtime(tmp_path, monkeypatch):
|
||||
original_chdir = os.chdir
|
||||
original_popen = subprocess.Popen
|
||||
os.chdir(tmp_path)
|
||||
os.environ.pop("DATA_DIR", None)
|
||||
os.environ.pop("ARCHIVE_DIR", None)
|
||||
os.environ.pop("USERS_DIR", None)
|
||||
os.environ.pop("CRAWL_DIR", None)
|
||||
os.environ.pop("SNAP_DIR", None)
|
||||
|
||||
def guarded_chdir(path: os.PathLike[str] | str) -> None:
|
||||
_assert_not_repo_path(Path(path), label="cwd")
|
||||
@ -160,6 +158,11 @@ def isolate_test_runtime(tmp_path, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(os, "chdir", guarded_chdir)
|
||||
monkeypatch.setattr(subprocess, "Popen", guarded_popen)
|
||||
os.environ["DATA_DIR"] = str(tmp_path)
|
||||
os.environ.pop("ARCHIVE_DIR", None)
|
||||
os.environ.pop("USERS_DIR", None)
|
||||
os.environ.pop("CRAWL_DIR", None)
|
||||
os.environ.pop("SNAP_DIR", None)
|
||||
try:
|
||||
_assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ)
|
||||
yield
|
||||
|
||||
@ -141,7 +141,6 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
|
||||
assert crawl.max_urls == 3
|
||||
assert crawl.crawl_max_size == 45 * 1024 * 1024
|
||||
assert crawl.snapshot_max_size == 5 * 1024 * 1024
|
||||
assert crawl.config.get("DEFAULT_PERSONA") == "Default"
|
||||
assert crawl.config["CRAWL_MAX_URLS"] == 3
|
||||
assert crawl.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024
|
||||
assert crawl.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
|
||||
@ -185,7 +184,7 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
|
||||
crawl = Crawl.objects.order_by("-created_at").first()
|
||||
assert crawl is not None
|
||||
assert crawl.persona_id == private_persona.id
|
||||
assert crawl.config.get("DEFAULT_PERSONA") == "Private"
|
||||
assert "DEFAULT_PERSONA" not in crawl.config
|
||||
assert crawl.resolve_persona() == private_persona
|
||||
runtime_config = get_config(crawl=crawl)
|
||||
assert runtime_config.ACTIVE_PERSONA == "Private"
|
||||
|
||||
@ -236,7 +236,7 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac
|
||||
conn.close()
|
||||
|
||||
assert persona_id
|
||||
assert default_persona == "Default"
|
||||
assert default_persona is None
|
||||
assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()
|
||||
|
||||
|
||||
|
||||
@ -105,6 +105,50 @@ def test_crawl_admin_form_saves_tags_editor_to_tags_str(crawl, admin_user):
|
||||
assert updated.config["URL_DENYLIST"] == "static.example.com"
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_crawl_tag_changes_sync_existing_snapshot_tags(crawl):
|
||||
snapshots = crawl.create_snapshots_from_urls()
|
||||
snapshots[0].save_tags(["alpha", "beta", "keep"])
|
||||
|
||||
crawl.tags_str = "beta,gamma"
|
||||
crawl.save(update_fields=["tags_str", "modified_at"])
|
||||
|
||||
assert set(snapshots[0].tags.values_list("name", flat=True)) == {"beta", "gamma", "keep"}
|
||||
assert set(snapshots[1].tags.values_list("name", flat=True)) == {"beta", "gamma"}
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_create_snapshots_from_urls_uses_current_crawl_tags_for_stale_crawl_instance(crawl):
|
||||
crawl.create_snapshots_from_urls()
|
||||
fresh_crawl = Crawl.objects.get(pk=crawl.pk)
|
||||
fresh_crawl.tags_str = "midcrawl"
|
||||
fresh_crawl.save(update_fields=["tags_str", "modified_at"])
|
||||
|
||||
crawl.urls = f"{crawl.urls}\nhttps://example.net/new"
|
||||
created = crawl.create_snapshots_from_urls()
|
||||
|
||||
assert [snapshot.url for snapshot in created] == ["https://example.net/new"]
|
||||
assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl"}
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_discovered_snapshots_inherit_current_crawl_tags(crawl):
|
||||
crawl.max_depth = 1
|
||||
crawl.save(update_fields=["max_depth", "modified_at"])
|
||||
parent_snapshot = crawl.create_snapshots_from_urls()[0]
|
||||
crawl.tags_str = "midcrawl"
|
||||
crawl.save(update_fields=["tags_str", "modified_at"])
|
||||
|
||||
created = crawl.create_discovered_snapshots(
|
||||
parent_snapshot,
|
||||
[{"url": "https://example.com/child", "tags": "discovered"}],
|
||||
depth=1,
|
||||
)
|
||||
|
||||
assert [snapshot.url for snapshot in created] == ["https://example.com/child"]
|
||||
assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl", "discovered"}
|
||||
|
||||
|
||||
def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, admin_user):
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com/remove-me",
|
||||
|
||||
@ -376,6 +376,85 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
|
||||
assert snapshot.retry_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot():
|
||||
from abx_dl.events import SnapshotCompletedEvent
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.personas.models import Persona
|
||||
from archivebox.services.runner import CrawlRunner
|
||||
|
||||
persona = Persona.objects.create(
|
||||
name="RuntimeConfig",
|
||||
config={
|
||||
"FAVICON_PROVIDER": "https://example.com/persona-first.ico",
|
||||
"FAVICON_TIMEOUT": 10,
|
||||
},
|
||||
)
|
||||
persona.ensure_dirs()
|
||||
crawl = Crawl.objects.create(
|
||||
urls="\n".join(
|
||||
[
|
||||
"https://www.python.org/",
|
||||
"https://www.djangoproject.com/",
|
||||
"https://www.wikipedia.org/",
|
||||
],
|
||||
),
|
||||
config={
|
||||
"PLUGINS": "favicon",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
|
||||
},
|
||||
persona_id=persona.id,
|
||||
created_by_id=get_or_create_system_user_pk(),
|
||||
)
|
||||
runner = CrawlRunner(crawl)
|
||||
completed_snapshot_ids: list[str] = []
|
||||
|
||||
async def update_config_between_snapshots(event: SnapshotCompletedEvent) -> None:
|
||||
if event.snapshot_id in completed_snapshot_ids:
|
||||
return
|
||||
completed_snapshot_ids.append(event.snapshot_id)
|
||||
if len(completed_snapshot_ids) == 1:
|
||||
persona.config = {
|
||||
**(persona.config or {}),
|
||||
"FAVICON_PROVIDER": "https://example.com/persona-second.ico",
|
||||
}
|
||||
await persona.asave(update_fields=["config"])
|
||||
elif len(completed_snapshot_ids) == 2:
|
||||
fresh_crawl = await Crawl.objects.aget(id=crawl.id)
|
||||
fresh_crawl.config = {
|
||||
**(fresh_crawl.config or {}),
|
||||
"FAVICON_PROVIDER": "https://example.com/crawl-third.ico",
|
||||
}
|
||||
await fresh_crawl.asave(update_fields=["config"])
|
||||
|
||||
runner.bus.on(SnapshotCompletedEvent, update_config_between_snapshots)
|
||||
|
||||
asyncio.run(runner.run())
|
||||
|
||||
favicon_processes = [
|
||||
process
|
||||
for process in Process.objects.filter(process_type=Process.TypeChoices.HOOK).order_by("started_at")
|
||||
if process.cmd and "on_Snapshot__11_favicon.finite.bg.py" in str(process.cmd[0])
|
||||
]
|
||||
providers = [process.env.get("FAVICON_PROVIDER") for process in favicon_processes]
|
||||
|
||||
crawl.refresh_from_db()
|
||||
assert crawl.status == Crawl.StatusChoices.SEALED
|
||||
assert Snapshot.objects.filter(crawl=crawl, status=Snapshot.StatusChoices.SEALED).count() == 3
|
||||
assert (
|
||||
ArchiveResult.objects.filter(snapshot__crawl=crawl, plugin="favicon").exclude(status=ArchiveResult.StatusChoices.FAILED).count()
|
||||
== 3
|
||||
)
|
||||
assert providers == [
|
||||
"https://example.com/persona-first.ico",
|
||||
"https://example.com/persona-second.ico",
|
||||
"https://example.com/crawl-third.ico",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_run_snapshot_seals_descendant_when_crawl_max_size_is_reached(tmp_path):
|
||||
from abx_dl.events import CrawlStartEvent, SnapshotEvent
|
||||
|
||||
@ -14,6 +14,21 @@ def test_session_data_dir_is_outside_repo_root():
|
||||
assert test_harness.PYTEST_BASETEMP_ROOT in (Path.cwd(), *Path.cwd().parents)
|
||||
|
||||
|
||||
def test_in_process_archivebox_config_uses_temp_data_dir():
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
data_dir = Path(os.environ["DATA_DIR"]).resolve()
|
||||
assert data_dir == Path.cwd().resolve()
|
||||
assert test_harness.REPO_ROOT not in data_dir.parents
|
||||
assert CONSTANTS.DATA_DIR != test_harness.REPO_ROOT
|
||||
|
||||
config = get_config(include_machine=False)
|
||||
assert config.DATA_DIR == data_dir
|
||||
assert config.ARCHIVE_DIR == data_dir / "archive"
|
||||
assert config.USERS_DIR == data_dir / "archive" / "users"
|
||||
|
||||
|
||||
def test_cli_helpers_reject_repo_root_runtime_paths():
|
||||
with pytest.raises(AssertionError, match="repo root"):
|
||||
test_harness.run_archivebox_cmd(["version"], data_dir=test_harness.REPO_ROOT)
|
||||
|
||||
32
conftest.py
Normal file
32
conftest.py
Normal file
@ -0,0 +1,32 @@
|
||||
"""Root pytest bootstrap.
|
||||
|
||||
This file is intentionally outside the ``archivebox`` package so pytest can
|
||||
load it before importing ``archivebox/__init__.py``. ArchiveBox constants are
|
||||
computed at import time, so DATA_DIR must already point at a temp collection.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
SESSION_DATA_DIR = Path(
|
||||
os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
|
||||
).resolve()
|
||||
|
||||
os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
|
||||
os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
|
||||
os.environ.pop("ARCHIVE_DIR", None)
|
||||
os.environ.pop("USERS_DIR", None)
|
||||
os.environ.pop("CRAWL_DIR", None)
|
||||
os.environ.pop("SNAP_DIR", None)
|
||||
|
||||
|
||||
def pytest_configure():
|
||||
import django
|
||||
from django.apps import apps
|
||||
|
||||
if not apps.ready:
|
||||
django.setup()
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.32rc23",
|
||||
"version": "0.9.32rc24",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.32rc23"
|
||||
version = "0.9.32rc24"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@ -201,7 +201,6 @@ skip = "*.json,*.min.js,*.min.css,uv.lock,old/*,publicsite/*"
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = [ "archivebox/tests" ]
|
||||
norecursedirs = ["archivebox/tests/data"]
|
||||
DJANGO_SETTINGS_MODULE = "archivebox.core.settings"
|
||||
# Note: Plugin tests under abx_plugins/plugins/ must NOT load Django
|
||||
# They use a conftest.py to disable Django automatically
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user