mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-13 18:46:17 +05:00
release: archivebox 0.9.33rc23
This commit is contained in:
parent
145168621c
commit
57ce660db3
@ -293,11 +293,13 @@ def run_plugins(
|
||||
rprint("[red]No snapshots to process[/red]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
|
||||
if show_progress:
|
||||
rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
|
||||
|
||||
# Run orchestrator if --wait (default)
|
||||
if wait:
|
||||
rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
|
||||
if show_progress:
|
||||
rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
|
||||
snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
|
||||
for snapshot_id, crawl_id in existing_snapshots:
|
||||
snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
|
||||
|
||||
@ -180,7 +180,7 @@ def update(
|
||||
before: float | None = None,
|
||||
after: float | None = None,
|
||||
resume: str | None = None,
|
||||
batch_size: int = 100,
|
||||
batch_size: int = 500,
|
||||
continuous: bool = False,
|
||||
index_only: bool = False,
|
||||
migrate_only: bool = False,
|
||||
@ -403,7 +403,7 @@ def update(
|
||||
resume_cmd.append("--migrate-only")
|
||||
if index_only:
|
||||
resume_cmd.append("--index-only")
|
||||
if batch_size != 100:
|
||||
if batch_size != 500:
|
||||
resume_cmd.extend(["--batch-size", str(batch_size)])
|
||||
if exact_resume or resume:
|
||||
resume_cmd.extend(["--resume", str(exact_resume or resume)])
|
||||
@ -446,7 +446,7 @@ def update(
|
||||
stop_own_supervisord_process()
|
||||
|
||||
|
||||
def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100) -> dict[str, int]:
|
||||
def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500) -> dict[str, int]:
|
||||
"""
|
||||
Drain old archive/ directories (0.8.x → 0.9.x migration).
|
||||
|
||||
@ -605,7 +605,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
|
||||
return stats
|
||||
|
||||
|
||||
def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
|
||||
def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
|
||||
"""
|
||||
O(n) scan over entire DB from most recent to least recent.
|
||||
|
||||
@ -880,7 +880,7 @@ def print_index_stats(stats: dict[str, Any]) -> None:
|
||||
@click.option("--before", type=float, help="Only snapshots before timestamp")
|
||||
@click.option("--after", type=float, help="Only snapshots after timestamp")
|
||||
@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
|
||||
@click.option("--batch-size", type=int, default=100, help="Commit every N snapshots")
|
||||
@click.option("--batch-size", type=int, default=500, help="Commit every N records")
|
||||
@click.option("--continuous", is_flag=True, help="Run continuously as background worker")
|
||||
@click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
|
||||
@click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
|
||||
|
||||
@ -100,8 +100,9 @@ def foreground_shutdown_signals(
|
||||
previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
|
||||
|
||||
def raise_keyboard_interrupt(signum, _frame):
|
||||
state.signal_name = signal.Signals(signum).name
|
||||
os.write(sys.stdout.fileno(), f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n".encode())
|
||||
if state.signal_name is None:
|
||||
state.signal_name = signal.Signals(signum).name
|
||||
os.write(sys.stdout.fileno(), f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n".encode())
|
||||
raise KeyboardInterrupt
|
||||
|
||||
try:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.33rc22",
|
||||
"version": "0.9.33rc23",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.33rc22"
|
||||
version = "0.9.33rc23"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@ -79,9 +79,9 @@ dependencies = [
|
||||
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.7", # EventBus API
|
||||
"abxpkg>=1.11.48", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.54", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.54", # shared ArchiveBox downloader package with blocking install preflight
|
||||
"abxpkg>=1.11.49", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.55", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.55", # shared ArchiveBox downloader package with blocking install preflight
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user