diff --git a/.claude/settings.json b/.claude/settings.json
new file mode 100644
index 00000000..44279555
--- /dev/null
+++ b/.claude/settings.json
@@ -0,0 +1,5 @@
+{
+ "enabledPlugins": {
+ "pyright-lsp@claude-plugins-official": true
+ }
+}
diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py
index 6c296c46..b9273e31 100644
--- a/archivebox/cli/archivebox_server.py
+++ b/archivebox/cli/archivebox_server.py
@@ -30,10 +30,13 @@ def server(runserver_args: Iterable[str]=(SERVER_CONFIG.BIND_ADDR,),
from archivebox.misc.checks import check_data_folder
check_data_folder()
- from django.core.management import call_command
- from django.contrib.auth.models import User
-
from archivebox.config.common import SHELL_CONFIG
+
+ run_in_debug = SHELL_CONFIG.DEBUG or debug or reload
+ if debug or reload:
+ SHELL_CONFIG.DEBUG = True
+
+ from django.contrib.auth.models import User
if not User.objects.filter(is_superuser=True).exclude(username='system').exists():
print()
@@ -56,7 +59,8 @@ def server(runserver_args: Iterable[str]=(SERVER_CONFIG.BIND_ADDR,),
except IndexError:
pass
- if SHELL_CONFIG.DEBUG:
+ if run_in_debug:
+ from django.core.management import call_command
print('[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]')
print(f' [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]')
print(f' [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]')
diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py
index 4064d85e..b4c420b7 100644
--- a/archivebox/core/admin_archiveresults.py
+++ b/archivebox/core/admin_archiveresults.py
@@ -57,7 +57,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
# Build output link - use embed_path() which checks output_files first
embed_path = result.embed_path() if hasattr(result, 'embed_path') else None
- output_link = f'/archive/{result.snapshot.timestamp}/{embed_path}' if embed_path and result.status == 'succeeded' else f'/archive/{result.snapshot.timestamp}/'
+ output_link = f'/{result.snapshot.archive_path}/{embed_path}' if embed_path and result.status == 'succeeded' else f'/{result.snapshot.archive_path}/'
# Get version - try cmd_version field
version = result.cmd_version if result.cmd_version else '-'
@@ -83,8 +83,8 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
{icon}
-
@@ -301,8 +301,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
)
def snapshot_info(self, result):
return format_html(
- '[{}] {} {}
',
- result.snapshot.timestamp,
+ '[{}] {} {}
',
+ result.snapshot.archive_path,
str(result.snapshot.id)[:8],
result.snapshot.bookmarked_at.strftime('%Y-%m-%d %H:%M'),
result.snapshot.url[:128],
@@ -336,8 +336,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
embed_path = result.embed_path() if hasattr(result, 'embed_path') else None
output_path = embed_path if (result.status == 'succeeded' and embed_path) else 'index.html'
return format_html(
- '↗️{}',
- result.snapshot.timestamp,
+ '↗️{}',
+ result.snapshot.archive_path,
output_path,
result.output_str,
)
@@ -348,7 +348,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
'{}
',
result.output_str,
)
- output_html += format_html('See result files ...
', str(result.snapshot.timestamp))
+ output_html += format_html('See result files ...
', str(result.snapshot.archive_path))
embed_path = result.embed_path() if hasattr(result, 'embed_path') else ''
path_from_embed = (snapshot_dir / (embed_path or ''))
output_html += format_html('{}/{}
', str(snapshot_dir), str(embed_path))
diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py
index 8b3cd77a..d75198ff 100644
--- a/archivebox/core/admin_snapshots.py
+++ b/archivebox/core/admin_snapshots.py
@@ -237,13 +237,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
'''
📄 Summary Page
📁 Result Files
@@ -291,8 +291,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
Tip: Action buttons link to the list view with this snapshot pre-selected. Select it and use the action dropdown to execute.
''',
- obj.timestamp,
- obj.timestamp,
+ obj.archive_path,
+ obj.archive_path,
obj.url,
obj.pk,
obj.pk,
@@ -310,7 +310,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
'✅' if obj.is_archived else '❌',
obj.num_outputs,
self.size(obj) or '0kb',
- f'/archive/{obj.timestamp}/favicon.ico',
+ f'/{obj.archive_path}/favicon.ico',
obj.extension or '-',
)
diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py
index 1cbe540e..a5343196 100644
--- a/archivebox/core/middleware.py
+++ b/archivebox/core/middleware.py
@@ -1,6 +1,7 @@
__package__ = 'archivebox.core'
import ipaddress
+import re
from django.utils import timezone
from django.contrib.auth.middleware import RemoteUserMiddleware
from django.core.exceptions import ImproperlyConfigured
@@ -28,10 +29,11 @@ def TimezoneMiddleware(get_response):
def CacheControlMiddleware(get_response):
+ snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
def middleware(request):
response = get_response(request)
- if '/archive/' in request.path or '/static/' in request.path:
+ if '/archive/' in request.path or '/static/' in request.path or snapshot_path_re.match(request.path):
policy = 'public' if SERVER_CONFIG.PUBLIC_SNAPSHOTS else 'private'
response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
# print('Set Cache-Control header to', response['Cache-Control'])
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index bd943a29..4f48cf20 100755
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -1296,7 +1296,6 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
)}
path = self.archive_path
- canon = self.canonical_outputs()
output = ""
output_template = '{} '
@@ -1313,10 +1312,11 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
if not icon.strip() and not existing:
continue
+ embed_path = result.embed_path() if result else f'{plugin}/'
output += format_html(
output_template,
path,
- canon.get(plugin, plugin + '/'),
+ embed_path,
str(bool(existing)),
plugin,
icon
@@ -1402,9 +1402,38 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
return
@cached_property
- def archive_path(self):
+ def legacy_archive_path(self) -> str:
return f'{CONSTANTS.ARCHIVE_DIR_NAME}/{self.timestamp}'
+ @cached_property
+ def url_path(self) -> str:
+ """URL path matching the current snapshot output_dir layout."""
+ try:
+ rel_path = Path(self.output_dir).resolve().relative_to(CONSTANTS.DATA_DIR)
+ except Exception:
+ return self.legacy_archive_path
+
+ parts = rel_path.parts
+ # New layout: users/ /snapshots////
+ if len(parts) >= 6 and parts[0] == 'users' and parts[2] == 'snapshots':
+ username = parts[1]
+ if username == 'system':
+ username = 'web'
+ date_str = parts[3]
+ domain = parts[4]
+ snapshot_id = parts[5]
+ return f'{username}/{date_str}/{domain}/{snapshot_id}'
+
+ # Legacy layout: archive//
+ if len(parts) >= 2 and parts[0] == CONSTANTS.ARCHIVE_DIR_NAME:
+ return f'{parts[0]}/{parts[1]}'
+
+ return '/'.join(parts)
+
+ @cached_property
+ def archive_path(self):
+ return self.url_path
+
@cached_property
def archive_size(self):
try:
@@ -1467,8 +1496,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
for pid_file in Path(self.output_dir).glob('**/*.pid'):
pid_file.unlink(missing_ok=True)
- # Update all STARTED ArchiveResults from filesystem
- results = self.archiveresult_set.filter(status=ArchiveResult.StatusChoices.STARTED)
+ # Update all background ArchiveResults from filesystem (in case output arrived late)
+ results = self.archiveresult_set.filter(hook_name__contains='.bg.')
for ar in results:
ar.update_from_output()
@@ -1914,153 +1943,6 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
# Output Path Methods (migrated from Link schema)
# =========================================================================
- def canonical_outputs(self) -> Dict[str, Optional[str]]:
- """
- Intelligently discover the best output file for each plugin.
- Uses actual ArchiveResult data and filesystem scanning with smart heuristics.
- """
- FAVICON_PROVIDER = 'https://www.google.com/s2/favicons?domain={}'
-
- # Mimetypes that can be embedded/previewed in an iframe
- IFRAME_EMBEDDABLE_EXTENSIONS = {
- 'html', 'htm', 'pdf', 'txt', 'md', 'json', 'jsonl',
- 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg', 'ico',
- 'mp4', 'webm', 'mp3', 'opus', 'ogg', 'wav',
- }
-
- MIN_DISPLAY_SIZE = 15_000 # 15KB - filter out tiny files
- MAX_SCAN_FILES = 50 # Don't scan massive directories
-
- def find_best_output_in_dir(dir_path: Path, plugin_name: str) -> Optional[str]:
- """Find the best representative file in a plugin's output directory"""
- if not dir_path.exists() or not dir_path.is_dir():
- return None
-
- candidates = []
- file_count = 0
-
- # Special handling for media plugin - look for thumbnails
- is_media_dir = plugin_name == 'media'
-
- # Scan for suitable files
- for file_path in dir_path.rglob('*'):
- file_count += 1
- if file_count > MAX_SCAN_FILES:
- break
-
- if file_path.is_dir() or file_path.name.startswith('.'):
- continue
-
- ext = file_path.suffix.lstrip('.').lower()
- if ext not in IFRAME_EMBEDDABLE_EXTENSIONS:
- continue
-
- try:
- size = file_path.stat().st_size
- except OSError:
- continue
-
- # For media dir, allow smaller image files (thumbnails are often < 15KB)
- min_size = 5_000 if (is_media_dir and ext in ('png', 'jpg', 'jpeg', 'webp', 'gif')) else MIN_DISPLAY_SIZE
- if size < min_size:
- continue
-
- # Prefer main files: index.html, output.*, content.*, etc.
- priority = 0
- name_lower = file_path.name.lower()
-
- if is_media_dir:
- # Special prioritization for media directories
- if any(keyword in name_lower for keyword in ('thumb', 'thumbnail', 'cover', 'poster')):
- priority = 200 # Highest priority for thumbnails
- elif ext in ('png', 'jpg', 'jpeg', 'webp', 'gif'):
- priority = 150 # High priority for any image
- elif ext in ('mp4', 'webm', 'mp3', 'opus', 'ogg'):
- priority = 100 # Lower priority for actual media files
- else:
- priority = 50
- elif 'index' in name_lower:
- priority = 100
- elif name_lower.startswith(('output', 'content', plugin_name)):
- priority = 50
- elif ext in ('html', 'htm', 'pdf'):
- priority = 30
- elif ext in ('png', 'jpg', 'jpeg', 'webp'):
- priority = 20
- else:
- priority = 10
-
- candidates.append((priority, size, file_path))
-
- if not candidates:
- return None
-
- # Sort by priority (desc), then size (desc)
- candidates.sort(key=lambda x: (x[0], x[1]), reverse=True)
- best_file = candidates[0][2]
- return str(best_file.relative_to(Path(self.output_dir)))
-
- canonical = {
- 'index_path': 'index.html',
- 'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
- 'archivedotorg_path': f'https://web.archive.org/web/{self.base_url}',
- }
-
- # Scan each ArchiveResult's output directory for the best file
- snap_dir = Path(self.output_dir)
- for result in self.archiveresult_set.filter(status='succeeded'):
- if not result.output_files and not result.output_str:
- continue
-
- # Try to find the best output file for this plugin
- plugin_dir = snap_dir / result.plugin
- best_output = None
-
- # Check output_files first (new field)
- if result.output_files:
- first_file = next(iter(result.output_files.keys()), None)
- if first_file and (plugin_dir / first_file).exists():
- best_output = f'{result.plugin}/{first_file}'
-
- # Fallback to output_str if it looks like a path
- if not best_output and result.output_str and (snap_dir / result.output_str).exists():
- best_output = result.output_str
-
- if not best_output and plugin_dir.exists():
- # Intelligently find the best file in the plugin's directory
- best_output = find_best_output_in_dir(plugin_dir, result.plugin)
-
- if best_output:
- canonical[f'{result.plugin}_path'] = best_output
-
- # Also scan top-level for legacy outputs (backwards compatibility)
- for file_path in snap_dir.glob('*'):
- if file_path.is_dir() or file_path.name in ('index.html', 'index.json'):
- continue
-
- ext = file_path.suffix.lstrip('.').lower()
- if ext not in IFRAME_EMBEDDABLE_EXTENSIONS:
- continue
-
- try:
- size = file_path.stat().st_size
- if size >= MIN_DISPLAY_SIZE:
- # Add as generic output with stem as key
- key = f'{file_path.stem}_path'
- if key not in canonical:
- canonical[key] = file_path.name
- except OSError:
- continue
-
- if self.is_static:
- static_path = f'warc/{self.timestamp}'
- canonical.update({
- 'title': self.basename,
- 'wget_path': static_path,
- })
-
- return canonical
-
def latest_outputs(self, status: Optional[str] = None) -> Dict[str, Any]:
"""Get the latest output that each plugin produced"""
from archivebox.hooks import get_plugins
@@ -2078,6 +1960,96 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
latest[plugin] = result.embed_path() if result else None
return latest
+ def discover_outputs(self) -> list[dict]:
+ """Discover output files from ArchiveResults and filesystem."""
+ from archivebox.misc.util import ts_to_date_str
+
+ ArchiveResult = self.archiveresult_set.model
+ snap_dir = Path(self.output_dir)
+ outputs: list[dict] = []
+ seen: set[str] = set()
+
+ text_exts = ('.json', '.jsonl', '.txt', '.csv', '.tsv', '.xml', '.yml', '.yaml', '.md', '.log')
+
+ def is_metadata_path(path: str | None) -> bool:
+ lower = (path or '').lower()
+ return lower.endswith(text_exts)
+
+ def is_compact_path(path: str | None) -> bool:
+ lower = (path or '').lower()
+ return lower.endswith(text_exts)
+
+ for result in self.archiveresult_set.all().order_by('start_ts'):
+ embed_path = result.embed_path()
+ if not embed_path or embed_path.strip() in ('.', '/', './'):
+ continue
+ abs_path = snap_dir / embed_path
+ if not abs_path.exists():
+ continue
+ if abs_path.is_dir():
+ if not any(p.is_file() for p in abs_path.rglob('*')):
+ continue
+ size = sum(p.stat().st_size for p in abs_path.rglob('*') if p.is_file())
+ else:
+ size = abs_path.stat().st_size
+ outputs.append({
+ 'name': result.plugin,
+ 'path': embed_path,
+ 'ts': ts_to_date_str(result.end_ts),
+ 'size': size or 0,
+ 'is_metadata': is_metadata_path(embed_path),
+ 'is_compact': is_compact_path(embed_path),
+ 'result': result,
+ })
+ seen.add(result.plugin)
+
+ embeddable_exts = {
+ 'html', 'htm', 'pdf', 'txt', 'md', 'json', 'jsonl', 'csv', 'tsv',
+ 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg', 'ico',
+ 'mp4', 'webm', 'mp3', 'opus', 'ogg', 'wav',
+ }
+
+ for entry in snap_dir.iterdir():
+ if entry.name in ('index.html', 'index.json', 'favicon.ico', 'warc'):
+ continue
+ if entry.is_dir():
+ plugin = entry.name
+ if plugin in seen:
+ continue
+ best_file = ArchiveResult._find_best_output_file(entry, plugin)
+ if not best_file:
+ continue
+ rel_path = str(best_file.relative_to(snap_dir))
+ outputs.append({
+ 'name': plugin,
+ 'path': rel_path,
+ 'ts': ts_to_date_str(best_file.stat().st_mtime or 0),
+ 'size': best_file.stat().st_size or 0,
+ 'is_metadata': is_metadata_path(rel_path),
+ 'is_compact': is_compact_path(rel_path),
+ 'result': None,
+ })
+ seen.add(plugin)
+ elif entry.is_file():
+ ext = entry.suffix.lstrip('.').lower()
+ if ext not in embeddable_exts:
+ continue
+ plugin = entry.stem
+ if plugin in seen:
+ continue
+ outputs.append({
+ 'name': plugin,
+ 'path': entry.name,
+ 'ts': ts_to_date_str(entry.stat().st_mtime or 0),
+ 'size': entry.stat().st_size or 0,
+ 'is_metadata': is_metadata_path(entry.name),
+ 'is_compact': is_compact_path(entry.name),
+ 'result': None,
+ })
+ seen.add(plugin)
+
+ return outputs
+
# =========================================================================
# Serialization Methods
# =========================================================================
@@ -2114,8 +2086,6 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
'num_outputs': self.num_outputs,
'num_failures': self.num_failures,
}
- if extended:
- result['canonical'] = self.canonical_outputs()
return result
def to_json_str(self, indent: int = 4) -> str:
@@ -2146,23 +2116,29 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
SAVE_ARCHIVE_DOT_ORG = config.get('SAVE_ARCHIVE_DOT_ORG', True)
TITLE_LOADING_MSG = 'Not yet archived...'
- canonical = self.canonical_outputs()
preview_priority = [
- 'singlefile_path',
- 'screenshot_path',
- 'wget_path',
- 'dom_path',
- 'pdf_path',
- 'readability_path',
+ 'singlefile',
+ 'screenshot',
+ 'wget',
+ 'dom',
+ 'pdf',
+ 'readability',
]
- best_preview_path = next(
- (canonical.get(key) for key in preview_priority if canonical.get(key)),
- canonical.get('index_path', 'index.html'),
- )
+
+ outputs = self.discover_outputs()
+ outputs_by_plugin = {out['name']: out for out in outputs}
+
+ best_preview_path = 'about:blank'
+ for plugin in preview_priority:
+ out = outputs_by_plugin.get(plugin)
+ if out and out.get('path'):
+ best_preview_path = out['path']
+ break
+
+ if best_preview_path == 'about:blank' and outputs:
+ best_preview_path = outputs[0].get('path') or 'about:blank'
context = {
**self.to_dict(extended=True),
- **{f'{k}_path': v for k, v in canonical.items()},
- 'canonical': {f'{k}_path': v for k, v in canonical.items()},
'title': htmlencode(self.title or (self.base_url if self.is_archived else TITLE_LOADING_MSG)),
'url_str': htmlencode(urldecode(self.base_url)),
'archive_url': urlencode(f'warc/{self.timestamp}' or (self.domain if self.is_archived else '')) or 'about:blank',
@@ -2175,6 +2151,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
'best_preview_path': best_preview_path,
+ 'archiveresults': outputs,
}
rendered_html = render_to_string('snapshot.html', context)
atomic_write(str(Path(out_dir) / CONSTANTS.HTML_INDEX_FILENAME), rendered_html)
@@ -2496,6 +2473,61 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWi
def output_exists(self) -> bool:
return os.path.exists(Path(self.snapshot_dir) / self.plugin)
+ @staticmethod
+ def _find_best_output_file(dir_path: Path, plugin_name: str | None = None) -> Optional[Path]:
+ if not dir_path.exists() or not dir_path.is_dir():
+ return None
+
+ embeddable_exts = {
+ 'html', 'htm', 'pdf', 'txt', 'md', 'json', 'jsonl', 'csv', 'tsv',
+ 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg', 'ico',
+ 'mp4', 'webm', 'mp3', 'opus', 'ogg', 'wav',
+ }
+
+ for name in ('index.html', 'index.htm'):
+ candidate = dir_path / name
+ if candidate.exists() and candidate.is_file():
+ return candidate
+
+ candidates = []
+ file_count = 0
+ max_scan = 200
+ plugin_lower = (plugin_name or '').lower()
+ for file_path in dir_path.rglob('*'):
+ file_count += 1
+ if file_count > max_scan:
+ break
+ if file_path.is_dir() or file_path.name.startswith('.'):
+ continue
+ ext = file_path.suffix.lstrip('.').lower()
+ if ext not in embeddable_exts:
+ continue
+ try:
+ size = file_path.stat().st_size
+ except OSError:
+ continue
+ name_lower = file_path.name.lower()
+ priority = 0
+ if name_lower.startswith('index'):
+ priority = 100
+ elif plugin_lower and name_lower.startswith(('output', 'content', plugin_lower)):
+ priority = 60
+ elif ext in ('html', 'htm', 'pdf'):
+ priority = 40
+ elif ext in ('png', 'jpg', 'jpeg', 'webp', 'svg', 'gif', 'ico'):
+ priority = 30
+ elif ext in ('json', 'jsonl', 'txt', 'md', 'csv', 'tsv'):
+ priority = 20
+ else:
+ priority = 10
+ candidates.append((priority, size, file_path))
+
+ if not candidates:
+ return None
+
+ candidates.sort(key=lambda x: (x[0], x[1]), reverse=True)
+ return candidates[0][2]
+
def embed_path(self) -> Optional[str]:
"""
Get the relative path to the embeddable output file for this result.
@@ -2503,25 +2535,45 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWi
Returns the first file from output_files if set, otherwise tries to
find a reasonable default based on the plugin type.
"""
- # Check output_files dict for primary output
+ snapshot_dir = Path(self.snapshot_dir)
+ plugin_dir = snapshot_dir / self.plugin
+
+ # Fallback: treat output_str as a file path only if it exists on disk
+ if self.output_str:
+ try:
+ output_path = Path(self.output_str)
+
+ if output_path.is_absolute():
+ # If absolute and within snapshot dir, normalize to relative
+ if snapshot_dir in output_path.parents and output_path.exists():
+ return str(output_path.relative_to(snapshot_dir))
+ else:
+ # If relative, prefer plugin-prefixed path, then direct path
+ if (plugin_dir / output_path).exists():
+ return f'{self.plugin}/{output_path}'
+ if output_path.name in ('index.html', 'index.json') and output_path.parent == Path('.'):
+ return None
+ if (snapshot_dir / output_path).exists():
+ return str(output_path)
+ except Exception:
+ pass
+
+ # Check output_files dict for primary output (ignore non-output files)
if self.output_files:
- # Return first file from output_files (dict preserves insertion order)
- first_file = next(iter(self.output_files.keys()), None)
- if first_file:
+ ignored = {'stdout.log', 'stderr.log', 'hook.pid', 'listener.pid', 'cmd.sh'}
+ output_candidates = [
+ f for f in self.output_files.keys()
+ if Path(f).name not in ignored
+ ]
+ first_file = output_candidates[0] if output_candidates else None
+ if first_file and (plugin_dir / first_file).exists():
return f'{self.plugin}/{first_file}'
- # Fallback: check output_str if it looks like a file path
- if self.output_str and ('/' in self.output_str or '.' in self.output_str):
- return self.output_str
+ best_file = self._find_best_output_file(plugin_dir, self.plugin)
+ if best_file:
+ return str(best_file.relative_to(snapshot_dir))
- # Try to find output file based on plugin's canonical output path
- canonical = self.snapshot.canonical_outputs()
- plugin_key = f'{self.plugin}_path'
- if plugin_key in canonical:
- return canonical[plugin_key]
-
- # Fallback to plugin directory
- return f'{self.plugin}/'
+ return None
def create_output_dir(self):
output_dir = Path(self.snapshot_dir) / self.plugin
@@ -2779,7 +2831,7 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWi
self.output_str = 'Hook did not output ArchiveResult record'
# Walk filesystem and populate output_files, output_size, output_mimetypes
- exclude_names = {'stdout.log', 'stderr.log', 'hook.pid', 'listener.pid'}
+ exclude_names = {'stdout.log', 'stderr.log', 'hook.pid', 'listener.pid', 'cmd.sh'}
mime_sizes = defaultdict(int)
total_size = 0
output_files = {}
diff --git a/archivebox/core/settings_logging.py b/archivebox/core/settings_logging.py
index 85d6404a..6c2cfd52 100644
--- a/archivebox/core/settings_logging.py
+++ b/archivebox/core/settings_logging.py
@@ -48,6 +48,19 @@ class CustomOutboundWebhookLogFormatter(logging.Formatter):
result = super().format(record)
return result.replace('HTTP Request: ', 'OutboundWebhook: ')
+class StripANSIColorCodesFilter(logging.Filter):
+ _ansi_re = re.compile(r'\x1b\[[0-9;]*m')
+ _bare_re = re.compile(r'\[[0-9;]*m')
+
+ def filter(self, record) -> bool:
+ msg = record.getMessage()
+ if isinstance(msg, str) and ('\x1b[' in msg or '[m' in msg):
+ msg = self._ansi_re.sub('', msg)
+ msg = self._bare_re.sub('', msg)
+ record.msg = msg
+ record.args = ()
+ return True
+
ERROR_LOG = tempfile.NamedTemporaryFile().name
@@ -87,6 +100,9 @@ SETTINGS_LOGGING = {
"noisyrequestsfilter": {
"()": NoisyRequestsFilter,
},
+ "stripansi": {
+ "()": StripANSIColorCodesFilter,
+ },
"require_debug_false": {
"()": "django.utils.log.RequireDebugFalse",
},
@@ -101,7 +117,7 @@ SETTINGS_LOGGING = {
"level": "DEBUG",
"markup": False,
"rich_tracebacks": False, # Use standard Python tracebacks (no frame/box)
- "filters": ["noisyrequestsfilter"],
+ "filters": ["noisyrequestsfilter", "stripansi"],
},
"logfile": {
"level": "INFO",
@@ -110,7 +126,7 @@ SETTINGS_LOGGING = {
"maxBytes": 1024 * 1024 * 25, # 25 MB
"backupCount": 10,
"formatter": "rich",
- "filters": ["noisyrequestsfilter"],
+ "filters": ["noisyrequestsfilter", "stripansi"],
},
"outbound_webhooks": {
"class": "rich.logging.RichHandler",
diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py
index 7e201f94..bcf7f10d 100644
--- a/archivebox/core/templatetags/core_tags.py
+++ b/archivebox/core/templatetags/core_tags.py
@@ -1,8 +1,10 @@
from django import template
from django.contrib.admin.templatetags.base import InclusionAdminNode
from django.utils.safestring import mark_safe
+from django.utils.html import escape
from typing import Union
+from pathlib import Path
from archivebox.hooks import (
get_plugin_icon, get_plugin_template, get_plugin_name,
@@ -57,15 +59,18 @@ def plugin_icon(plugin: str) -> str:
Usage: {% plugin_icon "screenshot" %}
"""
- return mark_safe(get_plugin_icon(plugin))
+ icon_html = get_plugin_icon(plugin)
+ return mark_safe(
+ f'{icon_html}'
+ )
@register.simple_tag(takes_context=True)
-def plugin_thumbnail(context, result) -> str:
+def plugin_card(context, result) -> str:
"""
- Render the thumbnail template for an archive result.
+ Render the card template for an archive result.
- Usage: {% plugin_thumbnail result %}
+ Usage: {% plugin_card result %}
Context variables passed to template:
- result: ArchiveResult object
@@ -74,46 +79,97 @@ def plugin_thumbnail(context, result) -> str:
- plugin: Plugin base name
"""
plugin = get_plugin_name(result.plugin)
- template_str = get_plugin_template(plugin, 'thumbnail')
+ template_str = get_plugin_template(plugin, 'card')
- if not template_str:
- return ''
+ # Use embed_path() for the display path
+ output_path = result.embed_path() if hasattr(result, 'embed_path') else ''
- # Use embed_path() for the display path (includes canonical paths)
- output_path = result.embed_path() if hasattr(result, 'embed_path') else (result.output_str or '')
+ icon_html = get_plugin_icon(plugin)
+
+ output_lower = (output_path or '').lower()
+ text_preview_exts = ('.json', '.jsonl', '.txt', '.csv', '.tsv', '.xml', '.yml', '.yaml', '.md', '.log')
+ force_text_preview = output_lower.endswith(text_preview_exts)
# Create a mini template and render it with context
try:
- tpl = template.Template(template_str)
- ctx = template.Context({
- 'result': result,
- 'snapshot': result.snapshot,
- 'output_path': output_path,
- 'plugin': plugin,
- })
- rendered = tpl.render(ctx)
- # Only return non-empty content (strip whitespace to check)
- if rendered.strip():
- return mark_safe(rendered)
- return ''
+ if template_str and output_path and str(output_path).strip() not in ('.', '/', './') and not force_text_preview:
+ tpl = template.Template(template_str)
+ ctx = template.Context({
+ 'result': result,
+ 'snapshot': result.snapshot,
+ 'output_path': output_path,
+ 'plugin': plugin,
+ 'plugin_icon': icon_html,
+ })
+ rendered = tpl.render(ctx)
+ # Only return non-empty content (strip whitespace to check)
+ if rendered.strip():
+ return mark_safe(rendered)
except Exception:
- return ''
+ pass
+
+ if force_text_preview and output_path and str(output_path).strip() not in ('.', '/', './'):
+ output_file = Path(output_path)
+ if not output_file.is_absolute():
+ output_file = Path(result.snapshot_dir) / output_path
+ try:
+ output_file = output_file.resolve()
+ snap_dir = Path(result.snapshot_dir).resolve()
+ if snap_dir not in output_file.parents and output_file != snap_dir:
+ output_file = None
+ except Exception:
+ output_file = None
+ if output_file and output_file.exists() and output_file.is_file():
+ try:
+ with output_file.open('rb') as f:
+ raw = f.read(4096)
+ text = raw.decode('utf-8', errors='replace').strip()
+ if text:
+ lines = text.splitlines()[:6]
+ snippet = '\n'.join(lines)
+ escaped = escape(snippet)
+ preview = (
+ f''
+ f''
+ f' {escaped}'
+ f' '
+ f'{icon_html}'
+ f'{plugin}'
+ f'{fallback_label}'
+ f' '
+ )
+ return mark_safe(fallback)
@register.simple_tag(takes_context=True)
-def plugin_embed(context, result) -> str:
+def plugin_full(context, result) -> str:
"""
- Render the embed iframe template for an archive result.
+ Render the full template for an archive result.
- Usage: {% plugin_embed result %}
+ Usage: {% plugin_full result %}
"""
plugin = get_plugin_name(result.plugin)
- template_str = get_plugin_template(plugin, 'embed')
+ template_str = get_plugin_template(plugin, 'full')
if not template_str:
return ''
- output_path = result.embed_path() if hasattr(result, 'embed_path') else (result.output_str or '')
+ output_path = result.embed_path() if hasattr(result, 'embed_path') else ''
try:
tpl = template.Template(template_str)
@@ -132,36 +188,6 @@ def plugin_embed(context, result) -> str:
return ''
-@register.simple_tag(takes_context=True)
-def plugin_fullscreen(context, result) -> str:
- """
- Render the fullscreen template for an archive result.
-
- Usage: {% plugin_fullscreen result %}
- """
- plugin = get_plugin_name(result.plugin)
- template_str = get_plugin_template(plugin, 'fullscreen')
-
- if not template_str:
- return ''
-
- output_path = result.embed_path() if hasattr(result, 'embed_path') else (result.output_str or '')
-
- try:
- tpl = template.Template(template_str)
- ctx = template.Context({
- 'result': result,
- 'snapshot': result.snapshot,
- 'output_path': output_path,
- 'plugin': plugin,
- })
- rendered = tpl.render(ctx)
- # Only return non-empty content (strip whitespace to check)
- if rendered.strip():
- return mark_safe(rendered)
- return ''
- except Exception:
- return ''
@register.filter
diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py
index 94e3582d..708705a6 100644
--- a/archivebox/core/urls.py
+++ b/archivebox/core/urls.py
@@ -8,7 +8,7 @@ from django.views.generic.base import RedirectView
from archivebox.misc.serve_static import serve_static
from archivebox.core.admin_site import archivebox_admin
-from archivebox.core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView, live_progress_view
+from archivebox.core.views import HomepageView, SnapshotView, SnapshotPathView, PublicIndexView, AddView, HealthCheckView, live_progress_view
from archivebox.workers.views import JobsDashboardView
@@ -32,6 +32,8 @@ urlpatterns = [
path('archive/', RedirectView.as_view(url='/')),
path('archive/', SnapshotView.as_view(), name='Snapshot'),
+ re_path(r'^(?P[^/]+)/(?P\d{4}(?:\d{2})?(?:\d{2})?)/(?Phttps?://.*)$', SnapshotPathView.as_view(), name='snapshot-path-url'),
+ re_path(r'^(?P[^/]+)/(?P\d{4}(?:\d{2})?(?:\d{2})?)/(?P[^/]+)(?:/(?P[0-9a-fA-F-]{8,36})(?:/(?P.*))?)?$', SnapshotPathView.as_view(), name='snapshot-path'),
path('admin/core/snapshot/add/', RedirectView.as_view(url='/add/')),
path('add/', AddView.as_view(), name='add'),
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index eec08661..4dd7afea 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -1,7 +1,6 @@
__package__ = 'archivebox.core'
import os
-import sys
from django.utils import timezone
import inspect
from typing import Callable, get_type_hints
@@ -26,7 +25,7 @@ import archivebox
from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, DATA_DIR, VERSION
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
from archivebox.config.configset import get_flat_config, get_config, get_all_configs
-from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
+from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.misc.logging_util import printable_filesize
from archivebox.search import query_search_index
@@ -52,70 +51,44 @@ class HomepageView(View):
class SnapshotView(View):
# render static html index from filesystem archive//index.html
+ @staticmethod
+ def find_snapshots_for_url(path: str):
+ """Return a queryset of snapshots matching a URL-ish path."""
+ normalized = path
+ if path.startswith(('http://', 'https://')):
+ # try exact match on full url / ID first
+ qs = Snapshot.objects.filter(Q(url=path) | Q(id__icontains=path))
+ if qs.exists():
+ return qs
+ normalized = path.split('://', 1)[1]
+
+ # try exact match on full url / ID (without scheme)
+ qs = Snapshot.objects.filter(
+ Q(url='http://' + normalized) | Q(url='https://' + normalized) | Q(id__icontains=normalized)
+ )
+ if qs.exists():
+ return qs
+
+ # fall back to match on exact base_url
+ base = base_url(normalized)
+ qs = Snapshot.objects.filter(
+ Q(url='http://' + base) | Q(url='https://' + base)
+ )
+ if qs.exists():
+ return qs
+
+ # fall back to matching base_url as prefix
+ return Snapshot.objects.filter(
+ Q(url__startswith='http://' + base) | Q(url__startswith='https://' + base)
+ )
+
@staticmethod
def render_live_index(request, snapshot):
TITLE_LOADING_MSG = 'Not yet archived...'
- # Dict of plugin -> ArchiveResult object
- archiveresult_objects = {}
- # Dict of plugin -> result info dict (for template compatibility)
- archiveresults = {}
-
- results = snapshot.archiveresult_set.all()
-
- for result in results:
- embed_path = result.embed_path()
- abs_path = result.snapshot_dir / (embed_path or 'None')
-
- if (result.status == 'succeeded'
- and embed_path
- and os.access(abs_path, os.R_OK)
- and abs_path.exists()):
- if os.path.isdir(abs_path) and not any(abs_path.glob('*.*')):
- continue
-
- # Store the full ArchiveResult object for template tags
- archiveresult_objects[result.plugin] = result
-
- result_info = {
- 'name': result.plugin,
- 'path': embed_path,
- 'ts': ts_to_date_str(result.end_ts),
- 'size': abs_path.stat().st_size or '?',
- 'result': result, # Include the full object for template tags
- }
- archiveresults[result.plugin] = result_info
-
- # Use canonical_outputs for intelligent discovery
- # This method now scans ArchiveResults and uses smart heuristics
- canonical = snapshot.canonical_outputs()
-
- # Add any newly discovered outputs from canonical_outputs to archiveresults
+ outputs = snapshot.discover_outputs()
+ archiveresults = {out['name']: out for out in outputs}
snap_dir = Path(snapshot.output_dir)
- for key, path in canonical.items():
- if not key.endswith('_path') or not path or path.startswith('http'):
- continue
-
- plugin_name = key.replace('_path', '')
- if plugin_name in archiveresults:
- continue # Already have this from ArchiveResult
-
- file_path = snap_dir / path
- if not file_path.exists() or not file_path.is_file():
- continue
-
- try:
- file_size = file_path.stat().st_size
- if file_size >= 15_000: # Only show files > 15KB
- archiveresults[plugin_name] = {
- 'name': plugin_name,
- 'path': path,
- 'ts': ts_to_date_str(file_path.stat().st_mtime or 0),
- 'size': file_size,
- 'result': None,
- }
- except OSError:
- continue
# Get available extractor plugins from hooks (sorted by numeric prefix for ordering)
# Convert to base names for display ordering
@@ -131,7 +104,7 @@ class SnapshotView(View):
preferred_types = tuple(preview_priority + [p for p in all_plugins if p not in preview_priority])
all_types = preferred_types + tuple(result_type for result_type in archiveresults.keys() if result_type not in preferred_types)
- best_result = {'path': 'None', 'result': None}
+ best_result = {'path': 'about:blank', 'result': None}
for result_type in preferred_types:
if result_type in archiveresults:
best_result = archiveresults[result_type]
@@ -146,7 +119,6 @@ class SnapshotView(View):
context = {
**snapshot_info,
- **snapshot_info.get('canonical', {}),
'title': htmlencode(
snapshot.title
or (snapshot.base_url if snapshot.is_archived else TITLE_LOADING_MSG)
@@ -188,6 +160,14 @@ class SnapshotView(View):
try:
try:
snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
+ canonical_base = snapshot.url_path
+ if canonical_base != snapshot.legacy_archive_path:
+ target_path = f'/{canonical_base}/{archivefile or "index.html"}'
+ query = request.META.get('QUERY_STRING')
+ if query:
+ target_path = f'{target_path}?{query}'
+ return redirect(target_path)
+
if archivefile == 'index.html':
# if they requested snapshot index, serve live rendered template instead of static html
response = self.render_live_index(request, snapshot)
@@ -221,9 +201,9 @@ class SnapshotView(View):
except Snapshot.MultipleObjectsReturned:
snapshot_hrefs = mark_safe('
').join(
format_html(
- '{} {} {} {}',
+ '{} {} {} {}',
snap.bookmarked_at.strftime('%Y-%m-%d %H:%M:%S'),
- snap.timestamp,
+ snap.archive_path,
snap.timestamp,
snap.url,
snap.title_stripped[:64] or '',
@@ -259,9 +239,9 @@ class SnapshotView(View):
#''
''
'
'
- f'Snapshot [{snapshot.timestamp}]: {snapshot.url}
'
+ f'Snapshot [{snapshot.timestamp}]: {snapshot.url}
'
f'was queued on {str(snapshot.bookmarked_at).split(".")[0]}, '
- f'but no files have been saved yet in:
{snapshot.timestamp}/'
+ f'but no files have been saved yet in:
{snapshot.timestamp}/'
'{}'
f'
'
'It\'s possible {} '
@@ -270,8 +250,8 @@ class SnapshotView(View):
f'archivebox update -t timestamp {snapshot.timestamp}
'
''
' Next steps:'
- f'- list all the Snapshot files .*'
- f'- view the Snapshot ./index.html'
+ f'- list all the Snapshot files .*'
+ f'- view the Snapshot ./index.html'
f'- go to the Snapshot admin to edit '
f'- go to the Snapshot actions to re-archive '
'- or return to the main index...
').join(
format_html(
- '{} {} {} {} {}',
+ '{} {} {} {} {}',
snap.bookmarked_at.strftime('%Y-%m-%d %H:%M:%S'),
str(snap.id)[:8],
- snap.timestamp,
+ snap.archive_path,
snap.timestamp,
snap.url,
snap.title_stripped[:64] or '',
)
- for snap in Snapshot.objects.filter(
- Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
- | Q(id__icontains=path)
- ).only('url', 'timestamp', 'title', 'bookmarked_at').order_by('-bookmarked_at')
+ for snap in snapshots.only('url', 'timestamp', 'title', 'bookmarked_at').order_by('-bookmarked_at')
)
return HttpResponse(
format_html(
@@ -353,6 +318,108 @@ class SnapshotView(View):
status=404,
)
+ target_path = f'/{snapshot.archive_path}/index.html'
+ query = request.META.get('QUERY_STRING')
+ if query:
+ target_path = f'{target_path}?{query}'
+ return redirect(target_path)
+
+
+class SnapshotPathView(View):
+ """Serve snapshots by the new URL scheme: /////..."""
+
+ def get(self, request, username: str, date: str, domain: str | None = None, snapshot_id: str | None = None, path: str = "", url: str | None = None):
+ if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
+ return redirect(f'/admin/login/?next={request.path}')
+
+ if username == 'system':
+ return redirect(request.path.replace('/system/', '/web/', 1))
+
+ requested_url = url
+ if not requested_url and domain and domain.startswith(('http://', 'https://')):
+ requested_url = domain
+
+ snapshot = None
+ if snapshot_id:
+ try:
+ snapshot = Snapshot.objects.get(pk=snapshot_id)
+ except Snapshot.DoesNotExist:
+ try:
+ snapshot = Snapshot.objects.get(id__startswith=snapshot_id)
+ except Snapshot.DoesNotExist:
+ snapshot = None
+ except Snapshot.MultipleObjectsReturned:
+ snapshot = Snapshot.objects.filter(id__startswith=snapshot_id).first()
+ else:
+ # fuzzy lookup by date + domain/url (most recent)
+ username_lookup = 'system' if username == 'web' else username
+ if requested_url:
+ qs = SnapshotView.find_snapshots_for_url(requested_url).filter(crawl__created_by__username=username_lookup)
+ else:
+ qs = Snapshot.objects.filter(crawl__created_by__username=username_lookup)
+
+ try:
+ if len(date) == 4:
+ qs = qs.filter(created_at__year=int(date))
+ elif len(date) == 6:
+ qs = qs.filter(created_at__year=int(date[:4]), created_at__month=int(date[4:6]))
+ elif len(date) == 8:
+ qs = qs.filter(
+ created_at__year=int(date[:4]),
+ created_at__month=int(date[4:6]),
+ created_at__day=int(date[6:8]),
+ )
+ except ValueError:
+ pass
+
+ if requested_url:
+ snapshot = qs.order_by('-created_at', '-bookmarked_at', '-timestamp').first()
+ else:
+ requested_domain = domain or ''
+ if requested_domain.startswith(('http://', 'https://')):
+ requested_domain = Snapshot.extract_domain_from_url(requested_domain)
+ else:
+ requested_domain = Snapshot.extract_domain_from_url(f'https://{requested_domain}')
+
+ # Prefer exact domain matches
+ matches = [s for s in qs.order_by('-created_at', '-bookmarked_at') if Snapshot.extract_domain_from_url(s.url) == requested_domain]
+ snapshot = matches[0] if matches else qs.order_by('-created_at', '-bookmarked_at', '-timestamp').first()
+
+ if not snapshot:
+ return HttpResponse(
+ format_html(
+ (
+ '
'
+ 'No Snapshots match the given id or url: {}
'
+ 'Return to the Main Index'
+ ''
+ ),
+ snapshot_id or requested_url or domain,
+ ),
+ content_type="text/html",
+ status=404,
+ )
+
+ canonical_base = snapshot.url_path
+ requested_base = f'{username}/{date}/{domain or url or ""}'
+ if snapshot_id:
+ requested_base = f'{requested_base}/{snapshot_id}'
+ if canonical_base != requested_base:
+ target = f'/{canonical_base}/{path or "index.html"}'
+ query = request.META.get('QUERY_STRING')
+ if query:
+ target = f'{target}?{query}'
+ return redirect(target)
+
+ archivefile = path or "index.html"
+
+ if archivefile == "index.html":
+ return SnapshotView.render_live_index(request, snapshot)
+
+ return serve_static_with_byterange_support(
+ request, archivefile, document_root=snapshot.output_dir, show_indexes=True,
+ )
+
class PublicIndexView(ListView):
template_name = 'public_index.html'
@@ -592,7 +659,7 @@ def live_progress_view(request):
'snapshot_id': str(ar.snapshot_id),
'snapshot_url': ar.snapshot.url[:60] if ar.snapshot else '',
'embed_path': embed,
- 'archive_path': f'/archive/{ar.snapshot.timestamp}/{embed}' if ar.snapshot else '',
+ 'archive_path': f'/{ar.snapshot.archive_path}/{embed}' if ar.snapshot else '',
'end_ts': ar.end_ts.isoformat() if ar.end_ts else None,
})
diff --git a/archivebox/crawls/admin.py b/archivebox/crawls/admin.py
index 110fe941..01b18375 100644
--- a/archivebox/crawls/admin.py
+++ b/archivebox/crawls/admin.py
@@ -71,8 +71,8 @@ def render_snapshots_list(snapshots_qs, limit=20):
color: {color}; background: {bg};">{status}
|
-
- 
+ 
diff --git a/archivebox/hooks.py b/archivebox/hooks.py
index e5483e59..08cedf0f 100644
--- a/archivebox/hooks.py
+++ b/archivebox/hooks.py
@@ -940,9 +940,8 @@ def get_plugin_special_config(plugin_name: str, config: Dict[str, Any]) -> Dict[
# archivebox/plugins//
# templates/
# icon.html # Icon for admin table view (small inline HTML)
-# thumbnail.html # Preview thumbnail for snapshot cards
-# embed.html # Iframe embed content for main preview
-# fullscreen.html # Fullscreen view template
+# card.html # Preview card for snapshot header
+# full.html # Fullscreen view template
#
# Template context variables available:
# {{ result }} - ArchiveResult object
@@ -953,21 +952,22 @@ def get_plugin_special_config(plugin_name: str, config: Dict[str, Any]) -> Dict[
# Default templates used when plugin doesn't provide one
DEFAULT_TEMPLATES = {
- 'icon': '''{{ icon }}''',
- 'thumbnail': '''
- 
+ 'icon': '''
+
+ {{ icon }}
+
''',
- 'embed': '''
+ 'card': '''
''',
- 'fullscreen': '''
+ 'full': '''
@@ -981,7 +981,7 @@ def get_plugin_template(plugin: str, template_name: str, fallback: bool = True)
Args:
plugin: Plugin name (e.g., 'screenshot', '15_singlefile')
- template_name: One of 'icon', 'thumbnail', 'embed', 'fullscreen'
+ template_name: One of 'icon', 'card', 'full'
fallback: If True, return default template if plugin template not found
Returns:
@@ -1050,7 +1050,7 @@ def discover_plugin_templates() -> Dict[str, Dict[str, str]]:
Returns:
Dict mapping plugin names to dicts of template_name -> template_path.
- e.g., {'screenshot': {'icon': '/path/to/icon.html', 'thumbnail': '/path/to/thumbnail.html'}}
+ e.g., {'screenshot': {'icon': '/path/to/icon.html', 'card': '/path/to/card.html'}}
"""
templates: Dict[str, Dict[str, str]] = {}
@@ -1068,7 +1068,7 @@ def discover_plugin_templates() -> Dict[str, Dict[str, str]]:
plugin_templates = {}
for template_file in templates_dir.glob('*.html'):
- template_name = template_file.stem # icon, thumbnail, embed, fullscreen
+ template_name = template_file.stem # icon, card, full
plugin_templates[template_name] = str(template_file)
if plugin_templates:
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index b63845ac..a47f32ea 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -237,8 +237,8 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
sha256 = models.CharField(max_length=64, default='', null=False, blank=True)
# State machine fields
- status = models.CharField(max_length=16, choices=StatusChoices.choices, default=StatusChoices.QUEUED, db_index=True)
- retry_at = models.DateTimeField(default=timezone.now, null=True, blank=True, db_index=True,
+ status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
+ retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now,
help_text="When to retry this binary installation")
# Health stats
@@ -246,6 +246,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
num_uses_succeeded = models.PositiveIntegerField(default=0)
state_machine_name: str = 'archivebox.machine.models.BinaryMachine'
+ active_state: str = StatusChoices.QUEUED
objects: BinaryManager = BinaryManager()
diff --git a/archivebox/plugins/accessibility/templates/thumbnail.html b/archivebox/plugins/accessibility/templates/thumbnail.html
deleted file mode 100644
index e69de29b..00000000
diff --git a/archivebox/plugins/archivedotorg/templates/thumbnail.html b/archivebox/plugins/archivedotorg/templates/card.html
similarity index 100%
rename from archivebox/plugins/archivedotorg/templates/thumbnail.html
rename to archivebox/plugins/archivedotorg/templates/card.html
diff --git a/archivebox/plugins/chrome/on_Crawl__90_chrome_launch.bg.js b/archivebox/plugins/chrome/on_Crawl__90_chrome_launch.bg.js
index c50eb847..17185786 100644
--- a/archivebox/plugins/chrome/on_Crawl__90_chrome_launch.bg.js
+++ b/archivebox/plugins/chrome/on_Crawl__90_chrome_launch.bg.js
@@ -49,6 +49,111 @@ const OUTPUT_DIR = '.';
let chromePid = null;
let browserInstance = null;
+function parseCookiesTxt(contents) {
+ const cookies = [];
+ let skipped = 0;
+
+ for (const rawLine of contents.split(/\r?\n/)) {
+ const line = rawLine.trim();
+ if (!line) continue;
+
+ let httpOnly = false;
+ let dataLine = line;
+
+ if (dataLine.startsWith('#HttpOnly_')) {
+ httpOnly = true;
+ dataLine = dataLine.slice('#HttpOnly_'.length);
+ } else if (dataLine.startsWith('#')) {
+ continue;
+ }
+
+ const parts = dataLine.split('\t');
+ if (parts.length < 7) {
+ skipped += 1;
+ continue;
+ }
+
+ const [domainRaw, includeSubdomainsRaw, pathRaw, secureRaw, expiryRaw, name, value] = parts;
+ if (!name || !domainRaw) {
+ skipped += 1;
+ continue;
+ }
+
+ const includeSubdomains = (includeSubdomainsRaw || '').toUpperCase() === 'TRUE';
+ let domain = domainRaw;
+ if (includeSubdomains && !domain.startsWith('.')) domain = `.${domain}`;
+ if (!includeSubdomains && domain.startsWith('.')) domain = domain.slice(1);
+
+ const cookie = {
+ name,
+ value,
+ domain,
+ path: pathRaw || '/',
+ secure: (secureRaw || '').toUpperCase() === 'TRUE',
+ httpOnly,
+ };
+
+ const expires = parseInt(expiryRaw, 10);
+ if (!isNaN(expires) && expires > 0) {
+ cookie.expires = expires;
+ }
+
+ cookies.push(cookie);
+ }
+
+ return { cookies, skipped };
+}
+
+async function importCookiesFromFile(browser, cookiesFile, userDataDir) {
+ if (!cookiesFile) return;
+
+ if (!fs.existsSync(cookiesFile)) {
+ console.error(`[!] Cookies file not found: ${cookiesFile}`);
+ return;
+ }
+
+ let contents = '';
+ try {
+ contents = fs.readFileSync(cookiesFile, 'utf-8');
+ } catch (e) {
+ console.error(`[!] Failed to read COOKIES_TXT_FILE: ${e.message}`);
+ return;
+ }
+
+ const { cookies, skipped } = parseCookiesTxt(contents);
+ if (cookies.length === 0) {
+ console.error('[!] No cookies found to import');
+ return;
+ }
+
+ console.error(`[*] Importing ${cookies.length} cookies from ${cookiesFile}...`);
+ if (skipped) {
+ console.error(`[*] Skipped ${skipped} malformed cookie line(s)`);
+ }
+ if (!userDataDir) {
+ console.error('[!] CHROME_USER_DATA_DIR not set; cookies will not persist beyond this session');
+ }
+
+ const page = await browser.newPage();
+ const client = await page.target().createCDPSession();
+ await client.send('Network.enable');
+
+ const chunkSize = 200;
+ let imported = 0;
+ for (let i = 0; i < cookies.length; i += chunkSize) {
+ const chunk = cookies.slice(i, i + chunkSize);
+ try {
+ await client.send('Network.setCookies', { cookies: chunk });
+ imported += chunk.length;
+ } catch (e) {
+ console.error(`[!] Failed to import cookies ${i + 1}-${i + chunk.length}: ${e.message}`);
+ }
+ }
+
+ await page.close();
+ console.error(`[+] Imported ${imported}/${cookies.length} cookies`);
+}
+
// Parse command line arguments
function parseArgs() {
const args = {};
@@ -118,10 +223,14 @@ async function main() {
// Load installed extensions
const extensionsDir = getExtensionsDir();
const userDataDir = getEnv('CHROME_USER_DATA_DIR');
+ const cookiesFile = getEnv('COOKIES_TXT_FILE') || getEnv('COOKIES_FILE');
if (userDataDir) {
console.error(`[*] Using user data dir: ${userDataDir}`);
}
+ if (cookiesFile) {
+ console.error(`[*] Using cookies file: ${cookiesFile}`);
+ }
const installedExtensions = [];
const extensionPaths = [];
@@ -179,6 +288,9 @@ async function main() {
});
browserInstance = browser;
+ // Import cookies into Chrome profile at crawl start
+ await importCookiesFromFile(browser, cookiesFile, userDataDir);
+
// Get actual extension IDs from chrome://extensions page
if (extensionPaths.length > 0) {
await new Promise(r => setTimeout(r, 2000));
diff --git a/archivebox/plugins/chrome/tests/test_chrome.py b/archivebox/plugins/chrome/tests/test_chrome.py
index c23a48d9..554a2539 100644
--- a/archivebox/plugins/chrome/tests/test_chrome.py
+++ b/archivebox/plugins/chrome/tests/test_chrome.py
@@ -38,6 +38,82 @@ from archivebox.plugins.chrome.tests.chrome_test_helpers import (
CHROME_NAVIGATE_HOOK,
)
+def _get_cookies_via_cdp(port: int, env: dict) -> list[dict]:
+ node_script = r"""
+const http = require('http');
+const WebSocket = require('ws');
+const port = process.env.CDP_PORT;
+
+function getTargets() {
+ return new Promise((resolve, reject) => {
+ const req = http.get(`http://127.0.0.1:${port}/json/list`, (res) => {
+ let data = '';
+ res.on('data', (chunk) => (data += chunk));
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(data));
+ } catch (e) {
+ reject(e);
+ }
+ });
+ });
+ req.on('error', reject);
+ });
+}
+
+(async () => {
+ const targets = await getTargets();
+ const pageTarget = targets.find(t => t.type === 'page') || targets[0];
+ if (!pageTarget) {
+ console.error('No page target found');
+ process.exit(2);
+ }
+
+ const ws = new WebSocket(pageTarget.webSocketDebuggerUrl);
+ const timer = setTimeout(() => {
+ console.error('Timeout waiting for cookies');
+ process.exit(3);
+ }, 10000);
+
+ ws.on('open', () => {
+ ws.send(JSON.stringify({ id: 1, method: 'Network.getAllCookies' }));
+ });
+
+ ws.on('message', (data) => {
+ const msg = JSON.parse(data);
+ if (msg.id === 1) {
+ clearTimeout(timer);
+ ws.close();
+ if (!msg.result || !msg.result.cookies) {
+ console.error('No cookies in response');
+ process.exit(4);
+ }
+ process.stdout.write(JSON.stringify(msg.result.cookies));
+ process.exit(0);
+ }
+ });
+
+ ws.on('error', (err) => {
+ console.error(String(err));
+ process.exit(5);
+ });
+})().catch((err) => {
+ console.error(String(err));
+ process.exit(1);
+});
+"""
+
+ result = subprocess.run(
+ ['node', '-e', node_script],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ env=env | {'CDP_PORT': str(port)},
+ )
+ assert result.returncode == 0, f"Failed to read cookies via CDP: {result.stderr}\nStdout: {result.stdout}"
+ return json.loads(result.stdout or '[]')
+
+
@pytest.fixture(scope="session", autouse=True)
def ensure_chromium_and_puppeteer_installed(tmp_path_factory):
"""Ensure Chromium and puppeteer are installed before running tests."""
@@ -197,6 +273,77 @@ def test_chrome_launch_and_tab_creation():
pass
+def test_cookies_imported_on_launch():
+ """Integration test: COOKIES_TXT_FILE is imported at crawl start."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ crawl_dir = Path(tmpdir) / 'crawl'
+ crawl_dir.mkdir()
+ chrome_dir = crawl_dir / 'chrome'
+ chrome_dir.mkdir()
+
+ cookies_file = Path(tmpdir) / 'cookies.txt'
+ cookies_file.write_text(
+ '\n'.join([
+ '# Netscape HTTP Cookie File',
+ '# https://curl.se/docs/http-cookies.html',
+ '# This file was generated by a test',
+ '',
+ 'example.com\tTRUE\t/\tFALSE\t2147483647\tabx_test_cookie\thello',
+ '',
+ ])
+ )
+
+ profile_dir = Path(tmpdir) / 'profile'
+ env = get_test_env()
+ env.update({
+ 'CHROME_HEADLESS': 'true',
+ 'CHROME_USER_DATA_DIR': str(profile_dir),
+ 'COOKIES_TXT_FILE': str(cookies_file),
+ })
+
+ chrome_launch_process = subprocess.Popen(
+ ['node', str(CHROME_LAUNCH_HOOK), '--crawl-id=test-crawl-cookies'],
+ cwd=str(chrome_dir),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ env=env
+ )
+
+ for _ in range(15):
+ if (chrome_dir / 'port.txt').exists():
+ break
+ time.sleep(1)
+
+ assert (chrome_dir / 'port.txt').exists(), "port.txt should exist"
+ chrome_pid = int((chrome_dir / 'chrome.pid').read_text().strip())
+ port = int((chrome_dir / 'port.txt').read_text().strip())
+
+ cookie_found = False
+ for _ in range(15):
+ cookies = _get_cookies_via_cdp(port, env)
+ cookie_found = any(
+ c.get('name') == 'abx_test_cookie' and c.get('value') == 'hello'
+ for c in cookies
+ )
+ if cookie_found:
+ break
+ time.sleep(1)
+
+ assert cookie_found, "Imported cookie should be present in Chrome session"
+
+ # Cleanup
+ try:
+ chrome_launch_process.send_signal(signal.SIGTERM)
+ chrome_launch_process.wait(timeout=5)
+ except:
+ pass
+ try:
+ os.kill(chrome_pid, signal.SIGKILL)
+ except OSError:
+ pass
+
+
def test_chrome_navigation():
"""Integration test: Navigate to a URL."""
with tempfile.TemporaryDirectory() as tmpdir:
diff --git a/archivebox/plugins/consolelog/templates/thumbnail.html b/archivebox/plugins/consolelog/templates/thumbnail.html
deleted file mode 100644
index e69de29b..00000000
diff --git a/archivebox/plugins/dns/templates/thumbnail.html b/archivebox/plugins/dns/templates/thumbnail.html
deleted file mode 100644
index e69de29b..00000000
diff --git a/archivebox/plugins/dom/templates/thumbnail.html b/archivebox/plugins/dom/templates/card.html
similarity index 100%
rename from archivebox/plugins/dom/templates/thumbnail.html
rename to archivebox/plugins/dom/templates/card.html
diff --git a/archivebox/plugins/favicon/templates/thumbnail.html b/archivebox/plugins/favicon/templates/card.html
similarity index 100%
rename from archivebox/plugins/favicon/templates/thumbnail.html
rename to archivebox/plugins/favicon/templates/card.html
diff --git a/archivebox/plugins/forumdl/templates/thumbnail.html b/archivebox/plugins/forumdl/templates/card.html
similarity index 100%
rename from archivebox/plugins/forumdl/templates/thumbnail.html
rename to archivebox/plugins/forumdl/templates/card.html
diff --git a/archivebox/plugins/forumdl/templates/embed.html b/archivebox/plugins/forumdl/templates/embed.html
deleted file mode 100644
index 936b7562..00000000
--- a/archivebox/plugins/forumdl/templates/embed.html
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
diff --git a/archivebox/plugins/forumdl/templates/fullscreen.html b/archivebox/plugins/forumdl/templates/full.html
similarity index 100%
rename from archivebox/plugins/forumdl/templates/fullscreen.html
rename to archivebox/plugins/forumdl/templates/full.html
diff --git a/archivebox/plugins/gallerydl/templates/thumbnail.html b/archivebox/plugins/gallerydl/templates/card.html
similarity index 100%
rename from archivebox/plugins/gallerydl/templates/thumbnail.html
rename to archivebox/plugins/gallerydl/templates/card.html
diff --git a/archivebox/plugins/gallerydl/templates/embed.html b/archivebox/plugins/gallerydl/templates/embed.html
deleted file mode 100644
index b8e10f2f..00000000
--- a/archivebox/plugins/gallerydl/templates/embed.html
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
diff --git a/archivebox/plugins/gallerydl/templates/fullscreen.html b/archivebox/plugins/gallerydl/templates/full.html
similarity index 100%
rename from archivebox/plugins/gallerydl/templates/fullscreen.html
rename to archivebox/plugins/gallerydl/templates/full.html
diff --git a/archivebox/plugins/git/templates/thumbnail.html b/archivebox/plugins/git/templates/card.html
similarity index 100%
rename from archivebox/plugins/git/templates/thumbnail.html
rename to archivebox/plugins/git/templates/card.html
diff --git a/archivebox/plugins/mercury/templates/thumbnail.html b/archivebox/plugins/mercury/templates/card.html
similarity index 100%
rename from archivebox/plugins/mercury/templates/thumbnail.html
rename to archivebox/plugins/mercury/templates/card.html
diff --git a/archivebox/plugins/papersdl/templates/thumbnail.html b/archivebox/plugins/papersdl/templates/card.html
similarity index 100%
rename from archivebox/plugins/papersdl/templates/thumbnail.html
rename to archivebox/plugins/papersdl/templates/card.html
diff --git a/archivebox/plugins/papersdl/templates/embed.html b/archivebox/plugins/papersdl/templates/embed.html
deleted file mode 100644
index 45ef7d71..00000000
--- a/archivebox/plugins/papersdl/templates/embed.html
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
diff --git a/archivebox/plugins/papersdl/templates/fullscreen.html b/archivebox/plugins/papersdl/templates/full.html
similarity index 100%
rename from archivebox/plugins/papersdl/templates/fullscreen.html
rename to archivebox/plugins/papersdl/templates/full.html
diff --git a/archivebox/plugins/parse_html_urls/on_Snapshot__70_parse_html_urls.py b/archivebox/plugins/parse_html_urls/on_Snapshot__70_parse_html_urls.py
index 1fc36552..462c72f0 100755
--- a/archivebox/plugins/parse_html_urls/on_Snapshot__70_parse_html_urls.py
+++ b/archivebox/plugins/parse_html_urls/on_Snapshot__70_parse_html_urls.py
@@ -271,12 +271,11 @@ def main(url: str, snapshot_id: str = None, crawl_id: str = None, depth: int = 0
records.append(record)
print(json.dumps(record))
- if records:
- URLS_FILE.write_text('\n'.join(json.dumps(r) for r in records) + '\n')
+ URLS_FILE.write_text('\n'.join(json.dumps(r) for r in records) + ('\n' if records else ''))
# Emit ArchiveResult record to mark completion
status = 'succeeded' if urls_found else 'skipped'
- output_str = f'Found {len(urls_found)} URLs' if urls_found else 'No URLs found'
+ output_str = URLS_FILE.name
ar_record = {
'type': 'ArchiveResult',
'status': status,
diff --git a/archivebox/plugins/parse_html_urls/tests/test_parse_html_urls.py b/archivebox/plugins/parse_html_urls/tests/test_parse_html_urls.py
index 96835493..8dbef37e 100644
--- a/archivebox/plugins/parse_html_urls/tests/test_parse_html_urls.py
+++ b/archivebox/plugins/parse_html_urls/tests/test_parse_html_urls.py
@@ -57,7 +57,7 @@ class TestParseHtmlUrls:
)
assert result.returncode == 0
- assert 'Found 3 URLs' in result.stderr
+ assert 'urls.jsonl' in result.stderr
# Parse Snapshot records from stdout
lines = [line for line in result.stdout.strip().split('\n') if line.strip() and '"type": "Snapshot"' in line]
@@ -78,6 +78,11 @@ class TestParseHtmlUrls:
assert '"type": "ArchiveResult"' in result.stdout
assert '"status": "succeeded"' in result.stdout
+ urls_file = tmp_path / 'urls.jsonl'
+ assert urls_file.exists(), "urls.jsonl not created"
+ file_lines = [line for line in urls_file.read_text().splitlines() if line.strip()]
+ assert len(file_lines) == 3, f"Expected 3 urls.jsonl entries, got {len(file_lines)}"
+
def test_ignores_non_http_schemes(self, tmp_path):
"""Test that non-http schemes are ignored."""
input_file = tmp_path / 'page.html'
@@ -194,7 +199,7 @@ class TestParseHtmlUrls:
)
assert result.returncode == 0
- assert 'No URLs found' in result.stderr
+ assert 'urls.jsonl' in result.stderr
assert '"status": "skipped"' in result.stdout
def test_handles_malformed_html(self, tmp_path):
diff --git a/archivebox/plugins/parse_jsonl_urls/on_Snapshot__74_parse_jsonl_urls.py b/archivebox/plugins/parse_jsonl_urls/on_Snapshot__74_parse_jsonl_urls.py
index 086c7f10..f9c060dc 100755
--- a/archivebox/plugins/parse_jsonl_urls/on_Snapshot__74_parse_jsonl_urls.py
+++ b/archivebox/plugins/parse_jsonl_urls/on_Snapshot__74_parse_jsonl_urls.py
@@ -18,6 +18,7 @@ Supports various field names for URL, title, timestamp, and tags.
import json
import os
import sys
+from pathlib import Path
from datetime import datetime
from html import unescape
from urllib.parse import urlparse
@@ -25,6 +26,7 @@ from urllib.parse import urlparse
import rich_click as click
PLUGIN_NAME = 'parse_jsonl_urls'
+URLS_FILE = Path('urls.jsonl')
def parse_bookmarked_at(link: dict) -> str | None:
@@ -188,9 +190,12 @@ def main(url: str, snapshot_id: str = None, crawl_id: str = None, depth: int = 0
for entry in urls_found:
print(json.dumps(entry))
+ # Write urls.jsonl to disk for crawl system
+ URLS_FILE.write_text('\n'.join(json.dumps(r) for r in urls_found) + ('\n' if urls_found else ''))
+
# Emit ArchiveResult record to mark completion
status = 'succeeded' if urls_found else 'skipped'
- output_str = f'Found {len(urls_found)} URLs, {len(all_tags)} tags' if urls_found else 'No URLs found'
+ output_str = URLS_FILE.name
ar_record = {
'type': 'ArchiveResult',
'status': status,
diff --git a/archivebox/plugins/parse_jsonl_urls/tests/test_parse_jsonl_urls.py b/archivebox/plugins/parse_jsonl_urls/tests/test_parse_jsonl_urls.py
index 39244ede..b425d3f3 100644
--- a/archivebox/plugins/parse_jsonl_urls/tests/test_parse_jsonl_urls.py
+++ b/archivebox/plugins/parse_jsonl_urls/tests/test_parse_jsonl_urls.py
@@ -32,7 +32,7 @@ class TestParseJsonlUrls:
)
assert result.returncode == 0
- assert 'Found 3 URLs' in result.stdout
+ assert 'urls.jsonl' in result.stderr or 'urls.jsonl' in result.stdout
# Output goes to stdout (JSONL)
lines = [line for line in result.stdout.strip().split('\n') if line.strip() and '\"type\": \"Snapshot\"' in line]
@@ -195,7 +195,7 @@ class TestParseJsonlUrls:
)
assert result.returncode == 0
- assert 'No URLs found' in result.stderr
+ assert 'urls.jsonl' in result.stderr
assert '"status": "skipped"' in result.stdout
def test_exits_1_when_file_not_found(self, tmp_path):
diff --git a/archivebox/plugins/parse_netscape_urls/on_Snapshot__73_parse_netscape_urls.py b/archivebox/plugins/parse_netscape_urls/on_Snapshot__73_parse_netscape_urls.py
index 99e3c8c1..1627e919 100755
--- a/archivebox/plugins/parse_netscape_urls/on_Snapshot__73_parse_netscape_urls.py
+++ b/archivebox/plugins/parse_netscape_urls/on_Snapshot__73_parse_netscape_urls.py
@@ -16,6 +16,7 @@ import json
import os
import re
import sys
+from pathlib import Path
from datetime import datetime, timezone
from html import unescape
from urllib.parse import urlparse
@@ -23,6 +24,7 @@ from urllib.parse import urlparse
import rich_click as click
PLUGIN_NAME = 'parse_netscape_urls'
+URLS_FILE = Path('urls.jsonl')
# Constants for timestamp epoch detection
UNIX_EPOCH = 0 # 1970-01-01 00:00:00 UTC
@@ -232,9 +234,12 @@ def main(url: str, snapshot_id: str = None, crawl_id: str = None, depth: int = 0
for entry in urls_found:
print(json.dumps(entry))
+ # Write urls.jsonl to disk for crawl system
+ URLS_FILE.write_text('\n'.join(json.dumps(r) for r in urls_found) + ('\n' if urls_found else ''))
+
# Emit ArchiveResult record to mark completion
status = 'succeeded' if urls_found else 'skipped'
- output_str = f'Found {len(urls_found)} URLs, {len(all_tags)} tags' if urls_found else 'No bookmarks found'
+ output_str = URLS_FILE.name
ar_record = {
'type': 'ArchiveResult',
'status': status,
diff --git a/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls.py b/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls.py
index e8fefc64..43754b59 100644
--- a/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls.py
+++ b/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls.py
@@ -37,7 +37,7 @@ class TestParseNetscapeUrls:
)
assert result.returncode == 0
- assert 'Found 3 URLs' in result.stdout
+ assert 'urls.jsonl' in result.stderr or 'urls.jsonl' in result.stdout
# Output goes to stdout (JSONL)
lines = [line for line in result.stdout.strip().split('\n') if line.strip() and '\"type\": \"Snapshot\"' in line]
@@ -135,7 +135,7 @@ class TestParseNetscapeUrls:
)
assert result.returncode == 0
- assert 'No bookmarks found' in result.stderr
+ assert 'urls.jsonl' in result.stderr
assert '"status": "skipped"' in result.stdout
def test_exits_1_when_file_not_found(self, tmp_path):
diff --git a/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls_comprehensive.py b/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls_comprehensive.py
index 6dd5576c..402b823f 100644
--- a/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls_comprehensive.py
+++ b/archivebox/plugins/parse_netscape_urls/tests/test_parse_netscape_urls_comprehensive.py
@@ -935,7 +935,7 @@ class TestEdgeCases:
)
assert result.returncode == 0
- assert 'Found 1000 URLs' in result.stdout
+ assert 'urls.jsonl' in result.stderr or 'urls.jsonl' in result.stdout
# Output goes to stdout (JSONL) - get all JSONL records
all_lines = [line for line in result.stdout.strip().split('\n') if line.strip() and line.startswith('{')]
diff --git a/archivebox/plugins/parse_rss_urls/on_Snapshot__72_parse_rss_urls.py b/archivebox/plugins/parse_rss_urls/on_Snapshot__72_parse_rss_urls.py
index bdc50afa..dbbaccd4 100755
--- a/archivebox/plugins/parse_rss_urls/on_Snapshot__72_parse_rss_urls.py
+++ b/archivebox/plugins/parse_rss_urls/on_Snapshot__72_parse_rss_urls.py
@@ -16,6 +16,7 @@ Examples:
import json
import os
import sys
+from pathlib import Path
from datetime import datetime, timezone
from html import unescape
from time import mktime
@@ -24,6 +25,7 @@ from urllib.parse import urlparse
import rich_click as click
PLUGIN_NAME = 'parse_rss_urls'
+URLS_FILE = Path('urls.jsonl')
try:
import feedparser
@@ -140,9 +142,12 @@ def main(url: str, snapshot_id: str = None, crawl_id: str = None, depth: int = 0
for entry in urls_found:
print(json.dumps(entry))
+ # Write urls.jsonl to disk for crawl system
+ URLS_FILE.write_text('\n'.join(json.dumps(r) for r in urls_found) + ('\n' if urls_found else ''))
+
# Emit ArchiveResult record to mark completion
status = 'succeeded' if urls_found else 'skipped'
- output_str = f'Found {len(urls_found)} URLs, {len(all_tags)} tags' if urls_found else 'No URLs found'
+ output_str = URLS_FILE.name
ar_record = {
'type': 'ArchiveResult',
'status': status,
diff --git a/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls.py b/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls.py
index 950a2252..3cd54f60 100644
--- a/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls.py
+++ b/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls.py
@@ -66,7 +66,7 @@ class TestParseRssUrls:
)
assert result.returncode == 0
- assert 'Found 2 URLs' in result.stdout
+ assert 'urls.jsonl' in result.stderr or 'urls.jsonl' in result.stdout
# Output goes to stdout (JSONL)
lines = [line for line in result.stdout.strip().split('\n') if line.strip() and '\"type\": \"Snapshot\"' in line]
@@ -134,7 +134,7 @@ class TestParseRssUrls:
)
assert result.returncode == 0
- assert 'No URLs found' in result.stderr
+ assert 'urls.jsonl' in result.stderr
assert '"status": "skipped"' in result.stdout
def test_exits_1_when_file_not_found(self, tmp_path):
diff --git a/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls_comprehensive.py b/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls_comprehensive.py
index 2c0e17d7..fbc415f9 100644
--- a/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls_comprehensive.py
+++ b/archivebox/plugins/parse_rss_urls/tests/test_parse_rss_urls_comprehensive.py
@@ -882,7 +882,7 @@ class TestEdgeCases:
)
assert result.returncode == 0
- assert 'Found 100 URLs' in result.stdout
+ assert 'urls.jsonl' in result.stderr or 'urls.jsonl' in result.stdout
# Output goes to stdout (JSONL)
lines = [line for line in result.stdout.strip().split('\n') if line.strip()]
diff --git a/archivebox/plugins/parse_txt_urls/on_Snapshot__71_parse_txt_urls.py b/archivebox/plugins/parse_txt_urls/on_Snapshot__71_parse_txt_urls.py
index 29265700..4ee3c4b2 100755
--- a/archivebox/plugins/parse_txt_urls/on_Snapshot__71_parse_txt_urls.py
+++ b/archivebox/plugins/parse_txt_urls/on_Snapshot__71_parse_txt_urls.py
@@ -26,6 +26,7 @@ from urllib.request import urlopen
import rich_click as click
PLUGIN_NAME = 'parse_txt_urls'
+URLS_FILE = Path('urls.jsonl')
# URL regex from archivebox/misc/util.py
# https://mathiasbynens.be/demo/url-regex
@@ -127,6 +128,7 @@ def main(url: str, snapshot_id: str = None, crawl_id: str = None, depth: int = 0
urls_found.add(cleaned_url)
# Emit Snapshot records to stdout (JSONL)
+ records = []
for found_url in sorted(urls_found):
record = {
'type': 'Snapshot',
@@ -138,11 +140,13 @@ def main(url: str, snapshot_id: str = None, crawl_id: str = None, depth: int = 0
record['parent_snapshot_id'] = snapshot_id
if crawl_id:
record['crawl_id'] = crawl_id
+ records.append(record)
print(json.dumps(record))
# Emit ArchiveResult record to mark completion
+ URLS_FILE.write_text('\n'.join(json.dumps(r) for r in records) + ('\n' if records else ''))
status = 'succeeded' if urls_found else 'skipped'
- output_str = f'Found {len(urls_found)} URLs' if urls_found else 'No URLs found'
+ output_str = URLS_FILE.name
ar_record = {
'type': 'ArchiveResult',
'status': status,
diff --git a/archivebox/plugins/parse_txt_urls/tests/test_parse_txt_urls.py b/archivebox/plugins/parse_txt_urls/tests/test_parse_txt_urls.py
index 82eccd8e..a3b53289 100644
--- a/archivebox/plugins/parse_txt_urls/tests/test_parse_txt_urls.py
+++ b/archivebox/plugins/parse_txt_urls/tests/test_parse_txt_urls.py
@@ -32,7 +32,7 @@ https://www.iana.org/domains/reserved
)
assert result.returncode == 0, f"Failed: {result.stderr}"
- assert 'Found 3 URLs' in result.stderr
+ assert 'urls.jsonl' in result.stderr
# Parse Snapshot records from stdout
lines = [line for line in result.stdout.strip().split('\n') if line.strip() and '"type": "Snapshot"' in line]
@@ -113,7 +113,7 @@ Also see https://github.com/user/repo for the code.
)
assert result.returncode == 0
- assert 'No URLs found' in result.stderr
+ assert 'urls.jsonl' in result.stderr
assert '"status": "skipped"' in result.stdout
def test_exits_1_when_file_not_found(self, tmp_path):
diff --git a/archivebox/plugins/pdf/templates/thumbnail.html b/archivebox/plugins/pdf/templates/card.html
similarity index 100%
rename from archivebox/plugins/pdf/templates/thumbnail.html
rename to archivebox/plugins/pdf/templates/card.html
diff --git a/archivebox/plugins/pdf/templates/embed.html b/archivebox/plugins/pdf/templates/embed.html
deleted file mode 100644
index 732a01bc..00000000
--- a/archivebox/plugins/pdf/templates/embed.html
+++ /dev/null
@@ -1,5 +0,0 @@
-
- |
{% if link.is_archived %}
- 
+ 
{% else %}
- 
+
{% endif %}
-
+
{{link.title|default:'Loading...'|truncatechars:128}}
@@ -29,7 +29,7 @@
{% if link.icons %}
{{link.icons}} {{link.num_outputs}}
{% else %}
-
+
📄
{{link.num_outputs}}
diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html
index 1b6b2bbd..f5cac194 100644
--- a/archivebox/templates/core/snapshot.html
+++ b/archivebox/templates/core/snapshot.html
@@ -113,6 +113,10 @@
border-radius: 10px;
background-color: black;
overflow: hidden;
+ min-height: 130px;
+ }
+ .header-bottom-frames .card:has([data-compact]) {
+ min-height: 0;
}
.card h4 {
font-size: 1.4vw;
@@ -154,6 +158,202 @@
transform: scale(0.25);
transform-origin: 0 0;
}
+ .row.header-bottom-frames {
+ display: block !important;
+ width: 100%;
+ max-width: 100%;
+ column-width: 180px;
+ column-gap: 8px;
+ column-fill: auto;
+ margin-left: 0px;
+ margin-right: 0px;
+ flex: none !important;
+ }
+ .header-bottom-frames .col-lg-2 {
+ padding-left: 0px;
+ padding-right: 0px;
+ max-width: 100%;
+ width: 100% !important;
+ display: inline-block !important;
+ float: none !important;
+ flex: none !important;
+ break-inside: avoid;
+ margin-bottom: 6px;
+ vertical-align: top;
+ }
+ .header-bottom-frames .card:has([data-compact]) .thumbnail-wrapper,
+ .header-bottom-frames .card:has([data-compact]) .thumbnail-wrapper.compact {
+ height: 32px;
+ }
+ .header-bottom-frames .card:has([data-compact]) .thumbnail-text {
+ height: auto;
+ max-height: 64px;
+ }
+ .header-bottom-frames .card:has([data-compact]) .card-body {
+ padding: 4px 8px;
+ max-height: 44px;
+ }
+ .thumbnail-wrapper {
+ height: 100px;
+ overflow: hidden;
+ background: #333;
+ }
+ .thumbnail-compact {
+ height: 32px;
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ padding: 0 8px;
+ font-size: 13px;
+ line-height: 1;
+ color: #bdbdbd;
+ background: #111;
+ border-bottom: 1px solid #222;
+ text-transform: uppercase;
+ letter-spacing: 0.02em;
+ }
+ .thumbnail-compact-label {
+ color: #e1e1e1;
+ }
+ .thumbnail-compact-meta {
+ color: #777;
+ font-size: 11px;
+ margin-left: auto;
+ }
+ .thumbnail-compact svg,
+ .thumbnail-compact img {
+ height: 12px;
+ width: 12px;
+ }
+ .thumbnail-text {
+ height: 100px;
+ background: #121212;
+ color: #d8d8d8;
+ padding: 6px 8px;
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
+ font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
+ font-size: 11px;
+ line-height: 1.2;
+ overflow: hidden;
+ }
+ .thumbnail-text-header {
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ font-size: 10px;
+ color: #9b9b9b;
+ text-transform: uppercase;
+ letter-spacing: 0.04em;
+ }
+ .thumbnail-text-pre {
+ margin: 0;
+ white-space: pre-wrap;
+ overflow: hidden;
+ display: -webkit-box;
+ -webkit-box-orient: vertical;
+ -webkit-line-clamp: 5;
+ }
+ .thumbnail-text[data-plugin="title"] .thumbnail-text-pre {
+ font-size: 13px;
+ font-weight: 600;
+ -webkit-line-clamp: 3;
+ }
+ .thumb-grid {
+ display: block;
+ column-width: 180px;
+ column-gap: 6px;
+ align-content: start;
+ width: 100%;
+ }
+ .thumb-card {
+ box-shadow: 2px 3px 14px 0px rgba(0,0,0,0.02);
+ border: 1px solid rgba(0,0,0,3);
+ border-radius: 10px;
+ background-color: black;
+ overflow: hidden;
+ display: inline-block;
+ width: 100%;
+ break-inside: avoid;
+ box-sizing: border-box;
+ margin-bottom: 6px;
+ height: 138px;
+ min-height: 138px;
+ max-height: 138px;
+ }
+ .thumb-card:has([data-compact]) {
+ height: 46px;
+ min-height: 46px;
+ max-height: 46px;
+ }
+ .thumb-card .thumb-body {
+ font-size: 14px;
+ padding: 3px 8px;
+ line-height: 1.2;
+ word-wrap: break-word;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ background-color: #1a1a1a;
+ color: #d3d3d3;
+ }
+ .thumb-card .thumb-body h4 {
+ font-size: 1.1em;
+ margin: 0 0 2px 0;
+ line-height: 1.1;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ }
+ .thumb-card .thumbnail-wrapper,
+ .thumb-card iframe.card-img-top {
+ display: block;
+ width: 100%;
+ }
+ .thumb-card:has([data-compact]) .thumbnail-wrapper,
+ .thumb-card:has([data-compact]) .thumbnail-wrapper.compact {
+ height: 24px;
+ flex: 0 0 auto;
+ }
+ .thumb-card:has([data-compact]) .thumb-body {
+ padding: 2px 6px;
+ font-size: 12px;
+ max-height: 20px;
+ }
+ .thumb-card:has([data-compact]) .thumb-body h4 {
+ font-size: 0.9em;
+ margin-bottom: 0px;
+ line-height: 1;
+ display: flex;
+ align-items: center;
+ gap: 4px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ }
+ .thumb-card.selected-card {
+ border: 2px solid orange;
+ box-shadow: 0px -6px 13px 1px rgba(0,0,0,0.05);
+ }
+ .thumb-compact .thumbnail-wrapper {
+ height: 32px;
+ }
+ .thumb-compact {
+ margin-bottom: 0px;
+ border-radius: 6px;
+ }
+ .thumb-compact .card-body {
+ display: block;
+ padding: 4px 8px;
+ font-size: 12px;
+ line-height: 1.2;
+ max-height: none;
+ }
+ .thumb-compact .thumbnail-compact,
+ .thumb-compact .thumbnail-text {
+ height: 32px;
+ max-height: 32px;
+ }
.full-page-iframe {
border-top: 1px solid #ddd;
width: 100%;
@@ -203,6 +403,10 @@
box-shadow: 4px 4px 4px rgba(0,0,0,0.2);
margin-top: 0px;
}
+ .header-bottom.container-fluid {
+ padding-left: 6px;
+ padding-right: 6px;
+ }
.header-bottom-info {
color: #6f6f6f;
padding-top: 0px;
@@ -357,15 +561,15 @@
- |