feat: refresh public snapshot index

This commit is contained in:
Nick Sweeting 2026-05-27 11:03:13 -07:00
parent 5fbd6383d8
commit b37dd20d21
No known key found for this signature in database
3 changed files with 537 additions and 146 deletions

View File

@ -943,6 +943,10 @@ class PublicIndexView(ListView):
for snapshot in context.get("object_list") or ():
snapshot._icons_compact = True
snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
results = getattr(snapshot, "_prefetched_objects_cache", {}).get("archiveresult_set")
if results is not None:
snapshot.output_size_sum = sum(result.output_size or result.output_size_from_files() for result in results)
snapshot.num_outputs_cached = len(results)
return context
def get_queryset(self, **kwargs):
@ -959,6 +963,8 @@ class PublicIndexView(ListView):
"snapshot_id",
"plugin",
"status",
"output_size",
"output_files",
),
),
)
@ -968,49 +974,24 @@ class PublicIndexView(ListView):
if not query:
return qs
query_type = self.request.GET.get("query_type")
search_mode = get_search_mode(self.request.GET.get("search_mode"))
if not query_type or query_type == "all":
metadata_qs = qs.filter(
Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
)
if search_mode == "meta":
qs = metadata_qs
else:
try:
qs = prioritize_metadata_matches(
qs,
metadata_qs,
query_search_index(query, search_mode=search_mode),
ordering=self.ordering,
)
except Exception as err:
print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
qs = metadata_qs
elif query_type == "fulltext":
if search_mode == "meta":
qs = qs.none()
else:
try:
qs = query_search_index(query, search_mode=search_mode).filter(pk__in=qs.values("pk"))
except Exception as err:
print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
qs = qs.none()
elif query_type == "meta":
qs = qs.filter(
Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
)
elif query_type == "url":
qs = qs.filter(Q(url__icontains=query))
elif query_type == "title":
qs = qs.filter(Q(title__icontains=query))
elif query_type == "timestamp":
qs = qs.filter(Q(timestamp__icontains=query))
elif query_type == "tags":
qs = qs.filter(Q(tags__name__icontains=query))
metadata_qs = qs.filter(
Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
)
if search_mode == "meta":
qs = metadata_qs
else:
print(f'[!] Unknown value for query_type: "{query_type}"')
try:
qs = prioritize_metadata_matches(
qs,
metadata_qs,
query_search_index(query, search_mode=search_mode),
ordering=self.ordering,
)
except Exception as err:
print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
qs = metadata_qs
return qs.distinct()

View File

@ -1,44 +1,89 @@
{% load static tz core_tags %}
<tr>
<td title="Bookmarked: {{link.bookmarked_date|localtime}} ({{link.timestamp}})" data-sort="{{link.bookmarked_at.timestamp}}">
{{ link.bookmarked_at|localtime }}
</td>
<td class="title-col" style="opacity: {% if link.title %}1{% else %}0.3{% endif %}" title="{{link.title|default:'Not yet archived...'}}">
{% if link.is_archived %}
<a href="{{ WEB_BASE_URL }}/{{link.archive_path}}/index.html"><img src="{% snapshot_url link 'favicon.ico' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async"></a>
{% else %}
<a href="{{ WEB_BASE_URL }}/{{link.archive_path}}/index.html"><img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="link-favicon" decoding="async" style="height: 15px"></a>
{% endif %}
{% snapshot_base_url link as snapshot_base %}
{% snapshot_url link 'screenshot/screenshot.png' as screenshot_plugin_url %}
{% snapshot_url link 'screenshot.png' as screenshot_root_url %}
{% snapshot_url link 'favicon/favicon.ico' as favicon_plugin_url %}
{% snapshot_url link 'favicon.ico' as favicon_root_url %}
<a href="{{ WEB_BASE_URL }}/{{link.archive_path}}/index.html" title="{{link.title|default:'Not yet archived...'}}">
<span data-title-for="{{link.url}}" data-archived="{{link.is_archived}}">
{{link.title|default:'Loading...'|truncatechars:128}}
</span>
{% if link.tags_str %}
{% for tag in link.tags_str|split:',' %}
<span class="tag" style="float: right; border-radius: 5px; background-color: #bfdfff; padding: 2px 5px; margin-left: 4px; margin-top: 1px;">
{{tag}}
</span>
{% endfor %}
<tr class="snapshot-row status-{{ link.status|default:'unknown' }}">
<td class="snapshot-time" title="Bookmarked: {{ link.bookmarked_at|localtime }} ({{ link.timestamp }})" data-sort="{{ link.bookmarked_at.timestamp }}">
<a href="{{ snapshot_base }}/index.html">
<span>{{ link.bookmarked_at|localtime|date:"Y-m-d" }}</span>
<small>{{ link.bookmarked_at|localtime|date:"H:i" }}</small>
</a>
</td>
<td class="snapshot-preview-cell">
<a href="{{ snapshot_base }}/index.html" title="Open archived snapshot">
{% if link.is_archived %}
<img src="{{ screenshot_plugin_url }}"
data-fallbacks="{{ screenshot_root_url }},{{ favicon_plugin_url }},{{ favicon_root_url }}"
onerror="nextPublicSnapshotPreview(this)"
class="snapshot-preview screenshot"
alt=""
decoding="async"
loading="lazy">
{% else %}
<img src="{% static 'spinner.gif' %}"
class="snapshot-preview"
alt=""
decoding="async"
loading="lazy">
{% endif %}
</a>
</td>
<td>
<span data-number-for="{{link.url}}" title="Fetching any missing files...">
{% if link.icons %}
{{link.icons}}&nbsp; <small style="float:right; opacity: 0.5">{{link.num_outputs}}</small>
<td class="snapshot-title-cell" title="{{ link.title|default:link.url }}">
<div class="snapshot-title-line">
<a href="{{ snapshot_base }}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
<img src="{{ favicon_plugin_url }}"
data-fallbacks="{{ favicon_root_url }}"
onerror="nextPublicSnapshotPreview(this)"
class="link-favicon"
alt=""
decoding="async"
loading="lazy">
</a>
<a href="{{ snapshot_base }}/index.html" class="snapshot-title">
{{ link.title|default:'Loading...'|truncatechars:110 }}
</a>
</div>
<a href="{{ link.url }}" class="snapshot-url" title="{{ link.url }}" target="_blank" rel="noopener noreferrer">
{{ link.url }}
</a>
</td>
<td class="snapshot-tags-cell">
{% if link.tags_str %}
<span class="snapshot-tags">
{% for tag in link.tags_str|split:',' %}
{% if tag %}
<span class="snapshot-tag">{{ tag }}</span>
{% endif %}
{% endfor %}
</span>
{% else %}
<span class="empty-value">...</span>
{% endif %}
</td>
<td class="snapshot-status-cell">
<span class="snapshot-status">{{ link.status|default:'unknown' }}</span>
</td>
<td class="snapshot-files-cell">
<span data-number-for="{{ link.url }}" title="{{ link.num_outputs }} successful outputs">
{% if link.icons %}
{{ link.icons }}
{% else %}
<a href="{{ WEB_BASE_URL }}/{{link.archive_path}}/index.html">
📄 &nbsp;
{{link.num_outputs}} <img src="{% static 'spinner.gif' %}" onerror="this.style.display='none'" class="files-spinner" decoding="async" style="height: 15px"/>
</a>
<span class="empty-value">...</span>
{% endif %}
</span>
</td>
<td style="text-align:left; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;" title="{{link.url}}">
<a href="{{link.url}}">
{{link.url}}
<td class="snapshot-size-cell">
<a href="{{ snapshot_base }}/?files=1" title="View archived files">
{% if link.archive_size %}
{{ link.archive_size|file_size }}
{% else %}
<span class="empty-value">...</span>
{% endif %}
</a>
<small>{{ link.num_outputs }} output{{ link.num_outputs|pluralize }}</small>
</td>
</tr>

View File

@ -2,79 +2,444 @@
{% load static tz %}
{% load core_tags %}
{% block body %}
<div id="toolbar">
<form id="changelist-search" action="{{ request.path }}" method="get">
<div>
<label for="searchbar"><img src="/static/admin/img/search.svg" alt="Search"></label>
<select name="query_type" id="query_type">
<option {% if request.GET.query_type == 'all' %}selected{% endif %} value="all">All</option>
<option {% if request.GET.query_type == 'fulltext' %}selected{% endif %} value="fulltext">Content</option>
<option {% if request.GET.query_type == 'meta' %}selected{% endif %} value="meta">Metadata</option>
<option {% if request.GET.query_type == 'url' %}selected{% endif %} value="url">URL</option>
<option {% if request.GET.query_type == 'title' %}selected{% endif %} value="title">Title</option>
<option {% if request.GET.query_type == 'timestamp' %}selected{% endif %} value="timestamp">Timestamp</option>
<option {% if request.GET.query_type == 'tags' %}selected{% endif %} value="tags">Tags</option>
</select>
<input type="text" size="40" name="q" value="{{ request.GET.q }}" id="searchbar" autofocus placeholder="Title, URL, tags, timestamp, or content...".>
<input type="submit" value="Search" style="height: 36px; padding-top: 6px; margin: 8px"/>
<span style="display:inline-flex; flex-wrap:wrap; gap:8px; align-items:center; margin-left:4px; font-size:12px;">
<label><input type="radio" name="search_mode" value="meta"{% if search_mode == "meta" %} checked{% endif %}> meta (db-only)</label>
<label><input type="radio" name="search_mode" value="contents"{% if search_mode == "contents" %} checked{% endif %}> contents (ripgrep/sonic/etc.)</label>
<label><input type="radio" name="search_mode" value="deep"{% if search_mode == "deep" %} checked{% endif %}> deep</label>
</span>
<input type="button"
value="♺"
title="Refresh..."
onclick="location.href='{{ request.get_full_path }}'"
style="background-color: rgba(121, 174, 200, 0.8); height: 30px; font-size: 0.8em; margin-top: 12px; padding-top: 6px; float:right">
</input>
&nbsp;
&nbsp;
{{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
&nbsp;
(Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})
</div>
</form>
</div>
<div style="width: 100%; overflow-x: auto;">
<table id="table-bookmarks" style="width: 100%; table-layout: fixed">
<thead>
<tr>
<th style="width: 132px">Bookmarked</th>
<th>Snapshot ({{page_obj.paginator.count}})</th>
<th style="width: 280px">Files</th>
<th>Original URL</th>
</tr>
</thead>
<tbody>
{% for link in object_list %}
{% include 'index_row.html' with link=link %}
{% endfor %}
</tbody>
</table>
</div>
<br/>
<center>
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
<br/>
<span class="step-links">
{% if page_obj.has_previous %}
<a href="?{% url_replace page='1' %}">&laquo; first</a> &nbsp;
<a href="?{% url_replace page=page_obj.previous_page_number %}">previous</a>
&nbsp;
{% endif %}
<span class="current">
Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
</span>
{% if page_obj.has_next %}
&nbsp;
<a href="?{% url_replace page=page_obj.next_page_number %}">next </a> &nbsp;
<a href="?{% url_replace page=page_obj.paginator.num_pages %}">last &raquo;</a>
{% endif %}
</span>
<br>
</center>
{% block extra_head %}
<style>
#content.flex {
display: block;
}
.public-snapshot-list {
width: 100%;
}
.public-snapshot-toolbar {
display: flex;
align-items: center;
justify-content: space-between;
gap: 14px;
margin: 0 0 14px;
padding: 10px 12px;
border: 1px solid #e2e8f0;
border-radius: 10px;
background: #fff;
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.04);
}
.public-snapshot-toolbar form {
display: flex;
align-items: center;
gap: 8px;
min-width: 0;
flex: 1;
margin: 0;
}
.public-snapshot-toolbar label.search-icon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 24px;
height: 24px;
margin: 0;
opacity: 0.65;
}
.public-snapshot-toolbar #searchbar {
flex: 1 1 320px;
min-width: 180px;
height: 34px;
padding: 6px 10px;
border: 1px solid #cbd5e1;
border-radius: 7px;
font-size: 13px;
}
.public-snapshot-toolbar input[type="submit"] {
height: 34px;
margin: 0;
padding: 0 14px;
border-radius: 7px;
}
.public-search-modes {
display: inline-flex;
align-items: center;
gap: 2px;
padding: 2px;
border: 1px solid #dbe3ee;
border-radius: 8px;
background: #f8fafc;
white-space: nowrap;
}
.public-search-modes label {
display: inline-flex;
align-items: center;
gap: 5px;
margin: 0;
padding: 5px 8px;
border-radius: 6px;
color: #475569;
font-size: 12px;
line-height: 1;
cursor: pointer;
}
.public-search-modes label:has(input:checked) {
background: #fff;
color: #0f172a;
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08);
}
.public-search-modes input {
margin: 0;
}
.public-snapshot-count {
color: #64748b;
font-size: 12px;
white-space: nowrap;
}
.public-snapshot-table-wrap {
width: 100%;
overflow-x: auto;
border: 1px solid #e2e8f0;
border-radius: 10px;
background: #fff;
box-shadow: 0 1px 2px rgba(15, 23, 42, 0.04);
}
#table-bookmarks {
width: 100%;
min-width: 980px;
table-layout: fixed;
border-collapse: collapse;
}
#table-bookmarks thead th {
padding: 9px 10px;
border-bottom: 1px solid #e2e8f0;
background: #f8fafc;
color: #475569;
font-size: 11px;
font-weight: 700;
letter-spacing: 0.04em;
text-transform: uppercase;
}
#table-bookmarks tbody td {
padding: 8px 10px;
border-bottom: 1px solid #edf2f7;
vertical-align: middle;
}
#table-bookmarks tbody tr:hover {
background: #f8fafc;
}
#table-bookmarks tbody tr:last-child td {
border-bottom: 0;
}
.snapshot-time {
width: 102px;
white-space: nowrap;
}
.snapshot-time a {
display: inline-flex;
flex-direction: column;
color: #334155;
text-decoration: none;
line-height: 1.2;
}
.snapshot-time small {
color: #94a3b8;
font-size: 11px;
}
.snapshot-preview-cell {
width: 106px;
}
#content .snapshot-preview-cell .snapshot-preview {
width: 86px;
height: 54px;
max-width: 86px;
max-height: 54px;
margin: 0;
border: 1px solid #e2e8f0;
border-radius: 8px;
background: #f8fafc;
object-fit: cover;
object-position: top;
}
.snapshot-title-cell {
min-width: 0;
}
.snapshot-title-line {
display: flex;
align-items: center;
gap: 8px;
min-width: 0;
margin-bottom: 2px;
}
.snapshot-favicon-link {
display: inline-flex;
align-items: center;
justify-content: center;
width: 22px;
min-width: 22px;
height: 22px;
}
#content .snapshot-favicon-link .link-favicon {
display: block;
width: 18px;
height: 18px;
max-width: 18px;
border-radius: 4px;
object-fit: contain;
}
.snapshot-title {
display: block;
min-width: 0;
overflow: hidden;
color: #0f172a;
font-size: 14px;
font-weight: 650;
line-height: 1.25;
text-overflow: ellipsis;
white-space: nowrap;
text-decoration: none;
}
.snapshot-url {
display: block;
margin-left: 30px;
overflow: hidden;
color: #64748b;
font-size: 12px;
line-height: 1.35;
text-overflow: ellipsis;
white-space: nowrap;
text-decoration: none;
}
.snapshot-url:hover,
.snapshot-title:hover {
text-decoration: underline;
}
.snapshot-tags-cell {
width: 190px;
}
.snapshot-tags {
display: flex;
flex-wrap: wrap;
gap: 4px;
max-height: 46px;
overflow: hidden;
}
.snapshot-tag {
display: inline-flex;
align-items: center;
max-width: 160px;
padding: 2px 7px;
border: 1px solid #bfdbfe;
border-radius: 999px;
background: #eff6ff;
color: #1d4ed8;
font-size: 11px;
font-weight: 600;
line-height: 1.35;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.snapshot-status-cell {
width: 96px;
}
.snapshot-status {
display: inline-flex;
align-items: center;
justify-content: center;
min-width: 68px;
padding: 3px 8px;
border-radius: 999px;
background: #f1f5f9;
color: #475569;
font-size: 11px;
font-weight: 700;
line-height: 1.2;
text-transform: uppercase;
}
.status-sealed .snapshot-status,
.status-succeeded .snapshot-status {
background: #d1fae5;
color: #047857;
}
.status-started .snapshot-status {
background: #dbeafe;
color: #1d4ed8;
}
.status-queued .snapshot-status,
.status-backoff .snapshot-status {
background: #fef3c7;
color: #b45309;
}
.status-failed .snapshot-status {
background: #fee2e2;
color: #b91c1c;
}
.snapshot-files-cell {
width: 220px;
overflow: hidden;
}
.snapshot-files-cell .files-icons {
max-width: 200px;
}
.snapshot-size-cell {
width: 102px;
white-space: nowrap;
}
.snapshot-size-cell a {
display: block;
color: #334155;
font-size: 12px;
font-weight: 700;
text-decoration: none;
}
.snapshot-size-cell small {
display: block;
color: #94a3b8;
font-size: 10px;
line-height: 1.2;
}
.empty-value {
color: #cbd5e1;
}
.public-pagination {
margin: 18px 0 0;
text-align: center;
color: #64748b;
font-size: 13px;
}
.public-pagination .step-links {
display: inline-flex;
gap: 12px;
align-items: center;
margin-top: 6px;
}
@media (max-width: 980px) {
.public-snapshot-toolbar,
.public-snapshot-toolbar form {
align-items: stretch;
flex-direction: column;
}
.public-snapshot-count {
align-self: flex-start;
}
.public-search-modes {
flex-wrap: wrap;
white-space: normal;
}
}
</style>
<script>
function nextPublicSnapshotPreview(img) {
const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean)
if (fallbacks.length) {
img.src = fallbacks.shift()
img.dataset.fallbacks = fallbacks.join(',')
return
}
img.style.opacity = 0
}
</script>
{% endblock %}
{% block body %}
<div class="public-snapshot-list">
<div class="public-snapshot-toolbar">
<form id="changelist-search" action="{{ request.path }}" method="get">
<label class="search-icon" for="searchbar"><img src="/static/admin/img/search.svg" alt="Search"></label>
<input type="text" size="40" name="q" value="{{ request.GET.q }}" id="searchbar" autofocus placeholder="Search title, URL, tags, timestamp, or content...">
<input type="submit" value="Search">
<span class="public-search-modes">
<label><input type="radio" name="search_mode" value="meta"{% if search_mode == "meta" %} checked{% endif %}> Meta</label>
<label><input type="radio" name="search_mode" value="contents"{% if search_mode == "contents" %} checked{% endif %}> Contents</label>
<label><input type="radio" name="search_mode" value="deep"{% if search_mode == "deep" %} checked{% endif %}> Deep</label>
</span>
</form>
<div class="public-snapshot-count">
{{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }}
<span>(page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})</span>
</div>
</div>
<div class="public-snapshot-table-wrap">
<table id="table-bookmarks">
<thead>
<tr>
<th class="snapshot-time">Saved</th>
<th class="snapshot-preview-cell">Preview</th>
<th>Snapshot ({{ page_obj.paginator.count }})</th>
<th class="snapshot-tags-cell">Tags</th>
<th class="snapshot-status-cell">Status</th>
<th class="snapshot-files-cell">Files</th>
<th class="snapshot-size-cell">Size</th>
</tr>
</thead>
<tbody>
{% for link in object_list %}
{% include 'index_row.html' with link=link %}
{% endfor %}
</tbody>
</table>
</div>
<div class="public-pagination">
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
<br>
<span class="step-links">
{% if page_obj.has_previous %}
<a href="?{% url_replace page='1' %}">&laquo; first</a>
<a href="?{% url_replace page=page_obj.previous_page_number %}">previous</a>
{% endif %}
<span class="current">
Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
</span>
{% if page_obj.has_next %}
<a href="?{% url_replace page=page_obj.next_page_number %}">next</a>
<a href="?{% url_replace page=page_obj.paginator.num_pages %}">last &raquo;</a>
{% endif %}
</span>
</div>
</div>
{% endblock %}