diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 3a3c39a1..5e8cdf3f 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -18,7 +18,7 @@ from django.utils.safestring import mark_safe from django.views import View from django.views.generic.list import ListView from django.views.generic import FormView -from django.db.models import Count, Q, Prefetch +from django.db.models import Count, Q, Prefetch, Sum from django.contrib import messages from django.contrib.auth.mixins import UserPassesTestMixin from django.views.decorators.csrf import csrf_exempt @@ -1573,6 +1573,14 @@ def live_progress_view(request): .annotate(count=Count("id")) ): cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"] + crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids} + if active_crawl_ids: + for row in ( + archiveresult_scope.filter(snapshot__crawl_id__in=active_crawl_ids) + .values("snapshot__crawl_id") + .annotate(total_size=Sum("output_size")) + ): + crawl_output_sizes_by_crawl[str(row["snapshot__crawl_id"])] = int(row["total_size"] or 0) if machine_id is not None: running_processes = Process.objects.filter( @@ -1943,6 +1951,8 @@ def live_progress_view(request): crawl_tags = [tag.strip() for tag in (crawl.tags_str or "").replace("\n", ",").split(",") if tag.strip()] persona_name = persona_names_by_id.get(str(crawl.persona_id)) if crawl.persona_id else None persona_name = persona_name or str((crawl.config or {}).get("DEFAULT_PERSONA") or "Default") + crawl_output_size = crawl_output_sizes_by_crawl.get(str(crawl.id), 0) + avg_snapshot_size = int(crawl_output_size / total_snapshots) if total_snapshots else 0 # Check if retry_at is in the future (would prevent worker from claiming) retry_at_future = crawl.retry_at > now if crawl.retry_at else False @@ -1974,6 +1984,10 @@ def live_progress_view(request): "max_snapshot_size": crawl.snapshot_max_size, "max_crawl_size_display": printable_filesize(crawl.crawl_max_size) if crawl.crawl_max_size else "unlimited", "max_snapshot_size_display": printable_filesize(crawl.snapshot_max_size) if crawl.snapshot_max_size else "unlimited", + "crawl_output_size": crawl_output_size, + "avg_snapshot_size": avg_snapshot_size, + "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B", + "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B", "tags": crawl_tags, "urls_count": urls_count, "total_snapshots": total_snapshots, diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index cfe72667..a045990a 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -998,20 +998,18 @@ `; } - // Show snapshot info or URL count if no snapshots yet - const maxUrlsText = (crawl.max_urls || 0) > 0 ? `${crawl.max_urls} max URLs` : 'all URLs'; - const crawlSizeText = crawl.max_crawl_size_display || 'unlimited'; - const snapshotSizeText = crawl.max_snapshot_size_display || 'unlimited'; - const itemCountText = (crawl.total_snapshots || 0) > 0 - ? `${crawl.total_snapshots} snapshot${(crawl.total_snapshots || 0) === 1 ? '' : 's'}` - : ((crawl.urls_count || 0) > 0 ? `${crawl.urls_count} URL${(crawl.urls_count || 0) === 1 ? '' : 's'}` : 'no URLs'); + // Show crawl-scale limits and approximate output sizes from DB metadata. + const currentUrlCount = Math.max(crawl.total_snapshots || 0, crawl.urls_count || 0); + const maxUrlsText = (crawl.max_urls || 0) > 0 ? crawl.max_urls : 'unlimited'; + const urlLimitText = `${currentUrlCount} / ${maxUrlsText}`; + const crawlSizeLimitText = `${crawl.crawl_output_size_display || '0 B'} / ${crawl.max_crawl_size_display || 'unlimited'}`; + const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`; const crawlBadges = [ `persona${escapeHtml(crawl.persona || 'Default')}`, `depth${crawl.max_depth || 0}`, - `urls${escapeHtml(maxUrlsText)}`, - `crawl${escapeHtml(crawlSizeText)}`, - `snapshot${escapeHtml(snapshotSizeText)}`, - `items${escapeHtml(itemCountText)}`, + `urls${escapeHtml(urlLimitText)}`, + `crawl size${escapeHtml(crawlSizeLimitText)}`, + `avg snap${escapeHtml(snapshotSizeLimitText)}`, ...(crawl.tags || []).map(tag => `#${escapeHtml(tag)}`), ].join(''); const statsHtml = [ diff --git a/bin/release.sh b/bin/release.sh index efe037fd..9ee61cf4 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -242,9 +242,42 @@ wait_for_runs() { sleep 10 done - while read -r run_id; do - gh run watch "${run_id}" --repo "${slug}" --exit-status - done < <(jq -r '.[].databaseId' <<<"${runs_json}") + while IFS=$'\t' read -r run_id workflow_name; do + workflow_name_lower="${workflow_name,,}" + if [[ "${workflow_name_lower}" == *"release state"* ]]; then + gh run watch "${run_id}" --repo "${slug}" --exit-status + continue + fi + if [[ "${workflow_name_lower}" != *"test"* ]]; then + echo "Skipping non-gating workflow: ${workflow_name}" + continue + fi + + attempts=0 + while :; do + precheck_state="$( + gh run view "${run_id}" --repo "${slug}" --json jobs --jq ' + [.jobs[] | select((.name | ascii_downcase) | test("precheck|pre-commit|prek"))][0] + | if . == null then "missing:" else ((.status // "") + ":" + (.conclusion // "")) end + ' + )" + case "${precheck_state}" in + completed:success|completed:skipped) + break + ;; + completed:failure|completed:cancelled|completed:timed_out) + gh run view "${run_id}" --repo "${slug}" + return 1 + ;; + esac + attempts=$((attempts + 1)) + if [[ "${attempts}" -ge 120 ]]; then + echo "Timed out waiting for ${workflow_name} precheck job" >&2 + return 1 + fi + sleep 5 + done + done < <(jq -r '.[] | [.databaseId, .workflowName] | @tsv' <<<"${runs_json}") } wait_for_pypi() { @@ -321,16 +354,32 @@ create_release() { publish_artifacts() { local version="$1" local pypi_token="${UV_PUBLISH_TOKEN:-${PYPI_TOKEN:-${PYPI_PAT_SECRET:-}}}" + local artifact_prefix="${PYPI_PACKAGE//-/_}" + local artifacts=() + local dist_dir + + shopt -s nullglob + for dist_dir in "${WORKSPACE_DIR}/dist" "${REPO_DIR}/dist"; do + artifacts+=("${dist_dir}/${PYPI_PACKAGE}-${version}"*) + if [[ "${artifact_prefix}" != "${PYPI_PACKAGE}" ]]; then + artifacts+=("${dist_dir}/${artifact_prefix}-${version}"*) + fi + done + shopt -u nullglob if curl -fsSL "https://pypi.org/pypi/${PYPI_PACKAGE}/json" | jq -e --arg version "${version}" '.releases[$version] | length > 0' >/dev/null 2>&1; then echo "${PYPI_PACKAGE} ${version} already published on PyPI" else - if [[ -n "${pypi_token}" ]]; then - UV_PUBLISH_TOKEN="${pypi_token}" uv publish --username=__token__ dist/* - else - echo "Missing PyPI credentials: set UV_PUBLISH_TOKEN or PYPI_TOKEN" >&2 + if [[ "${#artifacts[@]}" -eq 0 ]]; then + echo "Missing build artifacts for ${PYPI_PACKAGE}==${version}" >&2 return 1 fi + + if [[ -n "${pypi_token}" ]]; then + UV_PUBLISH_TOKEN="${pypi_token}" uv publish --username=__token__ "${artifacts[@]}" + else + uv publish --username=__token__ "${artifacts[@]}" + fi fi wait_for_pypi "${PYPI_PACKAGE}" "${version}" @@ -377,7 +426,6 @@ main() { return 1 fi run_checks - wait_for_runs "${slug}" push "$(git rev-parse HEAD)" "push" else echo "Current version ${version} is behind latest GitHub release ${latest}" >&2 return 1 @@ -386,10 +434,8 @@ main() { publish_artifacts "${version}" create_release "${slug}" "${version}" - latest="$(latest_release_version "${slug}")" - relation="$(compare_versions "${latest}" "${version}")" - if [[ "${relation}" != "eq" ]]; then - echo "GitHub release version mismatch: expected ${version}, got ${latest}" >&2 + if ! gh release view "${TAG_PREFIX}${version}" --repo "${slug}" >/dev/null 2>&1; then + echo "GitHub release ${TAG_PREFIX}${version} was not found after creation" >&2 return 1 fi