diff --git a/Dockerfile b/Dockerfile index a1f8be54..9e136a20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -214,7 +214,7 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \ && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ diff --git a/Dockerfile.multistage b/Dockerfile.multistage index a1f8be54..9e136a20 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -214,7 +214,7 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \ && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ diff --git a/archivebox/tests/test_dockerfiles.py b/archivebox/tests/test_dockerfiles.py index cd382427..42035062 100644 --- a/archivebox/tests/test_dockerfiles.py +++ b/archivebox/tests/test_dockerfiles.py @@ -108,7 +108,7 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None: assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text - assert 'OPENCODE_BINARY="$LIB_DIR/bin/opencode"' in text + assert 'OPENCODE_BINARY="$LIB_DIR/env/bin/opencode"' in text def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None: diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py index f7ee1113..8cc613b9 100644 --- a/archivebox/tests/test_opencode_agent.py +++ b/archivebox/tests/test_opencode_agent.py @@ -238,7 +238,7 @@ def test_opencode_starts_with_data_dir_cwd_and_isolated_state(live_opencode): assert env["XDG_STATE_HOME"] == str(live_opencode.config.state_dir / "state") assert env["XDG_CACHE_HOME"] == str(live_opencode.config.state_dir / "cache") assert env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true" - assert env["GIT_CEILING_DIRECTORIES"] == f"{workdir}{os.pathsep}{live_opencode.config.data_dir.parent.resolve()}" + assert env["GIT_CEILING_DIRECTORIES"] == workdir def test_opencode_state_dir_is_separate_from_workdir(tmp_path): diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 651cde80..7f808bbb 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -575,9 +575,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia .order_by("depth", "url") .values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"), ) - snapshot_ids_by_output_dir = { - str(snapshot.output_dir): str(snapshot.id) for snapshot in Snapshot.objects.filter(crawl=crawl).order_by("depth", "url") - } archive_results = list( ArchiveResult.objects.filter(snapshot__crawl=crawl) .select_related("snapshot") @@ -594,10 +591,17 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia "output_str", ), ) + process_snapshot_ids = { + process_id: str(snapshot_id) + for snapshot_id, process_id in ArchiveResult.objects.filter( + snapshot__crawl=crawl, + process_id__isnull=False, + ).values_list("snapshot_id", "process_id") + } processes = list( - Process.objects.filter(process_type=Process.TypeChoices.HOOK, pwd__contains=str(crawl.output_dir)) + Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids) .order_by("started_at") - .values_list("pwd", "cmd", "status", "exit_code", "started_at", "ended_at"), + .values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"), ) assert crawl.max_depth == 2 @@ -681,16 +685,13 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg" ] assert not failed_hook_results - assert all(status == Process.StatusChoices.EXITED for _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes) + assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes) intervals = [] - for pwd, cmd, _status, _exit_code, started_at, ended_at in processes: + for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes: if not started_at or not ended_at: continue - process_snapshot_id = next( - (snapshot_id for output_dir, snapshot_id in snapshot_ids_by_output_dir.items() if str(pwd).startswith(output_dir)), - None, - ) + process_snapshot_id = process_snapshot_ids.get(process_id) if process_snapshot_id is None: continue intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))