The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Use shared env opencode binary and process-linked crawl test

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-05 10:54:02 -07:00
parent 1b19736b2f
commit 1987edb2ee
No known key found for this signature in database
5 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -214,7 +214,7 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \

2
Dockerfile.multistage
View File

@ -214,7 +214,7 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \

2
archivebox/tests/test_dockerfiles.py
View File

@ -108,7 +108,7 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None:
assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text
assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text
assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text
assert 'OPENCODE_BINARY="$LIB_DIR/bin/opencode"' in text
assert 'OPENCODE_BINARY="$LIB_DIR/env/bin/opencode"' in text
def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None:

2
archivebox/tests/test_opencode_agent.py
View File

@ -238,7 +238,7 @@ def test_opencode_starts_with_data_dir_cwd_and_isolated_state(live_opencode):
assert env["XDG_STATE_HOME"] == str(live_opencode.config.state_dir / "state")
assert env["XDG_CACHE_HOME"] == str(live_opencode.config.state_dir / "cache")
assert env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true"
assert env["GIT_CEILING_DIRECTORIES"] == f"{workdir}{os.pathsep}{live_opencode.config.data_dir.parent.resolve()}"
assert env["GIT_CEILING_DIRECTORIES"] == workdir
def test_opencode_state_dir_is_separate_from_workdir(tmp_path):

23
archivebox/tests/test_recursive_crawl.py
View File

@ -575,9 +575,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
.order_by("depth", "url")
.values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
)
snapshot_ids_by_output_dir = {
str(snapshot.output_dir): str(snapshot.id) for snapshot in Snapshot.objects.filter(crawl=crawl).order_by("depth", "url")
}
archive_results = list(
ArchiveResult.objects.filter(snapshot__crawl=crawl)
.select_related("snapshot")
@ -594,10 +591,17 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
"output_str",
),
)
process_snapshot_ids = {
process_id: str(snapshot_id)
for snapshot_id, process_id in ArchiveResult.objects.filter(
snapshot__crawl=crawl,
process_id__isnull=False,
).values_list("snapshot_id", "process_id")
}
processes = list(
Process.objects.filter(process_type=Process.TypeChoices.HOOK, pwd__contains=str(crawl.output_dir))
Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids)
.order_by("started_at")
.values_list("pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
.values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
)
assert crawl.max_depth == 2
@ -681,16 +685,13 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg"
]
assert not failed_hook_results
assert all(status == Process.StatusChoices.EXITED for _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
intervals = []
for pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
if not started_at or not ended_at:
continue
process_snapshot_id = next(
(snapshot_id for output_dir, snapshot_id in snapshot_ids_by_output_dir.items() if str(pwd).startswith(output_dir)),
None,
)
process_snapshot_id = process_snapshot_ids.get(process_id)
if process_snapshot_id is None:
continue
intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))