diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index 6731262d..645aec3a 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -1,5 +1,9 @@ __package__ = "archivebox.core" +import json +from collections.abc import Mapping +from typing import Any + from django import forms from django.utils.html import format_html @@ -20,6 +24,84 @@ DEPTH_CHOICES = ( ("4", "depth = 4 (+ URLs four hops away)"), ) +PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__" +PLUGIN_GROUP_DEFINITIONS = ( + ( + "chrome_plugins", + "Chrome-dependent plugins", + "", + "chrome-plugins", + "chrome", + { + "accessibility", + "chrome", + "consolelog", + "dom", + "headers", + "parse_dom_outlinks", + "pdf", + "redirects", + "responses", + "screenshot", + "seo", + "singlefile", + "ssl", + "staticfile", + "title", + }, + ), + ( + "archiving_plugins", + "Archiving", + "", + "", + "", + { + "archivedotorg", + "defuddle", + "favicon", + "forumdl", + "gallerydl", + "git", + "htmltotext", + "mercury", + "papersdl", + "readability", + "trafilatura", + "wget", + "ytdlp", + }, + ), + ( + "parsing_plugins", + "Parsing", + "", + "", + "", + { + "parse_html_urls", + "parse_jsonl_urls", + "parse_netscape_urls", + "parse_rss_urls", + "parse_txt_urls", + }, + ), + ( + "search_plugins", + "Search", + "(defaults to SEARCH_BACKEND_ENGINE)", + "", + "", + { + "search_backend_ripgrep", + "search_backend_sonic", + "search_backend_sqlite", + }, + ), + ("binary_plugins", "Binary Providers", "", "", "", {"apt", "brew", "custom", "env", "npm", "pip"}), + ("extension_plugins", "Browser Extensions", "", "", "", {"twocaptcha", "istilldontcareaboutcookies", "ublock"}), +) + def get_plugin_choices(): """Get available extractor plugins from discovered hooks.""" @@ -49,7 +131,294 @@ def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField: return field -class AddLinkForm(forms.Form): +def _plugin_config_input_name(plugin_name: str, config_key: str) -> str: + return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}" + + +def _schema_types(schema: Mapping[str, Any]) -> list[str]: + raw_type = schema.get("type") or "string" + if isinstance(raw_type, list): + return [str(item) for item in raw_type] + return [str(raw_type)] + + +def _jsonish(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps(value, sort_keys=True, default=str) + + +def _same_config_value(left: Any, right: Any) -> bool: + return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str) + + +def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any: + schema_types = _schema_types(schema) + + if "boolean" in schema_types: + if isinstance(raw_value, bool): + return raw_value + value = str(raw_value).strip().lower() + if value in {"true", "1", "yes", "on"}: + return True + if value in {"false", "0", "no", "off", ""}: + return False + raise forms.ValidationError("Must be true or false.") + + if "integer" in schema_types: + value = int(str(raw_value).strip()) + minimum = schema.get("minimum") + maximum = schema.get("maximum") + if minimum is not None and value < int(minimum): + raise forms.ValidationError(f"Must be at least {minimum}.") + if maximum is not None and value > int(maximum): + raise forms.ValidationError(f"Must be at most {maximum}.") + return value + + if "number" in schema_types: + value = float(str(raw_value).strip()) + minimum = schema.get("minimum") + maximum = schema.get("maximum") + if minimum is not None and value < float(minimum): + raise forms.ValidationError(f"Must be at least {minimum}.") + if maximum is not None and value > float(maximum): + raise forms.ValidationError(f"Must be at most {maximum}.") + return value + + if "array" in schema_types: + if isinstance(raw_value, list): + return raw_value + value = str(raw_value).strip() + if not value: + return [] + if value.startswith("["): + parsed = json.loads(value) + if not isinstance(parsed, list): + raise forms.ValidationError("Must be a JSON array.") + return parsed + return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()] + + if "object" in schema_types: + value = str(raw_value).strip() + if not value: + return {} + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise forms.ValidationError("Must be a JSON object.") + return parsed + + value = str(raw_value) + enum = schema.get("enum") + if isinstance(enum, list) and enum and value not in {str(item) for item in enum}: + raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.") + return value + + +class PluginConfigFormMixin: + plugin_groups: list[dict[str, Any]] + + def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None: + all_plugins = get_plugins() + plugin_configs = discover_plugin_configs() + runtime_config = runtime_config or get_config() + grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS)) + + for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS: + if field_name in self.fields: + get_choice_field(self, field_name).choices = [ + (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in plugin_names + ] + + if "other_plugins" in self.fields: + get_choice_field(self, "other_plugins").choices = [ + (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p not in grouped_plugins + ] + + if "search_plugins" in self.fields: + required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip() + search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices] + if required_search_plugin in search_choices: + get_choice_field(self, "search_plugins").initial = [required_search_plugin] + + group_specs = (*PLUGIN_GROUP_DEFINITIONS, ("other_plugins", "Other", "", "", "", set(all_plugins) - grouped_plugins)) + self.plugin_groups = [ + { + "field_name": field_name, + "title": title, + "note": note, + "dom_id": dom_id, + "select_all_group": select_all_group, + "show_selectors": field_name in self.fields, + "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config), + } + for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs + if any(plugin in all_plugins for plugin in plugin_names) + ] + + def _build_plugin_cards( + self, + field_name: str, + plugin_names: set[str], + plugin_configs: dict[str, dict[str, Any]], + runtime_config: Mapping[str, Any], + ) -> list[dict[str, Any]]: + if field_name in self.fields: + choices = list(get_choice_field(self, field_name).choices) + selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or []) + else: + choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(get_plugins()) if p in plugin_names] + selected_values = set() + + cards = [] + for index, (plugin_name, label) in enumerate(choices): + schema = plugin_configs.get(str(plugin_name), {}) + properties = schema.get("properties") or {} + config_fields = [ + self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config) + for config_key, prop_schema in properties.items() + if isinstance(prop_schema, dict) + ] + cards.append( + { + "name": str(plugin_name), + "label": label, + "checked": str(plugin_name) in selected_values, + "checkbox_id": f"id_{field_name}_{index}", + "description": str(schema.get("description") or "").strip(), + "required_plugins": [str(item) for item in schema.get("required_plugins") or []], + "required_binaries_count": len(schema.get("required_binaries") or []), + "config_fields": config_fields, + "config_count": len(config_fields), + }, + ) + return cards + + def _build_plugin_config_field( + self, + plugin_name: str, + config_key: str, + prop_schema: Mapping[str, Any], + runtime_config: Mapping[str, Any], + ) -> dict[str, Any]: + schema_types = _schema_types(prop_schema) + enum = prop_schema.get("enum") + input_name = _plugin_config_input_name(plugin_name, config_key) + current_value = runtime_config.get(config_key, prop_schema.get("default", "")) + if self.is_bound and input_name in self.data: + try: + current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema) + except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError): + current_value = self.data.get(input_name) + + default_value = prop_schema.get("default", "") + is_sensitive = bool(prop_schema.get("x-sensitive")) + input_value = "" if is_sensitive else _jsonish(current_value) + field_kind = "text" + input_type = "text" + options = [] + + if "boolean" in schema_types: + field_kind = "boolean" + input_value = "true" if bool(current_value) else "false" + elif isinstance(enum, list) and enum: + field_kind = "select" + options = [ + { + "value": str(option), + "label": str(option), + "selected": str(option) == str(current_value), + } + for option in enum + ] + elif "integer" in schema_types or "number" in schema_types: + field_kind = "number" + input_type = "number" + elif "array" in schema_types or "object" in schema_types: + field_kind = "json" + input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str) + elif is_sensitive: + input_type = "password" + + return { + "key": config_key, + "input_name": input_name, + "kind": field_kind, + "input_type": input_type, + "value": input_value, + "checked": bool(current_value), + "options": options, + "description": str(prop_schema.get("description") or "").strip(), + "default": _jsonish(default_value), + "current": "configured" if is_sensitive and current_value else _jsonish(current_value), + "is_sensitive": is_sensitive, + "minimum": prop_schema.get("minimum"), + "maximum": prop_schema.get("maximum"), + "pattern": prop_schema.get("pattern"), + "type_label": " / ".join(schema_types), + } + + def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]: + if not self.is_bound: + return {} + + effective_config = effective_config or get_config() + overrides: dict[str, Any] = {} + sources: dict[str, str] = {} + + for plugin_name, schema in discover_plugin_configs().items(): + for config_key, prop_schema in (schema.get("properties") or {}).items(): + if not isinstance(prop_schema, dict): + continue + + input_name = _plugin_config_input_name(plugin_name, config_key) + if input_name not in self.data: + continue + + raw_value: Any = self.data.get(input_name) + if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list): + raw_value = self.data.getlist(input_name) + + if prop_schema.get("x-sensitive") and raw_value == "": + continue + + try: + coerced_value = _coerce_plugin_config_value(raw_value, prop_schema) + except (TypeError, ValueError, json.JSONDecodeError) as err: + self.add_error("config", forms.ValidationError(f"{config_key}: {err}")) + continue + except forms.ValidationError as err: + self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}")) + continue + + base_value = effective_config.get(config_key, prop_schema.get("default", "")) + if _same_config_value(coerced_value, base_value): + continue + + existing_value = overrides.get(config_key) + if config_key in overrides and not _same_config_value(existing_value, coerced_value): + self.add_error( + "config", + forms.ValidationError( + f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.", + ), + ) + continue + + overrides[config_key] = coerced_value + sources[config_key] = plugin_name + + return overrides + + def plugin_config_keys(self) -> set[str]: + return { + str(config_key) + for schema in discover_plugin_configs().values() + for config_key, prop_schema in (schema.get("properties") or {}).items() + if isinstance(prop_schema, dict) + } + + +class AddLinkForm(PluginConfigFormMixin, forms.Form): # Basic fields url = forms.CharField( label="URLs", @@ -149,6 +518,12 @@ class AddLinkForm(forms.Form): widget=forms.CheckboxSelectMultiple, choices=[], ) + other_plugins = forms.MultipleChoiceField( + label="Other plugins", + required=False, + widget=forms.CheckboxSelectMultiple, + choices=[], + ) # Advanced options schedule = forms.CharField( @@ -187,82 +562,7 @@ class AddLinkForm(forms.Form): self.fields["persona"].queryset = Persona.objects.order_by("name") self.fields["persona"].initial = default_persona.name - # Get all plugins - all_plugins = get_plugins() - plugin_configs = discover_plugin_configs() - - # Define plugin groups - chrome_dependent = { - "accessibility", - "chrome", - "consolelog", - "dom", - "headers", - "parse_dom_outlinks", - "pdf", - "redirects", - "responses", - "screenshot", - "seo", - "singlefile", - "ssl", - "staticfile", - "title", - } - archiving = { - "archivedotorg", - "defuddle", - "favicon", - "forumdl", - "gallerydl", - "git", - "htmltotext", - "mercury", - "papersdl", - "readability", - "trafilatura", - "wget", - "ytdlp", - } - parsing = { - "parse_html_urls", - "parse_jsonl_urls", - "parse_netscape_urls", - "parse_rss_urls", - "parse_txt_urls", - } - search = { - "search_backend_ripgrep", - "search_backend_sonic", - "search_backend_sqlite", - } - binary = {"apt", "brew", "custom", "env", "npm", "pip"} - extensions = {"twocaptcha", "istilldontcareaboutcookies", "ublock"} - - # Populate plugin field choices - get_choice_field(self, "chrome_plugins").choices = [ - (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in chrome_dependent - ] - get_choice_field(self, "archiving_plugins").choices = [ - (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in archiving - ] - get_choice_field(self, "parsing_plugins").choices = [ - (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in parsing - ] - get_choice_field(self, "search_plugins").choices = [ - (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in search - ] - get_choice_field(self, "binary_plugins").choices = [ - (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in binary - ] - get_choice_field(self, "extension_plugins").choices = [ - (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in extensions - ] - - required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip() - search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices] - if required_search_plugin in search_choices: - get_choice_field(self, "search_plugins").initial = [required_search_plugin] + self.build_plugin_groups() def clean(self): cleaned_data = super().clean() or {} @@ -276,6 +576,7 @@ class AddLinkForm(forms.Form): "search_plugins", "binary_plugins", "extension_plugins", + "other_plugins", ]: selected = cleaned_data.get(field) if isinstance(selected, list): @@ -284,6 +585,13 @@ class AddLinkForm(forms.Form): # Store combined list for easy access cleaned_data["plugins"] = all_selected_plugins + plugin_config_overrides = self.clean_plugin_config_overrides(get_config(persona=cleaned_data.get("persona"))) + custom_config = cleaned_data.get("config") or {} + if not isinstance(custom_config, dict): + custom_config = {} + cleaned_data["plugin_config"] = plugin_config_overrides + cleaned_data["config"] = {**plugin_config_overrides, **custom_config} + return cleaned_data def clean_url(self): diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 1bb2d323..4743d20e 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1008,8 +1008,26 @@ class AddView(UserPassesTestMixin, FormView): return custom_config def get_context_data(self, **kwargs): + from archivebox.personas.models import Persona + required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip() plugin_configs = discover_plugin_configs() + sensitive_keys = { + str(config_key) + for schema in plugin_configs.values() + for config_key, prop_schema in (schema.get("properties") or {}).items() + if isinstance(prop_schema, dict) and prop_schema.get("x-sensitive") + } + base_config = get_config() + persona_config_map = {} + for persona in Persona.objects.order_by("name"): + raw_config = {str(key): value for key, value in (persona.config or {}).items() if str(key) not in sensitive_keys} + persona_config_map[persona.name] = { + "config": raw_config, + "effective_config": { + str(key): value for key, value in {**base_config, **raw_config}.items() if str(key) not in sensitive_keys + }, + } plugin_dependency_map = { plugin_name: [ str(required_plugin).strip() for required_plugin in (schema.get("required_plugins") or []) if str(required_plugin).strip() @@ -1026,6 +1044,7 @@ class AddView(UserPassesTestMixin, FormView): "FOOTER_INFO": get_config().FOOTER_INFO, "required_search_plugin": required_search_plugin, "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True), + "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str), "stdout": "", } @@ -1044,6 +1063,9 @@ class AddView(UserPassesTestMixin, FormView): index_only = form.cleaned_data.get("index_only", False) notes = form.cleaned_data.get("notes", "") url_filters = form.cleaned_data.get("url_filters") or {} + plugin_config = form.cleaned_data.get("plugin_config") or {} + if not isinstance(plugin_config, dict): + plugin_config = {} custom_config = self._get_custom_config_overrides(form) persona_name = persona.name if persona else "Default" if persona: @@ -1078,6 +1100,7 @@ class AddView(UserPassesTestMixin, FormView): } # Merge custom config overrides + config.update(plugin_config) config.update(custom_config) config["DEFAULT_PERSONA"] = persona_name if url_filters.get("allowlist"): diff --git a/archivebox/personas/forms.py b/archivebox/personas/forms.py index 894f7af9..a801f356 100644 --- a/archivebox/personas/forms.py +++ b/archivebox/personas/forms.py @@ -5,6 +5,8 @@ from typing import Any from django import forms from django.utils.safestring import mark_safe +from archivebox.config.common import get_config +from archivebox.core.forms import PluginConfigFormMixin from archivebox.personas.importers import ( PersonaImportResult, PersonaImportSource, @@ -22,7 +24,7 @@ def _mode_label(title: str, description: str) -> str: ) -class PersonaAdminForm(forms.ModelForm): +class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm): import_mode = forms.ChoiceField( required=False, initial="none", @@ -112,6 +114,9 @@ class PersonaAdminForm(forms.ModelForm): "Use the custom path/CDP option if the browser data lives elsewhere." ) + persona_config = self.instance.config if self.instance and self.instance.pk and isinstance(self.instance.config, dict) else {} + self.build_plugin_groups({**get_config(), **persona_config}) + def clean_name(self) -> str: name = str(self.cleaned_data.get("name") or "").strip() is_valid, error_message = validate_persona_name(name) @@ -124,6 +129,16 @@ class PersonaAdminForm(forms.ModelForm): self._resolved_import_source = None import_mode = str(cleaned_data.get("import_mode") or "none").strip() or "none" + manual_config = cleaned_data.get("config") or {} + if not isinstance(manual_config, dict): + manual_config = {} + plugin_config_overrides = self.clean_plugin_config_overrides(get_config()) + cleaned_data["plugin_config"] = plugin_config_overrides + cleaned_data["config"] = { + **{key: value for key, value in manual_config.items() if key not in self.plugin_config_keys()}, + **plugin_config_overrides, + } + if import_mode == "none": return cleaned_data diff --git a/archivebox/templates/admin/personas/persona/change_form.html b/archivebox/templates/admin/personas/persona/change_form.html index 1bdde87a..b3bcb072 100644 --- a/archivebox/templates/admin/personas/persona/change_form.html +++ b/archivebox/templates/admin/personas/persona/change_form.html @@ -1,9 +1,11 @@ {% extends "admin/change_form.html" %} +{% load static %} {% block bodyclass %}{{ block.super }} app-personas model-persona{% endblock %} {% block extrastyle %} {{ block.super }} + {% endblock %} @@ -247,3 +281,12 @@ document.addEventListener('DOMContentLoaded', function () { {{ block.super }} {% endblock %} + +{% block after_field_sets %} +{{ block.super }} +
+

Plugin Config

+

These typed controls update the same Persona config JSON shown above. Shared config keys stay synced across plugin sections.

+ {% include "core/plugin_config_grid.html" with plugin_groups=adminform.form.plugin_groups %} +
+{% endblock %} diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html index 198bafdf..925f9cc8 100644 --- a/archivebox/templates/core/add.html +++ b/archivebox/templates/core/add.html @@ -169,65 +169,7 @@ -
-
-
- - -
-
- {{ form.chrome_plugins }} -
-
- -
-
- -
-
- {{ form.archiving_plugins }} -
-
- -
-
- -
-
- {{ form.parsing_plugins }} -
-
- -
-
- - (defaults to SEARCH_BACKEND_ENGINE) -
-
- {{ form.search_plugins }} -
-
- -
-
- -
-
- {{ form.binary_plugins }} -
-
- -
-
- -
-
- {{ form.extension_plugins }} -
-
-
+ {% include "core/plugin_config_grid.html" with plugin_groups=form.plugin_groups %} @@ -313,6 +255,8 @@ ]; const requiredSearchPlugin = '{{ required_search_plugin|default:""|escapejs }}'; const pluginDependencyMap = JSON.parse('{{ plugin_dependency_map_json|default:"{}"|escapejs }}'); + const personaConfigMap = JSON.parse('{{ persona_config_map_json|default:"{}"|escapejs }}'); + const personaSelect = document.querySelector('select[name="persona"]'); function dispatchChange(el) { el.dispatchEvent(new Event('input', { bubbles: true })); @@ -389,7 +333,7 @@ if (!rows || !updater) return; let row = findConfigRow(key); - if (!value) { + if (value === undefined || value === null || value === '') { if (row) { row.remove(); updater(); @@ -407,7 +351,7 @@ if (!keyInput || !valueInput) return; keyInput.value = key; - valueInput.value = value; + valueInput.value = typeof value === 'string' ? value : JSON.stringify(value); keyInput.dispatchEvent(new Event('input', { bubbles: true })); valueInput.dispatchEvent(new Event('input', { bubbles: true })); updater(); @@ -421,6 +365,26 @@ setConfigRow('ENABLED_PLUGINS', selectedPlugins.join(',')); } + function replaceConfigRows(config) { + const rows = getConfigEditorRows(); + const updater = getConfigUpdater(); + if (!rows || !updater || !config || typeof config !== 'object') return; + Array.from(rows.querySelectorAll('.key-value-row')).forEach(row => row.remove()); + Object.entries(config).forEach(([key, value]) => setConfigRow(key, value)); + if (!rows.querySelector('.key-value-row')) addConfigRow(); + updater(); + } + + function applyPersonaConfig(personaName) { + const personaData = personaConfigMap[personaName]; + if (!personaData) return; + if (typeof window.archiveboxSetPluginConfigValues === 'function') { + window.archiveboxSetPluginConfigValues(personaData.effective_config || {}); + } + replaceConfigRows(personaData.config || {}); + updateURLPreview(); + } + function ensureRequiredPluginsChecked(pluginName, visited = new Set()) { if (!pluginName || visited.has(pluginName)) { return; @@ -1157,6 +1121,7 @@ const state = {}; document.querySelectorAll('#add-form input, #add-form textarea, #add-form select').forEach(el => { if (el.name === 'csrfmiddlewaretoken') return; + if (el.name === 'config' || el.name.startsWith('plugin_config__') || el.closest('#id_config_container')) return; if (el.type === 'checkbox' || el.type === 'radio') { state[el.name + ':' + el.value] = el.checked; } else { @@ -1172,9 +1137,11 @@ for (const [key, value] of Object.entries(state)) { if (key.includes(':')) { const [name, val] = key.split(':'); + if (name === 'config' || name.startsWith('plugin_config__')) continue; const el = document.querySelector(`[name="${name}"][value="${val}"]`); if (el) el.checked = Boolean(value); } else { + if (key === 'config' || key.startsWith('plugin_config__')) continue; const el = document.querySelector(`[name="${key}"]`); if (el && el.type !== 'checkbox' && el.type !== 'radio') { if (el.tagName === 'SELECT') { @@ -1192,6 +1159,9 @@ } } updateURLPreview(); // Update preview after loading URLs + if (personaSelect) { + applyPersonaConfig(personaSelect.value); + } normalizePluginSelections(); applyRequiredSearchPlugin(); syncEnabledPluginsConfig(); @@ -1205,6 +1175,13 @@ el.addEventListener('change', saveFormState); }); + if (personaSelect) { + personaSelect.addEventListener('change', function() { + applyPersonaConfig(personaSelect.value); + saveFormState(); + }); + } + loadFormState(); // Form submission handler diff --git a/archivebox/templates/core/plugin_config_grid.html b/archivebox/templates/core/plugin_config_grid.html new file mode 100644 index 00000000..b4ed552b --- /dev/null +++ b/archivebox/templates/core/plugin_config_grid.html @@ -0,0 +1,169 @@ +
+
+ {% for group in plugin_groups %} + {% if group.plugins %} +
+
+ + {% if group.note %} + {{ group.note }} + {% endif %} + {% if group.show_selectors and group.select_all_group %} + + {% endif %} +
+
+ {% for plugin in group.plugins %} +
+
+ {% if group.show_selectors %} + + + {% else %} +
+ {{ plugin.label }} +
+ {% endif %} +
+ + Config + {{ plugin.config_count }} + + {% if plugin.required_plugins or plugin.required_binaries_count %} +
+ {% if plugin.required_plugins %} + Requires {{ plugin.required_plugins|join:", " }} + {% endif %} + {% if plugin.required_binaries_count %} + {% if plugin.required_plugins %} · {% endif %} + {{ plugin.required_binaries_count }} binary requirement{{ plugin.required_binaries_count|pluralize }} + {% endif %} +
+ {% endif %} + {% if plugin.config_fields %} +
+ {% for field in plugin.config_fields %} +
+ + {% if field.kind == "boolean" %} + + + {% elif field.kind == "select" %} + + {% elif field.kind == "json" %} + + {% else %} + + {% endif %} + {% if field.description %} +
{{ field.description }}
+ {% endif %} +
Current: {{ field.current }} · Default: {{ field.default }}
+
+ {% endfor %} +
+ {% else %} +
This plugin has no crawl-configurable options.
+ {% endif %} +
+
+
+ {% endfor %} +
+
+ {% endif %} + {% endfor %} +
+
+ diff --git a/archivebox/templates/static/add.css b/archivebox/templates/static/add.css index 0e854f63..54910e56 100755 --- a/archivebox/templates/static/add.css +++ b/archivebox/templates/static/add.css @@ -707,42 +707,40 @@ select { .plugin-checkboxes { display: grid; grid-template-columns: 1fr; - gap: 6px; -} - -.plugin-checkboxes > div { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 6px 10px; -} - -.plugin-checkboxes > div > div { - display: flex; - align-items: center; gap: 8px; - padding: 6px 8px; +} + +.plugin-card { border: 1px solid #e3e8ef; background-color: #fff; - border-radius: 4px; + border-radius: 6px; transition: background-color 0.2s; } -.plugin-checkboxes > div > div:hover { +.plugin-card:hover { background-color: #f5f5f5; } -.plugin-checkboxes input[type="checkbox"] { - grid-column: 2; - grid-row: 1 / span 2; - margin: 0; - margin-top: 2px; - width: auto; - flex: 0 0 auto; +.plugin-card-main { + display: grid; + grid-template-columns: 18px minmax(0, 1fr) auto; + gap: 8px; + align-items: start; + padding: 8px 10px; } -#add-form .plugin-checkboxes label { +.plugin-card-main > input[type="checkbox"] { + grid-column: 1; + grid-row: 1; + width: auto; + margin: 3px 0 0; +} + +.plugin-config-form .plugin-card-label { + grid-column: 2; + grid-row: 1; display: grid !important; - grid-template-columns: 18px 16px minmax(0, 1fr); + grid-template-columns: 20px minmax(0, 1fr); column-gap: 8px; row-gap: 3px; align-items: start; @@ -754,13 +752,13 @@ select { } .plugin-choice-name { - grid-column: 3; + grid-column: 2; grid-row: 1; font-weight: 500; color: #1f2937; } -#add-form .plugin-choice-icon { +.plugin-config-form .plugin-choice-icon { grid-column: 1; grid-row: 1 / span 2; display: inline-flex; @@ -770,19 +768,19 @@ select { flex: 0 0 auto; } -#add-form .plugin-choice-icon .abx-output-icon { +.plugin-config-form .plugin-choice-icon .abx-output-icon { display: inline-flex; align-items: center; justify-content: center; } -#add-form .plugin-choice-icon svg { +.plugin-config-form .plugin-choice-icon svg { width: 18px; height: 18px; } -#add-form .plugin-choice-description { - grid-column: 3; +.plugin-config-form .plugin-choice-description { + grid-column: 2; grid-row: 2; margin-left: 0; display: inline-block; @@ -792,19 +790,189 @@ select { text-align: left; } -#add-form .plugin-checkboxes label a.plugin-choice-description:link, -#add-form .plugin-checkboxes label a.plugin-choice-description:visited, -#add-form .plugin-checkboxes label a.plugin-choice-description:active { +.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:link, +.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:visited, +.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:active { color: #7a7a7a !important; text-decoration: none !important; } -#add-form .plugin-checkboxes label a.plugin-choice-description:hover, -#add-form .plugin-checkboxes label a.plugin-choice-description:focus { +.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:hover, +.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:focus { color: #4b5563 !important; text-decoration: underline !important; } +.plugin-config-details { + grid-column: 3; + grid-row: 1; + min-width: 0; +} + +.plugin-config-details summary { + display: inline-flex; + align-items: center; + gap: 6px; + min-height: 26px; + padding: 3px 8px; + border: 1px solid #d7e2eb; + border-radius: 999px; + background: #f8fafc; + color: #475569; + font-size: 12px; + font-weight: 700; + cursor: pointer; + user-select: none; + list-style: none; + white-space: nowrap; +} + +.plugin-config-details summary::-webkit-details-marker { + display: none; +} + +.plugin-config-details summary:before { + content: '▶'; + display: inline-block; + font-size: 10px; + transition: transform 0.2s; +} + +.plugin-config-details[open] { + grid-column: 1 / -1; + grid-row: 2; + padding-top: 8px; +} + +.plugin-config-details[open] summary { + margin-bottom: 8px; +} + +.plugin-config-details[open] summary:before { + transform: rotate(90deg); +} + +.plugin-config-count { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 18px; + height: 18px; + padding: 0 5px; + border-radius: 999px; + background: #e2e8f0; + color: #334155; + font-size: 11px; +} + +.plugin-config-meta, +.plugin-config-empty { + margin-bottom: 8px; + padding: 7px 9px; + border: 1px solid #e5edf5; + border-radius: 6px; + background: #f8fafc; + color: #64748b; + font-size: 12px; + line-height: 1.4; +} + +.plugin-config-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; + max-height: 360px; + overflow: auto; + padding: 10px; + border: 1px solid #e5edf5; + border-radius: 6px; + background: #fbfdff; +} + +.plugin-config-field { + min-width: 0; +} + +.plugin-config-form .plugin-config-field > label { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px; + margin-bottom: 5px; + font-size: 12px; + font-weight: 700; + color: #24303b; +} + +.plugin-config-field code { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.plugin-config-type { + flex: 0 0 auto; + color: #7a8794; + font-size: 11px; + font-weight: 500; +} + +.plugin-config-field input[type="text"], +.plugin-config-field input[type="password"], +.plugin-config-field input[type="number"], +.plugin-config-field select, +.plugin-config-field textarea { + width: 100%; + min-height: 34px; + padding: 6px 8px; + border: 1px solid #cbd5e1; + border-radius: 4px; + box-shadow: none; + font-family: inherit; + font-size: 12px; + background: #fff; +} + +.plugin-config-field textarea { + min-height: 70px; + resize: vertical; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} + +.plugin-config-form .plugin-config-toggle { + display: inline-flex !important; + align-items: center; + justify-content: flex-start; + gap: 8px; + min-height: 34px; + margin: 0; + padding: 6px 8px; + border: 1px solid #cbd5e1; + border-radius: 4px; + background: #fff; + font-size: 12px; + font-weight: 600; +} + +.plugin-config-toggle input[type="checkbox"] { + width: auto; + margin: 0; +} + +.plugin-config-help, +.plugin-config-default { + margin-top: 4px; + color: #64748b; + font-size: 11px; + line-height: 1.35; +} + +.plugin-config-default code { + color: #475569; + white-space: normal; + word-break: break-word; +} + /* Advanced section (collapsible) */ .advanced-section { background-color: white; @@ -872,7 +1040,16 @@ input:focus, select:focus, textarea:focus, button:focus { grid-template-columns: 1fr; } - .plugin-checkboxes > div { + .plugin-card-main { + grid-template-columns: 18px minmax(0, 1fr); + } + + .plugin-config-details { + grid-column: 1 / -1; + grid-row: 2; + } + + .plugin-config-grid { grid-template-columns: 1fr; } diff --git a/archivebox/templates/static/admin.css b/archivebox/templates/static/admin.css index 408f178f..c4b58b74 100755 --- a/archivebox/templates/static/admin.css +++ b/archivebox/templates/static/admin.css @@ -176,29 +176,6 @@ body.change-list #content .object-tools { } -/*#content #changelist .actions { - position: fixed; - bottom: 0px; - z-index: 800; -}*/ -#content #changelist .actions { - float: right; - margin-top: -34px; - padding: 0px; - background: none; - margin-right: 0px; - width: auto; - max-height: 40px; - display: block; - position: relative; - z-index: 2; -} -@media (max-width: 1000px) { - #content #changelist .actions { - max-height: 200px; - } -} - #content #changelist .actions .button { border-radius: 2px; background-color: #f5dd5d; @@ -346,16 +323,6 @@ body.change-list #content .object-tools { } } -@media (max-width: 1127px) { - #content #changelist .actions { - position: fixed; - bottom: 6px; - left: 10px; - float: left; - z-index: 1000; - } -} - #content a img.favicon { height: 20px; max-width: 28px; diff --git a/archivebox/tests/test_add_view.py b/archivebox/tests/test_add_view.py index d0793af7..af0b86c0 100644 --- a/archivebox/tests/test_add_view.py +++ b/archivebox/tests/test_add_view.py @@ -50,6 +50,30 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m assert 'id="url-highlight-layer"' in body assert 'id="detected-urls-list"' in body assert "detected-url-toggle-btn" in body + assert "plugin-config-details" in body + assert 'name="plugin_config__wget__WGET_TIMEOUT"' in body + assert 'name="plugin_config__chrome__CHROME_HEADLESS"' in body + assert "personaConfigMap" in body + assert "archiveboxSetPluginConfigValues" in body + assert "el.name === 'config' || el.name.startsWith('plugin_config__')" in body + + +def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + Persona.objects.create( + name="Private", + created_by=admin_user, + config={"WGET_TIMEOUT": 88, "CHROME_HEADLESS": False}, + ) + + response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) + body = response.content.decode() + + assert response.status_code == 200 + assert "Private" in body + assert "WGET_TIMEOUT" in body + assert "88" in body + assert "CHROME_HEADLESS" in body def test_add_view_checks_configured_search_backend_by_default(client, monkeypatch): @@ -146,6 +170,41 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin assert runtime_config.COOKIES_FILE == private_cookies_file +def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + client.force_login(admin_user) + + response = client.post( + reverse("add"), + data={ + "url": "https://example.com/plugin-config", + "tag": "", + "depth": "0", + "max_urls": "0", + "max_size": "0", + "url_filters_allowlist": "", + "url_filters_denylist": "", + "notes": "", + "schedule": "", + "persona": "Default", + "index_only": "", + "archiving_plugins": ["wget"], + "plugin_config__wget__WGET_TIMEOUT": "77", + "plugin_config__wget__WGET_WARC_ENABLED": "false", + "config": "{}", + }, + HTTP_HOST=WEB_HOST, + ) + + assert response.status_code == 302 + + crawl = Crawl.objects.order_by("-created_at").first() + assert crawl is not None + assert crawl.config["PLUGINS"] == "wget" + assert crawl.config["WGET_TIMEOUT"] == 77 + assert crawl.config["WGET_WARC_ENABLED"] is False + + def test_add_view_starts_background_runner_after_creating_crawl(client, admin_user, monkeypatch): monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) diff --git a/archivebox/tests/test_persona_admin.py b/archivebox/tests/test_persona_admin.py index 01c66495..c620b0a7 100644 --- a/archivebox/tests/test_persona_admin.py +++ b/archivebox/tests/test_persona_admin.py @@ -12,6 +12,7 @@ from archivebox.personas.importers import ( resolve_browser_profile_source, resolve_custom_import_source, ) +from archivebox.personas.models import Persona pytestmark = pytest.mark.django_db @@ -110,6 +111,8 @@ def test_persona_admin_add_view_renders_import_ui(client, admin_user, monkeypatc assert b"Bootstrap a persona from a real browser session" in response.content assert b"Google Chrome / Default" in response.content assert b"auth.json" in response.content + assert b"Plugin Config" in response.content + assert b'name="plugin_config__wget__WGET_TIMEOUT"' in response.content def test_import_persona_from_source_copies_user_agent_to_persona_config(admin_user, monkeypatch, tmp_path): @@ -189,3 +192,28 @@ def test_persona_admin_add_post_runs_shared_importer(client, admin_user, monkeyp } assert persona.COOKIES_FILE.endswith("cookies.txt") assert persona.AUTH_STORAGE_FILE.endswith("auth.json") + + +def test_persona_admin_saves_typed_plugin_config(client, admin_user, monkeypatch): + monkeypatch.setattr("archivebox.personas.forms.discover_local_browser_profiles", lambda: []) + monkeypatch.setattr("archivebox.personas.admin.discover_local_browser_profiles", lambda: []) + + client.login(username="personaadmin", password="testpassword") + response = client.post( + reverse("admin:personas_persona_add"), + { + "name": "PluginConfigPersona", + "created_by": str(admin_user.pk), + "config": "{}", + "import_mode": "none", + "plugin_config__wget__WGET_TIMEOUT": "77", + "plugin_config__wget__WGET_WARC_ENABLED": "false", + "_save": "Save", + }, + HTTP_HOST=ADMIN_HOST, + ) + + assert response.status_code == 302 + persona = Persona.objects.get(name="PluginConfigPersona") + assert persona.config["WGET_TIMEOUT"] == 77 + assert persona.config["WGET_WARC_ENABLED"] is False diff --git a/bin/build_docker.sh b/bin/build_docker.sh index a0c0b4d5..adef40eb 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -89,11 +89,6 @@ docker buildx use xbuilder >/dev/null 2>&1 || create_builder check_platforms || (recreate_builder && check_platforms) || exit 1 -# Make sure pyproject.toml, pdm{.dev}.lock, requirements{-dev}.txt, package{-lock}.json are all up-to-date -# echo "[!] Make sure you've run ./bin/lock_pkgs.sh recently!" -bash ./bin/lock_pkgs.sh - - echo "[+] Building archivebox:$VERSION docker image..." # docker builder prune # docker build . --no-cache -t archivebox-dev \ diff --git a/bin/build_pip.sh b/bin/build_pip.sh index 382ca6de..d61a7623 100755 --- a/bin/build_pip.sh +++ b/bin/build_pip.sh @@ -13,8 +13,6 @@ IFS=$'\n' REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" cd "$REPO_DIR" -# Generate pdm.lock, requirements.txt, and package-lock.json -bash ./bin/lock_pkgs.sh source .venv/bin/activate echo "[+] Building sdist, bdist_wheel, and egg_info" diff --git a/bin/deploy_dev_demo.sh b/bin/deploy_dev_demo.sh new file mode 100755 index 00000000..53215f6b --- /dev/null +++ b/bin/deploy_dev_demo.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +IFS=$'\n\t' + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" + +DEPLOY_HOST="${DEPLOY_HOST:-cabbage}" +DEPLOY_PORT="${DEPLOY_PORT:-44}" +DEPLOY_PATH="${DEPLOY_PATH:-/opt/archivebox.demo}" +DEPLOY_SERVICE="${DEPLOY_SERVICE:-archivebox}" +DEPLOY_IMAGE="${DEPLOY_IMAGE:-archivebox/archivebox:dev}" + +VERSION="$(grep '^version = ' pyproject.toml | awk -F'"' '{print $2}')" +GIT_SHA="sha-$(git rev-parse --short HEAD)" + +if [[ "$(git branch --show-current)" != "dev" ]]; then + echo "[X] Run this from the dev branch." >&2 + exit 1 +fi + +if [[ -n "$(git status --short)" ]]; then + echo "[X] Refusing to deploy with a dirty worktree. Commit or stash changes first." >&2 + git status --short >&2 + exit 1 +fi + +echo "[+] Pushing dev to GitHub..." +git push origin dev + +if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then + echo "[+] Publishing Docker image tags: dev ${VERSION} ${GIT_SHA}" + ./bin/release_docker.sh dev "$VERSION" "$GIT_SHA" +fi + +if [[ "${SKIP_DEMO:-0}" == "1" ]]; then + echo "[√] Skipped demo deploy." + exit 0 +fi + +echo "[+] Deploying ${DEPLOY_IMAGE} on ${DEPLOY_HOST}:${DEPLOY_PATH}..." +ssh -p "$DEPLOY_PORT" "$DEPLOY_HOST" DEPLOY_PATH="$DEPLOY_PATH" DEPLOY_SERVICE="$DEPLOY_SERVICE" DEPLOY_IMAGE="$DEPLOY_IMAGE" 'bash -s' <<'REMOTE' +set -Eeuo pipefail +cd "$DEPLOY_PATH" + +if [[ -f compose.yml || -f compose.yaml || -f docker-compose.yml ]]; then + COMPOSE=(docker compose) +else + echo "[X] No compose file found in $DEPLOY_PATH" >&2 + exit 1 +fi + +export ARCHIVEBOX_IMAGE="$DEPLOY_IMAGE" + +echo "[+] Pulling $ARCHIVEBOX_IMAGE..." +"${COMPOSE[@]}" pull "$DEPLOY_SERVICE" + +echo "[+] Restarting $DEPLOY_SERVICE..." +"${COMPOSE[@]}" up -d "$DEPLOY_SERVICE" + +echo "[+] Container status:" +"${COMPOSE[@]}" ps "$DEPLOY_SERVICE" + +echo "[+] ArchiveBox version:" +"${COMPOSE[@]}" exec -T "$DEPLOY_SERVICE" archivebox version | sed -n '1,40p' + +echo "[+] Health check:" +"${COMPOSE[@]}" exec -T "$DEPLOY_SERVICE" curl -fsS -H 'Host: admin.archivebox.io' http://127.0.0.1:8000/health/ +REMOTE + +echo "[√] Demo deploy finished." diff --git a/bin/lock_pkgs.sh b/bin/lock_pkgs.sh deleted file mode 120000 index a645eb30..00000000 --- a/bin/lock_pkgs.sh +++ /dev/null @@ -1 +0,0 @@ -setup.sh \ No newline at end of file diff --git a/bin/release_docker.sh b/bin/release_docker.sh index 3a87457d..160204e2 100755 --- a/bin/release_docker.sh +++ b/bin/release_docker.sh @@ -42,9 +42,6 @@ for TAG_NAME in "${TAG_NAMES[@]}"; do done echo "${FULL_TAG_NAMES[@]}" - -./bin/lock_pkgs.sh - # echo "[*] Logging in to Docker Hub & Github Container Registry" # docker login --username=nikisweeting # docker login ghcr.io --username=pirate diff --git a/bin/setup.sh b/bin/setup.sh index 1d540e67..46964651 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -20,7 +20,36 @@ set -o pipefail clear -if [ $(id -u) -eq 0 ]; then +ARCHIVEBOX_BRANCH="${ARCHIVEBOX_BRANCH:-dev}" +ARCHIVEBOX_IMAGE="${ARCHIVEBOX_IMAGE:-archivebox/archivebox:dev}" +ARCHIVEBOX_PLATFORM="${ARCHIVEBOX_PLATFORM:-linux/amd64}" +ARCHIVEBOX_COMPOSE_URL="${ARCHIVEBOX_COMPOSE_URL:-https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/${ARCHIVEBOX_BRANCH}/docker-compose.yml}" + +wait_for_archivebox() { + local url="http://127.0.0.1:8000/health/" + local host_header="admin.archivebox.localhost:8000" + local attempts=60 + local attempt + + for ((attempt = 1; attempt <= attempts; attempt++)); do + if curl -fsS -H "Host: ${host_header}" "$url" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + echo "[!] Server process started, but health check did not become ready at $url after ${attempts}s." + echo " Run the logs command below to inspect startup progress." + return 0 +} + +open_archivebox() { + if command -v open > /dev/null; then + open "http://127.0.0.1:8000" || true + fi +} + +if [ "$(id -u)" -eq 0 ]; then echo echo "[X] You cannot run this script as root. You must run it as a non-root user with sudo ability." echo " Create a new non-privileged user 'archivebox' if necessary." @@ -32,35 +61,34 @@ if [ $(id -u) -eq 0 ]; then exit 2 fi -if (which docker > /dev/null && docker pull archivebox/archivebox:latest); then +if (command -v docker > /dev/null && docker compose version > /dev/null && docker pull --platform "$ARCHIVEBOX_PLATFORM" "$ARCHIVEBOX_IMAGE"); then echo "[+] Initializing an ArchiveBox data folder at ~/archivebox/data using Docker Compose..." mkdir -p ~/archivebox/data || exit 1 cd ~/archivebox if [ -f "./index.sqlite3" ]; then mv -i ~/archivebox/* ~/archivebox/data/ fi - curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/docker-compose.yml' > docker-compose.yml - mkdir -p ./etc - curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/stable/etc/sonic.cfg' > ./etc/sonic.cfg - docker compose run --rm archivebox init --setup + curl -fsSL "$ARCHIVEBOX_COMPOSE_URL" > docker-compose.yml + export ARCHIVEBOX_IMAGE ARCHIVEBOX_PLATFORM + docker compose run --rm archivebox init echo echo "[+] Starting ArchiveBox server using: docker compose up -d..." docker compose up -d - sleep 7 - which open > /dev/null && open "http://127.0.0.1:8000" || true + wait_for_archivebox + open_archivebox echo echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized in ~/archivebox/data. Usage:" echo " cd ~/archivebox" echo " docker compose ps" echo " docker compose down" - echo " docker compose pull" + echo " ARCHIVEBOX_IMAGE=$ARCHIVEBOX_IMAGE docker compose pull" echo " docker compose up" echo " docker compose run archivebox manage createsuperuser" echo " docker compose run archivebox add 'https://example.com'" echo " docker compose run archivebox list" echo " docker compose run archivebox help" exit 0 -elif (which docker > /dev/null && docker pull archivebox/archivebox:latest); then +elif (command -v docker > /dev/null && docker pull --platform "$ARCHIVEBOX_PLATFORM" "$ARCHIVEBOX_IMAGE"); then echo "[+] Initializing an ArchiveBox data folder at ~/archivebox/data using Docker..." mkdir -p ~/archivebox/data || exit 1 cd ~/archivebox @@ -68,23 +96,23 @@ elif (which docker > /dev/null && docker pull archivebox/archivebox:latest); the mv -i ~/archivebox/* ~/archivebox/data/ fi cd ./data - docker run -v "$PWD":/data -it --rm archivebox/archivebox:latest init --setup + docker run --platform "$ARCHIVEBOX_PLATFORM" -v "$PWD":/data -it --rm "$ARCHIVEBOX_IMAGE" init echo echo "[+] Starting ArchiveBox server using: docker run -d archivebox/archivebox..." - docker run -v "$PWD":/data -it -d -p 8000:8000 --name=archivebox archivebox/archivebox:latest - sleep 7 - which open > /dev/null && open "http://127.0.0.1:8000" || true + docker run --platform "$ARCHIVEBOX_PLATFORM" -v "$PWD":/data -it -d -p 8000:8000 --name=archivebox "$ARCHIVEBOX_IMAGE" + wait_for_archivebox + open_archivebox echo echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized in ~/archivebox/data. Usage:" echo " cd ~/archivebox/data" echo " docker ps --filter name=archivebox" echo " docker kill archivebox" - echo " docker pull archivebox/archivebox" - echo " docker run -v $PWD:/data -d -p 8000:8000 --name=archivebox archivebox/archivebox" - echo " docker run -v $PWD:/data -it archivebox/archivebox manage createsuperuser" - echo " docker run -v $PWD:/data -it archivebox/archivebox add 'https://example.com'" - echo " docker run -v $PWD:/data -it archivebox/archivebox list" - echo " docker run -v $PWD:/data -it archivebox/archivebox help" + echo " docker pull $ARCHIVEBOX_IMAGE" + echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -d -p 8000:8000 --name=archivebox $ARCHIVEBOX_IMAGE" + echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE manage createsuperuser" + echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE add 'https://example.com'" + echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE list" + echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE help" exit 0 fi @@ -198,13 +226,13 @@ if [ -f "./index.sqlite3" ]; then mv -i ~/archivebox/* ~/archivebox/data/ fi cd ./data -: | python3 -m archivebox init --setup || true # pipe in empty command to make sure stdin is closed +: | python3 -m archivebox init --install || true # pipe in empty command to make sure stdin is closed # init shows version output at the end too echo echo "[+] Starting ArchiveBox server using: nohup archivebox server &..." nohup python3 -m archivebox server 0.0.0.0:8000 > ./logs/server.log 2>&1 & -sleep 7 -which open > /dev/null && open "http://127.0.0.1:8000" || true +wait_for_archivebox +open_archivebox echo echo "[√] Server started on http://0.0.0.0:8000 and data directory initialized in ~/archivebox/data. Usage:" echo " cd ~/archivebox/data # see your data dir" diff --git a/etc/package.json b/etc/package.json index 473e4aa7..84f6150e 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.0.1", + "version": "0.9.31rc2", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 315ade7b..42950677 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.31rc1" +version = "0.9.31rc2" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]