ArchiveBox/archivebox/core/forms.py

__package__ = 'archivebox.core'

from django import forms
from django.utils.html import format_html

from archivebox.misc.util import URL_REGEX, find_all_urls
from taggit.utils import edit_string_for_tags, parse_tags
from archivebox.base_models.admin import KeyValueWidget
from archivebox.crawls.schedule_utils import validate_schedule
from archivebox.config.common import SEARCH_BACKEND_CONFIG
from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
from archivebox.personas.models import Persona

DEPTH_CHOICES = (
    ('0', 'depth = 0 (archive just these URLs)'),
    ('1', 'depth = 1 (+ URLs one hop away)'),
    ('2', 'depth = 2 (+ URLs two hops away)'),
    ('3', 'depth = 3 (+ URLs three hops away)'),
    ('4', 'depth = 4 (+ URLs four hops away)'),
)


def get_plugin_choices():
    """Get available extractor plugins from discovered hooks."""
    return [(name, name) for name in get_plugins()]


def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
    schema = plugin_configs.get(plugin_name, {})
    description = str(schema.get('description') or '').strip()
    if not description:
        return plugin_name
    icon_html = get_plugin_icon(plugin_name)

    return format_html(
        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span><a class="plugin-choice-description" href="https://archivebox.github.io/abx-plugins/#{}" target="_blank" rel="noopener noreferrer">{}</a>',
        icon_html,
        plugin_name,
        plugin_name,
        description,
    )


def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
    field = form.fields[name]
    if not isinstance(field, forms.ChoiceField):
        raise TypeError(f'{name} must be a ChoiceField')
    return field


class AddLinkForm(forms.Form):
    # Basic fields
    url = forms.CharField(
        label="URLs",
        strip=True,
        widget=forms.Textarea(attrs={
            'data-url-regex': URL_REGEX.pattern,
        }),
        required=True
    )
    tag = forms.CharField(
        label="Tags",
        strip=True,
        required=False,
        widget=TagEditorWidget(),
    )
    depth = forms.ChoiceField(
        label="Archive depth",
        choices=DEPTH_CHOICES,
        initial='0',
        widget=forms.RadioSelect(attrs={"class": "depth-selection"})
    )
    notes = forms.CharField(
        label="Notes",
        strip=True,
        required=False,
        widget=forms.TextInput(attrs={
            'placeholder': 'Optional notes about this crawl',
        })
    )
    url_filters = forms.Field(
        label="URL allowlist / denylist",
        required=False,
        widget=URLFiltersWidget(source_selector='textarea[name="url"]'),
    )

    # Plugin groups
    chrome_plugins = forms.MultipleChoiceField(
        label="Chrome-dependent plugins",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],  # populated in __init__
    )
    archiving_plugins = forms.MultipleChoiceField(
        label="Archiving",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
    parsing_plugins = forms.MultipleChoiceField(
        label="Parsing",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
    search_plugins = forms.MultipleChoiceField(
        label="Search",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
    binary_plugins = forms.MultipleChoiceField(
        label="Binary providers",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )
    extension_plugins = forms.MultipleChoiceField(
        label="Browser extensions",
        required=False,
        widget=forms.CheckboxSelectMultiple,
        choices=[],
    )

    # Advanced options
    schedule = forms.CharField(
        label="Repeat schedule",
        max_length=64,
        required=False,
        widget=forms.TextInput(attrs={
            'placeholder': 'e.g., daily, weekly, 0 */6 * * * (every 6 hours)',
        })
    )
    persona = forms.ModelChoiceField(
        label="Persona (authentication profile)",
        required=False,
        queryset=Persona.objects.none(),
        empty_label=None,
        to_field_name='name',
    )
    index_only = forms.BooleanField(
        label="Index only dry run (add crawl but don't archive yet)",
        initial=False,
        required=False,
    )
    config = forms.JSONField(
        label="Custom config overrides",
        widget=KeyValueWidget(),
        initial=dict,
        required=False,
    )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        default_persona = Persona.get_or_create_default()
        self.fields['persona'].queryset = Persona.objects.order_by('name')
        self.fields['persona'].initial = default_persona.name

        # Get all plugins
        all_plugins = get_plugins()
        plugin_configs = discover_plugin_configs()

        # Define plugin groups
        chrome_dependent = {
            'accessibility', 'chrome', 'consolelog', 'dom', 'headers',
            'parse_dom_outlinks', 'pdf', 'redirects', 'responses',
            'screenshot', 'seo', 'singlefile', 'ssl', 'staticfile', 'title'
        }
        archiving = {
            'archivedotorg', 'defuddle', 'favicon', 'forumdl', 'gallerydl', 'git',
            'htmltotext', 'mercury', 'papersdl', 'readability', 'trafilatura', 'wget', 'ytdlp'
        }
        parsing = {
            'parse_html_urls', 'parse_jsonl_urls',
            'parse_netscape_urls', 'parse_rss_urls', 'parse_txt_urls'
        }
        search = {
            'search_backend_ripgrep', 'search_backend_sonic', 'search_backend_sqlite'
        }
        binary = {'apt', 'brew', 'custom', 'env', 'npm', 'pip'}
        extensions = {'twocaptcha', 'istilldontcareaboutcookies', 'ublock'}

        # Populate plugin field choices
        get_choice_field(self, 'chrome_plugins').choices = [
            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in chrome_dependent
        ]
        get_choice_field(self, 'archiving_plugins').choices = [
            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in archiving
        ]
        get_choice_field(self, 'parsing_plugins').choices = [
            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in parsing
        ]
        get_choice_field(self, 'search_plugins').choices = [
            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in search
        ]
        get_choice_field(self, 'binary_plugins').choices = [
            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in binary
        ]
        get_choice_field(self, 'extension_plugins').choices = [
            (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in extensions
        ]

        required_search_plugin = f'search_backend_{SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}'.strip()
        search_choices = [choice[0] for choice in get_choice_field(self, 'search_plugins').choices]
        if required_search_plugin in search_choices:
            get_choice_field(self, 'search_plugins').initial = [required_search_plugin]

    def clean(self):
        cleaned_data = super().clean() or {}

        # Combine all plugin groups into single list
        all_selected_plugins = []
        for field in ['chrome_plugins', 'archiving_plugins', 'parsing_plugins',
                      'search_plugins', 'binary_plugins', 'extension_plugins']:
            selected = cleaned_data.get(field)
            if isinstance(selected, list):
                all_selected_plugins.extend(selected)

        # Store combined list for easy access
        cleaned_data['plugins'] = all_selected_plugins

        return cleaned_data

    def clean_url(self):
        value = self.cleaned_data.get('url') or ''
        urls = '\n'.join(find_all_urls(value))
        if not urls:
            raise forms.ValidationError('Enter at least one valid URL.')
        return urls

    def clean_url_filters(self):
        from archivebox.crawls.models import Crawl

        value = self.cleaned_data.get('url_filters') or {}
        return {
            'allowlist': '\n'.join(Crawl.split_filter_patterns(value.get('allowlist', ''))),
            'denylist': '\n'.join(Crawl.split_filter_patterns(value.get('denylist', ''))),
            'same_domain_only': bool(value.get('same_domain_only')),
        }

    def clean_schedule(self):
        schedule = (self.cleaned_data.get('schedule') or '').strip()
        if not schedule:
            return ''

        try:
            validate_schedule(schedule)
        except ValueError as err:
            raise forms.ValidationError(str(err))

        return schedule


class TagWidget(forms.TextInput):
    def format_value(self, value):
        if value is not None and not isinstance(value, str):
            value = edit_string_for_tags(value)
        return super().format_value(value)


class TagField(forms.CharField):
    widget = TagWidget

    def clean(self, value):
        value = super().clean(value)
        try:
            return parse_tags(value)
        except ValueError:
            raise forms.ValidationError(
                "Please provide a comma-separated list of tags."
            )

    def has_changed(self, initial, data):
        # Always return False if the field is disabled since self.bound_data
        # always uses the initial value in this case.
        if self.disabled:
            return False

        try:
            cleaned_data = self.clean(data)
        except forms.ValidationError:
            cleaned_data = data

        initial_value = [] if initial is None else initial

        if not isinstance(initial_value, list):
            initial_value = list(initial_value)

        normalized_initial = sorted(tag.name for tag in initial_value)
        return normalized_initial != cleaned_data