diff --git a/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py b/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py index 86e86763..4e75e7f6 100644 --- a/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py +++ b/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py @@ -128,10 +128,13 @@ def upgrade_crawl_table_from_v086(apps, schema_editor): status, retry_at, created_by_id, schedule_id ) SELECT - REPLACE(id, '-', ''), created_at, modified_at, num_uses_succeeded, num_uses_failed, - '', config, max_depth, tags_str, REPLACE(persona_id, '-', ''), '', '', '', - status, retry_at, created_by_id, REPLACE(schedule_id, '-', '') - FROM crawls_crawl; + REPLACE(old_crawl.id, '-', ''), old_crawl.created_at, old_crawl.modified_at, + old_crawl.num_uses_succeeded, old_crawl.num_uses_failed, + COALESCE(old_seed.uri, ''), old_crawl.config, old_crawl.max_depth, old_crawl.tags_str, + NULL, '', '', '', old_crawl.status, old_crawl.retry_at, + old_crawl.created_by_id, old_crawl.schedule_id + FROM crawls_crawl AS old_crawl + LEFT JOIN seeds_seed AS old_seed ON old_seed.id = old_crawl.seed_id; """) cursor.execute("DROP TABLE crawls_crawl;") diff --git a/archivebox/tests/migrations_helpers.py b/archivebox/tests/migrations_helpers.py index bbbe14f9..1bdc3841 100644 --- a/archivebox/tests/migrations_helpers.py +++ b/archivebox/tests/migrations_helpers.py @@ -372,6 +372,20 @@ CREATE TABLE IF NOT EXISTS core_tag ( ); -- Crawls tables (new in 0.8.x) +CREATE TABLE IF NOT EXISTS seeds_seed ( + num_uses_failed INTEGER NOT NULL DEFAULT 0, + num_uses_succeeded INTEGER NOT NULL DEFAULT 0, + id CHAR(36) PRIMARY KEY, + abid VARCHAR(30) UNIQUE, + uri VARCHAR(2000) NOT NULL, + extractor VARCHAR(32) NOT NULL DEFAULT 'auto', + tags_str VARCHAR(255) NOT NULL DEFAULT '', + config TEXT DEFAULT '{}', + created_at DATETIME NOT NULL, + modified_at DATETIME NOT NULL, + created_by_id INTEGER NOT NULL REFERENCES auth_user(id) +); + CREATE TABLE IF NOT EXISTS crawls_crawlschedule ( id CHAR(36) PRIMARY KEY, created_at DATETIME NOT NULL, @@ -387,23 +401,22 @@ CREATE TABLE IF NOT EXISTS crawls_crawlschedule ( ); CREATE TABLE IF NOT EXISTS crawls_crawl ( + num_uses_failed INTEGER NOT NULL DEFAULT 0, + num_uses_succeeded INTEGER NOT NULL DEFAULT 0, id CHAR(36) PRIMARY KEY, + abid VARCHAR(30) UNIQUE, created_at DATETIME NOT NULL, created_by_id INTEGER NOT NULL REFERENCES auth_user(id), modified_at DATETIME, - urls TEXT NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'queued', + retry_at DATETIME, config TEXT DEFAULT '{}', max_depth SMALLINT UNSIGNED NOT NULL DEFAULT 0, tags_str VARCHAR(1024) NOT NULL DEFAULT '', - persona_id CHAR(36), - label VARCHAR(64) NOT NULL DEFAULT '', - notes TEXT NOT NULL DEFAULT '', + persona VARCHAR(32) NOT NULL DEFAULT 'auto', + seed_id CHAR(36) NOT NULL REFERENCES seeds_seed(id), schedule_id CHAR(36), - output_dir VARCHAR(256) NOT NULL DEFAULT '', - status VARCHAR(16) NOT NULL DEFAULT 'queued', - retry_at DATETIME, - num_uses_failed INTEGER NOT NULL DEFAULT 0, - num_uses_succeeded INTEGER NOT NULL DEFAULT 0 + FOREIGN KEY (schedule_id) REFERENCES crawls_crawlschedule(id) ); -- Core Snapshot table (0.8.x with UUID PK, status, crawl FK) @@ -887,6 +900,7 @@ def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]: created_data = { "users": [], + "seeds": [], "crawls": [], "snapshots": [], "tags": [], @@ -916,30 +930,43 @@ def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]: tag_id = cursor.lastrowid created_data["tags"].append({"id": tag_id, "name": name, "slug": name.lower()}) - # Create 2 Crawls (0.9.0 schema - no seeds) + # Create two real 0.8.x Seed/Crawl pairs. Crawl.urls replaced Seed.uri in 0.9. test_crawls = [ - ("https://example.com\nhttps://example.org", 0, "Example Crawl"), - ("https://github.com/ArchiveBox", 1, "GitHub Crawl"), + ("https://example.com", 0), + ("https://github.com/ArchiveBox", 1), ] - for i, (urls, max_depth, label) in enumerate(test_crawls): + for uri, max_depth in test_crawls: + seed_id = generate_uuid() + cursor.execute( + """ + INSERT INTO seeds_seed ( + id, created_at, created_by_id, modified_at, uri, extractor, + tags_str, config, num_uses_failed, num_uses_succeeded + ) + VALUES (?, datetime('now'), ?, datetime('now'), ?, 'auto', '', '{}', 0, 0) + """, + (seed_id, user_id, uri), + ) + created_data["seeds"].append({"id": seed_id, "uri": uri}) + crawl_id = generate_uuid() cursor.execute( """ - INSERT INTO crawls_crawl (id, created_at, created_by_id, modified_at, urls, - config, max_depth, tags_str, label, status, retry_at, + INSERT INTO crawls_crawl (id, created_at, created_by_id, modified_at, seed_id, + config, max_depth, tags_str, persona, status, retry_at, num_uses_failed, num_uses_succeeded) - VALUES (?, datetime('now'), ?, datetime('now'), ?, '{}', ?, '', ?, 'queued', datetime('now'), 0, 0) + VALUES (?, datetime('now'), ?, datetime('now'), ?, '{}', ?, '', 'auto', 'queued', datetime('now'), 0, 0) """, - (crawl_id, user_id, urls, max_depth, label), + (crawl_id, user_id, seed_id, max_depth), ) created_data["crawls"].append( { "id": crawl_id, - "urls": urls, + "urls": uri, "max_depth": max_depth, - "label": label, + "label": "", }, )