mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
363 lines
17 KiB
JavaScript
Executable File
363 lines
17 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const puppeteer = require('puppeteer');
|
|
|
|
const DEFAULT_URL = 'http://web.archivebox.localhost:8000/add/';
|
|
const DEFAULT_OUTPUT = 'tmp/add-page.png';
|
|
|
|
function usage() {
|
|
console.log(`
|
|
Usage:
|
|
node bin/take_screenshot.js [url] [output.png]
|
|
|
|
Environment:
|
|
SESSIONID Django session cookie value for admin.archivebox.localhost
|
|
SCREENSHOT_COOKIE_NAME Cookie name, defaults to sessionid
|
|
SCREENSHOT_COOKIE_DOMAIN Cookie domain, defaults to admin.archivebox.localhost
|
|
SCREENSHOT_USER_DATA_DIR Chrome profile directory (e.g. a persona's chrome_profile)
|
|
SCREENSHOT_LOGIN_USERNAME Log in through #id_username before capture
|
|
SCREENSHOT_LOGIN_PASSWORD Password used with SCREENSHOT_LOGIN_USERNAME
|
|
CHROME_BINARY Chromium/Chrome executable path
|
|
SCREENSHOT_WIDTH Viewport width, defaults to 1600
|
|
SCREENSHOT_HEIGHT Viewport height, defaults to 1400
|
|
SCREENSHOT_VARIANTS_JSON JSON array of {path,width,height} captures from one loaded page
|
|
SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only
|
|
SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture
|
|
SCREENSHOT_WAIT_SELECTOR Wait for this selector before capture
|
|
SCREENSHOT_CLICK_SELECTOR Click this selector before capture
|
|
SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR Wait for this selector after clicking
|
|
SCREENSHOT_HOST_RESOLVER_RULES Chrome host resolver rules
|
|
SCREENSHOT_SNAPSHOT_VIEW Set to list or grid before loading the page
|
|
SCREENSHOT_SNAPSHOT_HEADER Set to expanded or collapsed before loading a snapshot detail page
|
|
SCREENSHOT_EXPECT_PLUGIN Require this snapshot output plugin to be selected
|
|
SCREENSHOT_EXPECT_LIVE_PROGRESS Require real progress bars and a loaded screencast frame
|
|
SCREENSHOT_COLLAPSE_FILTERS Set to 1 to keep admin filters out of screenshots
|
|
SCREENSHOT_RESET_FILTERS Set to 1 to clear the admin filter collapsed preference
|
|
`);
|
|
}
|
|
|
|
function argValue(name) {
|
|
const idx = process.argv.indexOf(name);
|
|
return idx === -1 ? null : process.argv[idx + 1];
|
|
}
|
|
|
|
function firstPositionalArg(index) {
|
|
return process.argv.slice(2).filter((arg) => !arg.startsWith('--'))[index] || null;
|
|
}
|
|
|
|
function chromePath() {
|
|
const configured = process.env.CHROME_BINARY;
|
|
if (!configured || !path.isAbsolute(configured) || !fs.existsSync(configured)) {
|
|
throw new Error('CHROME_BINARY must be an absolute executable path resolved by abxpkg');
|
|
}
|
|
return configured;
|
|
}
|
|
|
|
function needsNoSandbox() {
|
|
return process.platform === 'linux' && !String(process.env.DISPLAY || '').trim();
|
|
}
|
|
|
|
function addLaunchArg(args, arg) {
|
|
if (!args.includes(arg)) args.push(arg);
|
|
}
|
|
|
|
async function main() {
|
|
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
|
usage();
|
|
return;
|
|
}
|
|
|
|
const url = argValue('--url') || firstPositionalArg(0) || DEFAULT_URL;
|
|
const output = path.resolve(argValue('--output') || firstPositionalArg(1) || DEFAULT_OUTPUT);
|
|
const width = Number(process.env.SCREENSHOT_WIDTH || 1600);
|
|
const height = Number(process.env.SCREENSHOT_HEIGHT || 1400);
|
|
const fullPage = process.env.SCREENSHOT_FULL_PAGE === '1';
|
|
const variants = process.env.SCREENSHOT_VARIANTS_JSON
|
|
? JSON.parse(process.env.SCREENSHOT_VARIANTS_JSON)
|
|
: [];
|
|
|
|
if (!Array.isArray(variants)) {
|
|
throw new Error('SCREENSHOT_VARIANTS_JSON must be a JSON array');
|
|
}
|
|
for (const variant of variants) {
|
|
if (!variant.path || !Number.isInteger(variant.width) || !Number.isInteger(variant.height)) {
|
|
throw new Error('Each screenshot variant requires path, integer width, and integer height');
|
|
}
|
|
}
|
|
|
|
fs.mkdirSync(path.dirname(output), { recursive: true });
|
|
|
|
const launchOptions = {
|
|
headless: true,
|
|
defaultViewport: { width, height },
|
|
executablePath: chromePath(),
|
|
args: [],
|
|
};
|
|
if (process.env.SCREENSHOT_USER_DATA_DIR) {
|
|
launchOptions.userDataDir = path.resolve(process.env.SCREENSHOT_USER_DATA_DIR);
|
|
}
|
|
if (process.env.SCREENSHOT_HOST_RESOLVER_RULES) {
|
|
addLaunchArg(launchOptions.args, `--host-resolver-rules=${process.env.SCREENSHOT_HOST_RESOLVER_RULES}`);
|
|
}
|
|
if (needsNoSandbox()) {
|
|
addLaunchArg(launchOptions.args, '--no-sandbox');
|
|
addLaunchArg(launchOptions.args, '--disable-setuid-sandbox');
|
|
}
|
|
const browser = await puppeteer.launch(launchOptions);
|
|
try {
|
|
// Persistent persona profiles can restore tabs left by earlier launches.
|
|
// Close them before navigating so repeated captures do not multiply live
|
|
// admin requests and overwhelm the server being documented.
|
|
const restoredPages = await browser.pages();
|
|
const page = await browser.newPage();
|
|
await page.setCacheEnabled(false);
|
|
await Promise.all(restoredPages.map((restoredPage) => restoredPage.close()));
|
|
page.setDefaultTimeout(45000);
|
|
|
|
const client = await page.createCDPSession();
|
|
const documentRequests = new Map();
|
|
await client.send('Network.enable');
|
|
client.on('Network.requestWillBeSent', (event) => {
|
|
if (event.type !== 'Document') return;
|
|
documentRequests.set(event.requestId, {
|
|
url: event.request.url,
|
|
requestTimestamp: event.timestamp,
|
|
});
|
|
});
|
|
client.on('Network.responseReceived', (event) => {
|
|
if (event.type !== 'Document') return;
|
|
const record = documentRequests.get(event.requestId);
|
|
if (!record) return;
|
|
record.responseTimestamp = event.timestamp;
|
|
record.status = event.response.status;
|
|
record.responseUrl = event.response.url;
|
|
if (Number.isFinite(record.requestTimestamp) && Number.isFinite(record.responseTimestamp)) {
|
|
record.ttfbMs = Math.round((record.responseTimestamp - record.requestTimestamp) * 1000);
|
|
}
|
|
});
|
|
|
|
if (process.env.SCREENSHOT_SNAPSHOT_VIEW || process.env.SCREENSHOT_SNAPSHOT_HEADER || process.env.SCREENSHOT_COLLAPSE_FILTERS === '1' || process.env.SCREENSHOT_RESET_FILTERS === '1') {
|
|
await page.evaluateOnNewDocument((snapshotView, snapshotHeader, collapseFilters, resetFilters) => {
|
|
if (snapshotView) localStorage.setItem('preferred_snapshot_view_mode', snapshotView);
|
|
if (snapshotHeader === 'expanded') localStorage.setItem('archivebox-snapshot-header-visible', 'true');
|
|
if (snapshotHeader === 'collapsed') localStorage.setItem('archivebox-snapshot-header-visible', 'false');
|
|
if (collapseFilters) localStorage.setItem('admin-filters-collapsed', 'true');
|
|
else if (resetFilters) localStorage.removeItem('admin-filters-collapsed');
|
|
}, process.env.SCREENSHOT_SNAPSHOT_VIEW || '', process.env.SCREENSHOT_SNAPSHOT_HEADER || '', process.env.SCREENSHOT_COLLAPSE_FILTERS === '1', process.env.SCREENSHOT_RESET_FILTERS === '1');
|
|
}
|
|
|
|
if (process.env.SESSIONID) {
|
|
const cookie = {
|
|
name: process.env.SCREENSHOT_COOKIE_NAME || 'sessionid',
|
|
value: process.env.SESSIONID,
|
|
path: '/',
|
|
};
|
|
if (process.env.SCREENSHOT_COOKIE_DOMAIN) {
|
|
cookie.domain = process.env.SCREENSHOT_COOKIE_DOMAIN;
|
|
} else {
|
|
cookie.url = new URL(url).origin;
|
|
}
|
|
await page.setCookie(cookie);
|
|
}
|
|
|
|
let navigationResponse = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
|
|
|
if (process.env.SCREENSHOT_LOGIN_USERNAME || process.env.SCREENSHOT_LOGIN_PASSWORD) {
|
|
if (!process.env.SCREENSHOT_LOGIN_USERNAME || !process.env.SCREENSHOT_LOGIN_PASSWORD) {
|
|
throw new Error('SCREENSHOT_LOGIN_USERNAME and SCREENSHOT_LOGIN_PASSWORD must be set together');
|
|
}
|
|
await page.waitForSelector('#id_username');
|
|
await page.type('#id_username', process.env.SCREENSHOT_LOGIN_USERNAME);
|
|
await page.type('#id_password', process.env.SCREENSHOT_LOGIN_PASSWORD);
|
|
const [loginResponse] = await Promise.all([
|
|
page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }),
|
|
page.click('button[type="submit"], input[type="submit"]'),
|
|
]);
|
|
navigationResponse = loginResponse || navigationResponse;
|
|
if (new URL(page.url()).pathname.includes('/admin/login/')) {
|
|
const loginError = await page.evaluate(() => Array.from(document.querySelectorAll('.errornote, .errorlist')).map((element) => element.textContent.trim()).filter(Boolean).join(' '));
|
|
const cookieDetails = (await page.cookies()).map((cookie) => `${cookie.name}@${cookie.domain}${cookie.path}${cookie.secure ? ';Secure' : ''}`).join(', ') || 'none';
|
|
throw new Error(`ArchiveBox persona login failed at ${page.url()}: ${loginError || 'no form error shown'} (cookies: ${cookieDetails})`);
|
|
}
|
|
}
|
|
|
|
await page.waitForSelector('body');
|
|
await page.waitForSelector('#progress-monitor, #add-form', { timeout: 5000 }).catch(() => {});
|
|
if (process.env.SCREENSHOT_WAIT_SELECTOR) {
|
|
await page.waitForSelector(process.env.SCREENSHOT_WAIT_SELECTOR, { timeout: 45000 });
|
|
}
|
|
if (process.env.SCREENSHOT_CLICK_SELECTOR) {
|
|
await page.waitForSelector(process.env.SCREENSHOT_CLICK_SELECTOR, { timeout: 45000 });
|
|
await page.click(process.env.SCREENSHOT_CLICK_SELECTOR);
|
|
}
|
|
if (process.env.SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR) {
|
|
await page.waitForSelector(process.env.SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR, { timeout: 45000 });
|
|
}
|
|
if (process.env.SCREENSHOT_SCROLL_SELECTOR) {
|
|
await page.waitForSelector(process.env.SCREENSHOT_SCROLL_SELECTOR, { timeout: 45000 }).catch(() => {});
|
|
await page.evaluate((selector) => {
|
|
document.querySelector(selector)?.scrollIntoView({ block: 'start', inline: 'nearest' });
|
|
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
|
|
}
|
|
|
|
if (process.env.SCREENSHOT_EXPECT_PLUGIN) {
|
|
const expectedPlugin = process.env.SCREENSHOT_EXPECT_PLUGIN.toLowerCase();
|
|
await page.waitForFunction((pluginName) => {
|
|
const selectedCard = document.querySelector('.thumb-card.selected-card[data-plugin-name]');
|
|
const frame = document.querySelector('#main-frame');
|
|
const frameRect = frame?.getBoundingClientRect();
|
|
return selectedCard?.dataset.pluginName?.toLowerCase() === pluginName
|
|
&& frame
|
|
&& frame.getAttribute('src')
|
|
&& frame.getAttribute('src') !== 'about:blank'
|
|
&& frameRect
|
|
&& frameRect.width >= Math.min(window.innerWidth * 0.9, window.innerWidth - 8)
|
|
&& frameRect.height >= Math.min(window.innerHeight * 0.35, 320);
|
|
}, { timeout: 45000 }, expectedPlugin);
|
|
await page.waitForFunction(() => {
|
|
const frames = [document.querySelector('#main-frame'), document.querySelector('.thumb-card.selected-card iframe')]
|
|
.filter(Boolean);
|
|
if (!frames.length) return true;
|
|
return frames.every((frame) => {
|
|
try {
|
|
const doc = frame.contentDocument || frame.contentWindow?.document;
|
|
if (!doc || doc.readyState === 'loading') return false;
|
|
const images = Array.from(doc.images || []);
|
|
return images.every((img) => img.complete && (img.naturalWidth > 0 || img.currentSrc.startsWith('data:')));
|
|
} catch (err) {
|
|
return true;
|
|
}
|
|
});
|
|
}, { timeout: 45000, polling: 250 }).catch(() => {});
|
|
}
|
|
|
|
if (process.env.SCREENSHOT_EXPECT_LIVE_PROGRESS === '1') {
|
|
await page.waitForFunction(() => {
|
|
const monitor = document.querySelector('#progress-monitor');
|
|
const bars = [...document.querySelectorAll('#progress-monitor .progress-bar')]
|
|
.filter((bar) => bar.getClientRects().length > 0 && bar.offsetWidth > 0 && bar.offsetHeight > 0);
|
|
const panel = document.querySelector('#progress-monitor .screencast-panel.visible');
|
|
const image = panel?.querySelector('img');
|
|
const placeholder = panel?.querySelector('.screencast-placeholder');
|
|
return monitor
|
|
&& getComputedStyle(monitor).display !== 'none'
|
|
&& !monitor.classList.contains('collapsed')
|
|
&& monitor.querySelector('.progress-content')?.getClientRects().length > 0
|
|
&& bars.length >= 2
|
|
&& panel?.getClientRects().length > 0
|
|
&& panel.offsetWidth > 0
|
|
&& panel.offsetHeight > 0
|
|
&& (
|
|
(
|
|
image?.complete
|
|
&& image.naturalWidth > 0
|
|
&& image.naturalHeight > 0
|
|
)
|
|
|| placeholder?.getClientRects().length > 0
|
|
);
|
|
}, { timeout: 120000, polling: 250 });
|
|
}
|
|
|
|
const frameHandle = await page.$('.crawl-snapshots-embed iframe');
|
|
if (frameHandle) {
|
|
const frame = await frameHandle.contentFrame();
|
|
if (frame) {
|
|
await frame.waitForSelector('#changelist-form', { timeout: 45000 }).catch(() => {});
|
|
await frame.waitForSelector('#result_list', { timeout: 45000 }).catch(() => {});
|
|
}
|
|
}
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 1200));
|
|
|
|
const checks = await page.evaluate(() => ({
|
|
url: location.href,
|
|
title: document.title,
|
|
progressMonitorDisplay: document.querySelector('#progress-monitor')
|
|
? getComputedStyle(document.querySelector('#progress-monitor')).display
|
|
: 'missing',
|
|
progressCrawls: document.querySelectorAll('#progress-monitor .crawl-item').length,
|
|
progressSnapshots: document.querySelectorAll('#progress-monitor .snapshot-item').length,
|
|
screencastVisible: Boolean(document.querySelector('#progress-monitor .screencast-panel.visible')),
|
|
screencastPlaceholderVisible: Boolean(document.querySelector('#progress-monitor .screencast-panel.visible .screencast-placeholder')),
|
|
screencastImageLoaded: (() => {
|
|
const img = document.querySelector('#progress-monitor .screencast-panel.visible img');
|
|
return Boolean(img && img.complete && img.naturalWidth > 0 && img.naturalHeight > 0);
|
|
})(),
|
|
screencastImageSize: (() => {
|
|
const img = document.querySelector('#progress-monitor .screencast-panel.visible img');
|
|
return img ? `${img.naturalWidth}x${img.naturalHeight}` : '';
|
|
})(),
|
|
snapshotEmbed: Boolean(document.querySelector('.crawl-snapshots-embed iframe')),
|
|
addForm: Boolean(document.querySelector('#add-form')),
|
|
limitFields: Array.from(document.querySelectorAll('.crawl-limit-field label')).map((el) => el.textContent.trim()),
|
|
snapshotOutputPlugins: [...new Set(
|
|
[...document.querySelectorAll('.thumb-card[data-plugin-name] a[target="preview"]')]
|
|
.map((link) => link.closest('.thumb-card')?.dataset.pluginName)
|
|
.filter(Boolean),
|
|
)],
|
|
snapshotOutputs: [...document.querySelectorAll('.thumb-card[data-plugin-name]')]
|
|
.map((card) => ({
|
|
plugin: card.dataset.pluginName || '',
|
|
previewUrl: card.dataset.previewUrl || card.querySelector('a[target="preview"]')?.getAttribute('href') || '',
|
|
}))
|
|
.filter((output) => output.plugin && output.previewUrl),
|
|
selectedSnapshotOutputPlugin: document.querySelector('.thumb-card.selected-card[data-plugin-name]')?.dataset.pluginName || '',
|
|
}));
|
|
checks.status = navigationResponse ? navigationResponse.status() : null;
|
|
checks.finalUrl = page.url();
|
|
const navigationUrl = navigationResponse?.url() || page.url();
|
|
const matchingDocumentRequests = [...documentRequests.values()]
|
|
.filter((record) => record.status === checks.status && Number.isFinite(record.ttfbMs))
|
|
.filter((record) => {
|
|
const responseUrl = record.responseUrl || record.url || '';
|
|
return responseUrl === navigationUrl || responseUrl === checks.finalUrl || responseUrl.split('#')[0] === checks.finalUrl.split('#')[0];
|
|
});
|
|
const timingRecord = matchingDocumentRequests.at(-1)
|
|
|| [...documentRequests.values()].filter((record) => Number.isFinite(record.ttfbMs)).at(-1);
|
|
if (timingRecord) {
|
|
checks.ttfbMs = timingRecord.ttfbMs;
|
|
}
|
|
|
|
let frameChecks = null;
|
|
const embeddedFrameHandle = await page.$('.crawl-snapshots-embed iframe');
|
|
if (embeddedFrameHandle) {
|
|
const frame = await embeddedFrameHandle.contentFrame();
|
|
if (frame) {
|
|
frameChecks = await frame.evaluate(() => ({
|
|
rows: document.querySelectorAll('#result_list tbody tr').length,
|
|
actionCheckboxes: document.querySelectorAll('#result_list input.action-select').length,
|
|
searchModeRadios: document.querySelectorAll('#changelist-search input[type="radio"][name="search_mode"]').length,
|
|
progressMonitorDisplay: document.querySelector('#progress-monitor')
|
|
? getComputedStyle(document.querySelector('#progress-monitor')).display
|
|
: 'missing',
|
|
}));
|
|
}
|
|
}
|
|
|
|
const screenshotPaths = [];
|
|
if (variants.length) {
|
|
for (const variant of variants) {
|
|
const variantPath = path.resolve(variant.path);
|
|
fs.mkdirSync(path.dirname(variantPath), { recursive: true });
|
|
await page.setViewport({width: variant.width, height: variant.height, deviceScaleFactor: 1});
|
|
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
await page.screenshot({ path: variantPath, fullPage });
|
|
screenshotPaths.push(variantPath);
|
|
}
|
|
} else {
|
|
await page.screenshot({ path: output, fullPage });
|
|
screenshotPaths.push(output);
|
|
}
|
|
console.log(JSON.stringify({ screenshotPath: screenshotPaths[0], screenshotPaths, checks, frameChecks }, null, 2));
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|