OSSForks/ArchiveBox
1
0
Fork 0
mirror of https://github.com/zebrajr/ArchiveBox.git synced 2026-01-15 12:15:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
Files
dev
ArchiveBox/old/archivebox.ts

6109 lines
267 KiB
TypeScript
Raw Permalink Normal View History

wip major changes
2025-12-24 20:09:51 -08:00
tring';
import { Readable } from 'node:stream';
import { finished } from 'node:stream/promises';
import { URL } from 'node:url';
import util from 'node:util';
const exec = util.promisify(child_process.exec);
import { Readability } from '@mozilla/readability';
import FileCookieStore from '@root/file-cookie-store';
import merge from 'deepmerge';
import { createCursor, getRandomPagePoint } from 'ghost-cursor';
import { JSDOM, VirtualConsole } from 'jsdom';
import mime from 'mime-types';
import ToughCookie from 'tough-cookie';
import unzip from 'unzip-crx-3';
import puppeteer from 'puppeteer';
import { Browser, Page, Cookie, HTTPResponse } from 'puppeteer';
import { Cluster } from 'puppeteer-cluster';
import PupeteerExtra from "puppeteer-extra";
import Stealth#!/usr/bin/env node --env-file .env
// https://gist.github.com/pirate/d9a350e83025a1e6cf452cddd815d0d4
// npm install request node-request minimist deepmerge mime-types decompress puppeteer-extra puppeteer-extra-plugin-repl puppeteer-extra-plugin-user-preferences puppeteer-extra-plugin-recaptcha puppeteer-extra-plugin-stealth puppeteer-screen-recorder puppeteer-cluster ghost-cursor @mozilla/readability jsdom unzip-crx-3 node-fetch@2
import assert from 'node:assert/strict';
import { Buffer } from 'node:buffer';
import child_process from 'node:child_process';
import crypto from 'node:crypto';
import fs from 'node:fs';
import { createServer } from 'node:http';
import os from 'node:os';
import path from 'node:path';
import querystring from 'node:querysPlugin from "puppeteer-extra-plugin-stealth";
import PrefsPlugin from 'puppeteer-extra-plugin-user-preferences';
import { PuppeteerScreenRecorder } from 'puppeteer-screen-recorder';
// import RecaptchaPlugin from 'puppeteer-extra-plugin-recaptcha';
// import ReplPlugin from 'puppeteer-extra-plugin-repl';
const __dirname = import.meta.dirname
import { getDatabase } from './models/init-models.js';
const { Tag, Snapshot, ArchiveResult } = await getDatabase({ dbpath: './index.sqlite3' })
// move mitm CA cert into /usr/local/share/ca-certificates/mitmproxy-ca-cert.crt
// update-ca-certificates
const ANSI = {
reset: "\x1b[0m",
blue: "\x1b[34m",
black: "\x1b[30m",
}
/************************* Main Input Arguments *******************************/
let URLS = [
// 'chrome://about',
// 'chrome://system/#chrome_root_store',
'https://facebook.com/815781663692514/?comment_id=1508571679703640',
'https://www.instagram.com/p/CrTY1fENHr5/',
'https://www.tiktok.com/@zemmour_eric/video/7342474065598319904?cid=7343316616878490400',
'https://twitter.com/DZasken68678/status/1799833933271687304',
'https://t.me/IONONMIARRENDOGROUP/13598',
'https://www.youtube.com/watch?v=rpD0qgzlCms',
'https://www.aap.com.au/factcheck/aboriginal-lands-claim-a-total-abdication-of-facts/',
'https://gologin.com/check-browser',
'https://arh.antoinevastel.com/bots/areyouheadless',
'https://2captcha.com/demo/hcaptcha',
'https://2captcha.com/demo/cloudflare-turnstile',
'https://2captcha.com/demo/recaptcha-v3',
'https://ipinfo.io/',
// 'https://2captcha.com/demo/recaptcha-v2',
// 'https://2captcha.com/demo/keycaptcha',
// 'https://browserleaks.com/canvas',
// 'https://bot.incolumitas.com/#botChallenge',
// 'https://infosimples.github.io/detect-headless/',
// 'https://coveryourtracks.eff.org/',
// 'https://fingerprint.com/demo/',
// 'https://nowsecure.nl',
// 'https://abrahamjuliot.github.io/creepjs/',
// 'https://scrapfly.io/web-scraping-tools/http2-fingerprint',
// 'https://scrapfly.io/web-scraping-tools/browser-fingerprint',
// 'https://scrapfly.io/web-scraping-tools/ja3-fingerprint',
// 'https://scrapfly.io/web-scraping-tools/canvas-fingerprint',
// 'https://scrapfly.io/web-scraping-tools/webgl-fingerprint',
// 'https://scrapfly.io/web-scraping-tools/audio-fingerprint',
// 'https://scrapfly.io/web-scraping-tools/screen-fingerprint',
// 'https://web-scraping.dev/',
// 'https://example.com',
// 'https://www.okta.com/',
// 'https://www.webflow.com/',
// 'https://docker-compose.archivebox.io',
// 'https://www.reddit.com/r/AskReddit/comments/1br0q9b/what_was_ok_10_years_ago_but_isnt_today/',
// 'https://www.quora.com/Is-the-website-2Captcha-true-or-fake-with-paying-money-for-working-on-it',
// 'https://x.com/yawnzzcalo7/status/1747853178849435894',
// 'https://twitter.com/yawnzzcalo7/status/1747853178849435894',
// 'https://rachdele.substack.com/p/is-the-job-market-dying',
// 'https://www.flowradar.com/cloneables/mouse-image-trail-effect',
// 'https://wrong.host.badssl.com/',
// 'http://docker-compose.archivebox.io',
// 'https://pptr.dev/api/puppeteer.page.setrequestinterception',
// 'https://blog.sweeting.me#Writing',
// 'https://github.com/yarnpkg/yarn/issues/9005',
// 'https://archive.md/739Oc',
// 'https://archive.md/Oc72d',
// 'https://archive.vn/fPUBe',
// 'https://archive.vn/mRz4P',
// 'https://archive.vn/Qct6Y',
// 'https://archive.vn/sv50h',
// 'https://facebook.com/815781663692514/?comment_id=1508571679703640',
// 'https://facebook.com/815781663692514/?comment_id=924451748966499',
// 'https://www.facebook.com/wayne.brennan.528/posts/pfbid02fvxFppng2WsHMavhBa62cXizCBGdmPQRH3CMhac79qzS5C1ADaSNC587d3u6qVbkl',
// 'https://www.facebook.com/wildeprods/posts/pfbid02YEPfoB7pZqMNzE4y2MpYSQbRAzASquvHyEMzHqrNngJCSL7onEg2jnsqS6epcQHWl',
// 'https://t.me/aubontouite_francais/9493',
// 'https://t.me/BC_BLACKMIROR/5044',
// 'https://t.me/IONONMIARRENDOGROUP/14004',
// 'https://t.me/newsfactory_pl/51014',
// 'https://t.me/oliverjanich/132574',
// 'https://t.me/tomaszgryguc/10449',
// 'https://t.me/amigosDisidentes/123177',
// 'https://twitter.com/1nfiltr4do_NN/status/1767238399943991389',
// 'https://twitter.com/4lmondcookie/status/1748519205438111914',
// 'https://twitter.com/4olll1ke/status/1753796944827199766',
// 'https://twitter.com/yeokiloss/status/1754908226179502345',
// 'https://twitter.com/YoungWaifLover/status/1735667278090297561',
// 'https://twitter.com/Z_Pour_Demain/status/1766133730278605182',
// 'https://www.aap.com.au/factcheck/aboriginal-lands-claim-a-total-abdication-of-facts/',
// 'https://www.aap.com.au/factcheck/absurd-albanese-clip-fools-voice-voters/',
// 'https://www.instagram.com/_the.forgotten.ones/p/CQQDyoqhsF6/',
// 'https://www.instagram.com/p/CqSM_f9MR4b/',
// 'https://www.instagram.com/p/CqSQgf1sv8B/',
// 'https://instagram.com/p/B-Q22Z_pxyC/',
// 'https://www.tiktok.com/@zitatezurzeit/photo/7342474065598319904?cid=7343316616878490400',
// 'https://tiktok.com/@zitatezurzeit/photo/7342474065598319904?cid=7343316616878490400',
// 'https://www.youtube.com/watch?v=rpD0qgzlCms',
]
const isTruthy = (env_value) => ['1', 'yes', 'true'].includes(env_value?.toLowerCase() || 'false')
/********************** Config: General High-Level Options ********************/
const PASSIVE_ARCHIVING = isTruthy(process.env.PASSIVE_ARCHIVING)
const CHROME_CLUSTER = isTruthy(process.env.CHROME_CLUSTER)
const CHROME_CLUSTER_WORKERS = 4
const API_SERVER_HOST = '0.0.0.0'
const API_SERVER_PORT = 9595
const CHROME_DEBUG_PORT = 9222 // 9222 is default, or use 0 for random port
/********************** Config: Keys & Secrets ********************************/
const API_KEY_2CAPTCHA = process.env.API_KEY_2CAPTCHA || 'YOUR_API_KEY_HERE'
const FLARESOLVERR_API_ENDPOINT = process.env.FLARESOLVERR_API_ENDPOINT || "http://localhost:8191/v1"
const ACTIVE_PERSONA = process.env.ACTIVE_PERSONA || 'Default'
const CHROME_PROFILE_USER = process.env.CHROME_PROFILE_USER || 'Default'
const LOAD_AUTH_STORAGE = isTruthy(process.env.LOAD_AUTH_STORAGE)
const SAVE_AUTH_STORAGE = isTruthy(process.env.SAVE_AUTH_STORAGE)
/********************** Config: Data Dir Locations ****************************/
const SRC_DIR = path.resolve(__dirname)
const DATA_DIR = process.env.DATA_DIR || await fs.promises.realpath(path.join(SRC_DIR, 'data'))
const INDEXES_DIR = path.join(DATA_DIR, 'index')
const ARCHIVE_DIR = path.join(DATA_DIR, 'archive')
if (!fs.existsSync(ARCHIVE_DIR))
throw 'Could not find data/archive, are you running in the right pwd?'
const PERSONA_DIR = path.join(DATA_DIR, 'personas', ACTIVE_PERSONA)
const CHROME_PROFILE_PATH = path.join(PERSONA_DIR, 'chrome_profile')
const CHROME_DOWNLOADS_DIR = path.join(PERSONA_DIR, 'chrome_downloads')
const CHROME_EXTENSIONS_DIR = path.join(PERSONA_DIR, 'chrome_extensions')
const CHROME_EXTENSIONS_JSON_PATH = path.join(CHROME_EXTENSIONS_DIR, 'extensions.json')
const AUTH_JSON_PATH = path.join(PERSONA_DIR, 'auth.json')
const COOKIES_TXT_PATH = path.join(PERSONA_DIR, 'cookies.txt')
const SPEEDTESTS_DIR = path.join(PERSONA_DIR, 'speedtests')
// const CHROME_PROFILE_IMPORT_USER = 'Profile 1'
// const CHROME_PROFILE_IMPORT_PATH = '/Volumes/NVME/Users/squash/Library/Application Support/Google/Chrome'
// chrome profile / persona directories
fs.mkdirSync(PERSONA_DIR, {recursive: true})
fs.mkdirSync(SPEEDTESTS_DIR, {recursive: true})
fs.mkdirSync(CHROME_PROFILE_PATH, {recursive: true})
fs.mkdirSync(CHROME_EXTENSIONS_DIR, {recursive: true})
fs.mkdirSync(CHROME_DOWNLOADS_DIR, {recursive: true})
// cruft directories
const ORPHANS_DIR = path.join(DATA_DIR, 'orphans')
const PARTIALS_DIR = path.join(DATA_DIR, 'partials')
const DUPLICATES_DIR = path.join(DATA_DIR, 'duplicates')
await fs.promises.mkdir(ORPHANS_DIR, {recursive: true})
await fs.promises.mkdir(PARTIALS_DIR, {recursive: true})
await fs.promises.mkdir(DUPLICATES_DIR, {recursive: true})
/********************** Config: Viewport Setup Opts ***************************/
// Config: Viewport
const DEFAULT_TIMEOUT = 20_000
const DEFAULT_GEOLOCATION = {latitude: 59.95, longitude: 30.31667}
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
const DEFAULT_ASPECT_RAIO = 16/9 // recommended: 16:9 (most common desktop window aspect ratio)
const SCREENSHOT_ASPECT_RATIO = 4/3 // recommended: 4:3 (easier to use as thumbnails when square-ish)
const DEFAULT_WINDOW_WIDTH = 1920 // recommended: 1920x1080p (1080p screenshots)
const DEFAULT_WINDOW_HEIGHT = Math.floor(DEFAULT_WINDOW_WIDTH/DEFAULT_ASPECT_RAIO)
const DEFAULT_VIEWPORT = {
width: DEFAULT_WINDOW_WIDTH,
height: DEFAULT_WINDOW_HEIGHT,
deviceScaleFactor: 2, // 2 gives much sharper text in screenshots/pdfs/etc but uses more CPU/GPU
isMobile: false,
hasTouch: false,
isLandscape: false,
}
const DEFAULT_COLOR_SCHEME = 'light'
const DEFAULT_HEADERS = {
// requires frequent tweaking to remain undetected by cloudflare/recaptcha/etc.
// 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
// 'accept-encoding': 'gzip, deflate, br, zstd',
// 'accept-language': accept_language,
// 'cache-Control': no_cache ? 'no-cache' : '',
// 'dnt': '1',
'sec-ch-ua': '"Google Chrome";v="122", "Not:A-Brand";v="8", "Chromium";v="122"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'connection-rtt': '50',
// 'pragma': no_cache ? 'no-cache' : '',
// 'sec-fetch-dest': 'document',
// 'sec-fetch-mode': 'navigate',
// 'sec-fetch-site': 'none',
// 'sec-fetch-user': '?1',
// // 'upgrade-insecure-requests': '1', // breaks some sites, e.g. https://www.flowradar.com/cloneables/mouse-image-trail-effect
// 'user-agent': user_agent,
}
const DEFAULT_REFERRERS = ["https://www.google.com", "https://www.facebook.com", "https://www.instagram.com"]
/****************** Config: Human Behavior Emulation **************************/
const SCROLL_LIMIT = 20; // e.g. 30 = 30 * (1000px/2s) => 30,000px scrolled in 60sec
const SCROLL_DELAY = 1350; // interval per scroll, e.g. 2000 = 2sec to travel 1 * SCROLL_DISTANCE
const SCROLL_DISTANCE = DEFAULT_VIEWPORT.height - 100; // make sure this is slightly less than viewport height so there is some overlap to make stitching easier
/********************** Config: URL Rewriting *********************************/
const URL_REWRITES = [
// replacements should come first
// {
// idx: 0,
// pattern: /\/\/(www\.)?x\.com/gi,
// replacement: '//$1twitter.com/',
// // TODO: scope: 'hostname',
// },
// {
// idx: 1,
// pattern: /\/\/(www\.)?twitter\.com/gi,
// replacement: '//$1nitter.net',
// // TODO: scope: 'hostname',
// },
// // blocks should come at the end
// {
// idx: 999,
// pattern: /\/\/(www\.)?notallowed\.com/gi,
// replacement: '',
// // TODO: scope: 'href',
// },
]
const URL_SCHEMES_IGNORED = [
'', // no scheme is also invalid (e.g. opening a new tab page without any url yet)
'chrome',
'chrome-extension',
'chrome-untrusted',
'file',
'data',
'about',
]
/**************** Load existing data/archive/<timestamp> snapshots *************/
const snapshots = await Snapshot.findAll({ attributes: ['id', 'timestamp', 'url'] }) // include: { model: ArchiveResult, as: 'archiveresults' }, });
const results = await ArchiveResult.findAll({ attributes: ['id', 'snapshot_id', 'extractor', 'start_ts'] }) // include: { model: Snapshot, as: 'snapshot' }, });
globalThis.snapshots = snapshots
globalThis.results = results
console.log(`[💿] Found ${snapshots.length} existing snapshots in index.sqlite3...`)
console.log(`[💿] Found ${results.length} existing results in index.sqlite3...`)
// debugger;
const locateExistingSnapshots = (archive_dir) => {
const urls_to_dirs = {}
// for each data/archive/<timestamp>/index.json found, store {url: data/archive/<timestamp>}
for (const snapshot_dir of fs.readdirSync(archive_dir)) {
const snapshot_json = path.join(archive_dir, snapshot_dir, 'index.json')
if (fs.existsSync(snapshot_json)) {
const {url, archive_path} = JSON.parse(fs.readFileSync(snapshot_json, 'utf-8'))
if (!snapshot_dir.includes(archive_path.replace('archive/', '')))
throw 'Found incorrect index.json inside snapshot dir' + snapshot_dir
if (url && url.includes('://')) {
urls_to_dirs[url] = path.join(archive_dir, snapshot_dir)
}
}
}
return urls_to_dirs
}
let SNAPSHOT_DIRS_BY_URL = locateExistingSnapshots(ARCHIVE_DIR)
let all_snap_dirs = (await fs.promises.readdir(ARCHIVE_DIR))
// const orphan_snap_dirs = all_snap_dirs.filter(dirname => dirname.startsWith('19999'))
// // scan through existing snapshot dirs, move orphans to orphans/ or correct archive/<snapid>
// for (const snap_id of orphan_snap_dirs) {
// if (snap_id.startsWith('.')) continue
// const src_dir = path.join(ARCHIVE_DIR, snap_id)
// let src_path = src_dir
// assert((await fs.promises.stat(src_dir)).isDirectory())
// let dest_path = null
// const orphan_metrics_path = path.join(src_dir, 'metrics.json')
// if (fs.existsSync(orphan_metrics_path)) {
// const orphan_metrics = JSON.parse(await fs.promises.readFile(orphan_metrics_path, 'utf-8'))
// const url = orphan_metrics.url || orphan_metrics.URL
// const version = orphan_metrics.VERSION || versionStrFromDate(orphan_metrics.start_time)
// // move all bare files into ./versions/YYYYMMDD/* and symlink ./* to latest version
// await symlinkBestSnapshotResults(src_dir)
// dest_path = SNAPSHOT_DIRS_BY_URL[url]
// const dest_id = dest_path?.split('/').at(-1)
// if (dest_id && (dest_id != snap_id)) {
// if (fs.existsSync(dest_path)) {
// console.log(` - moving duplicate snap_dir ${src_dir} -> ${dest_path}`)
// } else {
// console.log(` - moving valid snap_dir ${src_dir} -> ${dest_path}`)
// }
// } else if (dest_id == snap_id) {
// continue
// } else {
// dest_path = path.join(ORPHANS_DIR, snap_id)
// console.log(` - moving orphan snap_dir ${src_dir} -> ${dest_path}`)
// }
// } else {
// // corrupt/par
// dest_path = path.join(PARTIALS_DIR, snap_id)
// console.log(` - moving parial snap_dir ${src_dir} -> ${dest_path}`)
// }
// if (dest_path) {
// for (const version_dir of (await fs.promises.readdir(path.join(src_path, 'versions')))) {
// const version_src = path.join(src_path, 'versions', version_dir)
// const version_dst = path.join(dest_path, 'versions', version_dir)
// // move all bare files into ./versions/YYYYMMDD/* and symlink ./* to latest version
// await symlinkBestSnapshotResults(dest_path)
// assert(!fs.existsSync(version_dst))
// await fs.promises.rename(version_src, version_dst)
// console.log(' - ', version_src, '--->', version_dst)
// }
// await fs.promises.rename(src_dir, path.join(PARTIALS_DIR, snap_id))
// await symlinkBestSnapshotResults(dest_path)
// }
// }
// const duplicate_snap_dirs = (await fs.promises.readdir(DUPLICATES_DIR)).filter(dirname => dirname.startsWith('19999'))
// for (const snap_id of duplicate_snap_dirs) {
// const src_dir = path.join(DUPLICATES_DIR, snap_id)
// const metrics = JSON.parse(await fs.promises.readFile(path.join(src_dir, 'metrics.json'), 'utf-8'))
// }
// all_snap_dirs = (await fs.promises.readdir(ARCHIVE_DIR))
// for (const snap_id of all_snap_dirs) {
// if (snap_id.startsWith('.')) continue
// const snap_dir = path.join(ARCHIVE_DIR, snap_id)
// const metrics_path = path.join(snap_dir, 'metrics.json')
// if (fs.existsSync(metrics_path)) {
// // console.log(' - updating snap_dir', snap_dir)
// await symlinkBestSnapshotResults(snap_dir)
// }
// }
// SNAPSHOT_DIRS_BY_URL = locateExistingSnapshots(ARCHIVE_DIR)
fs.writeFileSync(path.join(DATA_DIR, 'queue.csv'), '')
const snapIdFromDir = (dir_path) =>
dir_path.split('/archive/').at(-1)
const snapshot_dir_list = (
Object.entries(SNAPSHOT_DIRS_BY_URL)
.sort(([_ak, a], [_bk, b]) =>
Number(snapIdFromDir(b)) - Number(snapIdFromDir(a)))
.reverse())
for (const [existing_url, snapshot_dir] of snapshot_dir_list) {
// if (existing_url.startsWith('https://www.facebook.com/')) {
const is_desired_url = !(existing_url.includes('facebook.com/') || existing_url.includes('instagram.com/'))
const already_archived = false // fs.existsSync(path.join(SNAPSHOT_DIRS_BY_URL[existing_url], 'versions'))
if (is_desired_url && !already_archived) {
// URLS.push(existing_url)
fs.appendFileSync(
path.join(DATA_DIR, 'queue.csv'),
`${SNAPSHOT_DIRS_BY_URL[existing_url]},${existing_url}\n`,
'utf-8',
)
}
}
URLS = [...new Set(URLS)]
console.log('[+] Added', URLS.length, 'existing urls to queue...')
/********************** Config: Output Paths **********************************/
// const TASK_PATH = (url) => path.join(DATA_DIR, 'results', `${hashCode(url)}`)
const TASK_PATH = (url) => SNAPSHOT_DIRS_BY_URL[url] || path.join(ARCHIVE_DIR, `1999999999.${hashCode(url)}`)
// const TASK_PATH = (url) => {
// const existing_snap_dir = SNAPSHOT_DIRS_BY_URL[url]
// assert(existing_snap_dir, `Could not find existing snapshot dir for ${url}`)
// return existing_snap_dir
// }
const OUTPUT_PATH = (page, filename, extname='') =>
path.join(TASK_PATH(page._original_url), `${filename}${extname}`)
const SSL_PATH = (page) => OUTPUT_PATH(page, 'ssl.json')
const CONSOLELOG_PATH = (page) => OUTPUT_PATH(page, 'console.log')
const HEADERS_PATH = (page) => OUTPUT_PATH(page, 'headers.json')
const REDIRECTS_PATH = (page) => OUTPUT_PATH(page, 'redirects.json')
const REQUESTS_PATH = (page) => OUTPUT_PATH(page, 'requests.json')
const TRACE_PATH = (page) => OUTPUT_PATH(page, 'trace.json')
const METRICS_PATH = (page) => OUTPUT_PATH(page, 'metrics.json')
const OUTLINKS_PATH = (page) => OUTPUT_PATH(page, 'outlinks.json')
const SEO_PATH = (page) => OUTPUT_PATH(page, 'seo.json')
const FAVICON_PATH = (page) => OUTPUT_PATH(page, 'favicon.json')
const TITLE_PATH = (page) => OUTPUT_PATH(page, 'title.txt')
const BODYTEXT_PATH = (page) => OUTPUT_PATH(page, 'body.txt')
const PANDOC_PATH = (page) => OUTPUT_PATH(page, 'pandoc.md')
const READABILITY_PATH = (page) => OUTPUT_PATH(page, 'readability.json')
const ACCESIBILITY_PATH = (page) => OUTPUT_PATH(page, 'accessibility.json')
const DOM_PATH = (page) => OUTPUT_PATH(page, 'dom.html')
const PDF_PATH = (page) => OUTPUT_PATH(page, 'output.pdf')
const SCREENSHOT_PATH = (page) => OUTPUT_PATH(page, 'screenshot.png')
const SCREENSHOT_JPG_PATH = (page) => OUTPUT_PATH(page, 'screenshot.jpg')
const AIQA_PATH = (page) => OUTPUT_PATH(page, 'aiqa.json')
const SINGLEFILE_PATH = (page) => OUTPUT_PATH(page, 'singlefile.html')
const YTDLP_PATH = (page) => OUTPUT_PATH(page, 'media/')
const GALLERYDL_PATH = (page) => OUTPUT_PATH(page, 'photos/')
const SCREENRECORDING_PATH = (page) => OUTPUT_PATH(page, 'screenrecording.mp4')
const SCREENRECORDGIF_PATH = (page) => OUTPUT_PATH(page, 'screenrecording.gif')
const RESPONSES_PATH = (page) => OUTPUT_PATH(page, 'responses')
const RAW_PATH = (page) => OUTPUT_PATH(page, 'raw')
/********************** Config: Chrome Extensions *****************************/
interface ChromeExtension {
name: string
webstore_id: string
}
interface LoadedChromeExtension extends ChromeExtension {
id?: string
webstore_url?: string
crx_url?: string
crx_path?: string
unpacked_path?: string
read_manifest?: () => any
read_version?: () => string | null
}
const CHROME_EXTENSIONS: LoadedChromeExtension[] = [
// Content access / unblocking / blocking plugins
new jsonl cli interface
2025-12-30 16:12:53 -08:00
{webstore_id: 'ifibfemgeogfhoebkmokieepdoobkbpo', name: 'twocaptcha'}, // https://2captcha.com/blog/how-to-use-2captcha-solver-extension-in-puppeteer
wip major changes
2025-12-24 20:09:51 -08:00
{webstore_id: 'edibdbjcniadpccecjdfdjjppcpchdlm', name: 'istilldontcareaboutcookies'},
{webstore_id: 'cjpalhdlnbpafiamejdnhcphjbkeiagm', name: 'ublock'},
// {webstore_id: 'mlomiejdfkolichcflejclcbmpeaniij', name: 'ghostery'},
// {webstore_id: 'mnjggcdmjocbbbhaepdhchncahnbgone', name: 'sponsorblock'},
// {webstore_id: 'iplffkdpngmdjhlpjmppncnlhomiipha', name: 'unpaywall'},
// {webstore_id: 'gofocbepaccnkpphbgjpolififgcakhn', name: 'spaywallnews'},
// Archiving plugins
{webstore_id: 'mpiodijhokgodhhofbcjdecpffjipkle', name: 'singlefile'},
// {webstore_id: 'fpeoodllldobpkbkabpblcfaogecpndd', name: 'archivewebpage'},
// {webstore_id: 'niloccemoadcdkdjlinkgdfekeahmflj', name: 'pocket'},
// {webstore_id: 'kenncghfghgolcbmckhiljgaabnpcaaa', name: 'warcreate'},
// {webstore_id: 'jjndjgheafjngoipoacpjgeicjeomjli', name: 'puppeteerstream'},
// Utilities for humans setting up/viewing/debugging the archiving session
// {webstore_id: 'aeblfdkhhhdcdjpifhhbdiojplfjncoa', name: '1password'},
// {webstore_id: 'fngmhnnpilhplaeedifhccceomclgfbg', name: 'editthiscookie'},
// {webstore_id: 'cgfpgnepljlgenjclbekbjdlgcodfmjp', name: 'simpletabsorter'},
// Scripting/automation plugins
// {webstore_id: 'jinjaccalgkegednnccohejagnlnfdag', name: 'violentmonkey'},
// {webstore_id: 'infppggnoaenmfagbfknfkancpbljcca', name: 'automa'},
// {webstore_id: 'pfegffhjcgkneoemnlniggnhkfioidjg', name: 'screenscraper'},
]
/******************** Config: Chrome Profile Preferences **********************/
// https://niek.github.io/chrome-features/
const CHROME_DISABLED_COMPONENTS = [
'Translate',
'AcceptCHFrame',
'OptimizationHints',
'ProcessPerSiteUpToMainFrameThreshold',
'InterestFeedContentSuggestions',
'CalculateNativeWinOcclusion',
'BackForwardCache',
'HeavyAdPrivacyMitigations',
'LazyFrameLoading',
'ImprovedCookieControls',
'PrivacySandboxSettings4',
'AutofillServerCommunication',
'CertificateTransparencyComponentUpdater',
'DestroyProfileOnBrowserClose',
'CrashReporting',
'OverscrollHistoryNavigation',
'InfiniteSessionRestore',
//'LockProfileCookieDatabase', // disabling allows multiple chrome instances to concurrently modify profile, but might make chrome much slower https://github.com/yt-dlp/yt-dlp/issues/7271 https://issues.chromium.org/issues/40901624
]
const CHROME_PREFERENCES_EXTRA = {}
const CHROME_PREFERENCES_DEFAULT = {
// https://chromium.googlesource.com/chromium/src/+/32352ad08ee673a4d43e8593ce988b224f6482d3/chrome/common/pref_names.cc
homepage: 'about:blank', // doesn't work here, managed by Secure Preferences
homepage_is_newtabpage: false, // doesn't work here, managed by Secure Preferences
session: { // doesn't work here, managed by Secure Preferences
restore_on_startup: 4, // doesn't work here, managed by Secure Preferences
startup_urls: 'about:blank', // doesn't work here, managed by Secure Preferences
},
default_apps: 'noinstall',
browser: {
confirm_to_quit: false,
enable_spellchecking: false,
check_default_browser: false,
show_update_promotion_info_bar: false,
},
profile: {
// name: 'ArchiveBox Persona: Default', // doesnt work to change display name, not sure why
// using_default_name: false,
exited_cleanly: true,
default_content_setting_values: {
automatic_downloads: 1,
},
},
bookmark_bar: {show_on_all_tabs: false},
safebrowsing: {enabled: false},
search: {suggest_enabled: false},
download: {
prompt_for_download: false,
open_pdf_in_system_reader: true,
// default_directory: CHROME_DOWNLOADS_DIR || path.join(__dirname, 'downloads'),
},
select_file_dialogs: {allowed: false},
autofill: {save_data: false},
printing: {enabled: false},
message_center: {welcome_notification_dismissed_local: true},
extensions: {
ui: {
developer_mode: true,
dismissed_adt_promo: true,
},
// pinned_extensions: CHROME_EXTENSIONS?.map(({id}) => id) || [],
},
webkit: {
webprefs: {
javascript_enabled: true,
minimum_font_size: 9,
// default_font_size: 12,
// web_security_enabled: false,
// allow_displaying_insecure_content: true,
// allow_running_insecure_content: true,
java_enabled: true,
loads_images_automatically: true,
},
},
settings: {
multi_profile_never_show_intro: true,
multi_profile_warning_show_dismissed: true,
first_run_tutorial_shown: true,
},
plugins: {
always_open_pdf_externally: true,
},
}
const CHROME_PREFERENCES_PATH = path.join(CHROME_PROFILE_PATH, 'Default', 'Preferences')
const getChromePreferences = ({CHROME_PREFERENCES_DEFAULT, CHROME_PREFERENCES_EXTRA, CHROME_EXTENSIONS, CHROME_DOWNLOADS_DIR}) =>
merge.all([CHROME_PREFERENCES_DEFAULT, CHROME_PREFERENCES_EXTRA, {
extensions: {
pinned_extensions: CHROME_EXTENSIONS?.map(({id}) => id) || [],
},
download: {
default_directory: CHROME_DOWNLOADS_DIR || path.join(__dirname, 'downloads'),
},
}])
function applyChromePreferences(puppeteer, prefs_path, preferences) {
if (fs.existsSync(prefs_path)) {
const preferences_existing = JSON.parse(fs.readFileSync(prefs_path, 'utf-8'))
const preferences_merged = merge(preferences_existing, preferences)
// console.log(JSON.stringify(preferences_merged, null, 4))
fs.writeFileSync(prefs_path, JSON.stringify(preferences_merged))
} else {
// otherwise profile has not been created yet, use plugin instead (plugin only works on first creation)
puppeteer.use(PrefsPlugin({userPrefs: preferences}))
}
return puppeteer
}
/******************** Config: Chrome Launch Args ******************************/
const CHROME_ARGS_DEFAULT = [
// Headless behavior tuning, determinstic behavior settings
// '--headless=new',
'--test-type',
'--test-type=gpu', // https://github.com/puppeteer/puppeteer/issues/10516
'--deterministic-mode',
'--js-flags=--random-seed=1157259159', // make all JS random numbers deterministic by providing a seed
'--allow-pre-commit-input', // allow JS mutations before page rendering is complete
'--disable-blink-features=AutomationControlled', // hide the signatures that announce browser is being remote-controlled
'--enable-automation', // <- DONT USE THIS, it makes you easily detectable / blocked by cloudflare
// `--proxy-server=https://43.159.28.126:2334:u7ce652b7568805c4-zone-custom-region-us-session-szGWq3FRU-sessTime-60:u7ce652b7568805c4`, // send all network traffic through a proxy https://2captcha.com/proxy
// `--proxy-bypass-list=127.0.0.1`,
// Docker-specific options
// https://github.com/GoogleChrome/lighthouse-ci/tree/main/docs/recipes/docker-client#--no-sandbox-issues-explained
// '--no-sandbox', // rely on docker sandboxing in docker, otherwise we need cap_add: SYS_ADM to use host sandboxing
// '--disable-gpu-sandbox',
// '--disable-setuid-sandbox',
// '--disable-dev-shm-usage', // docker 75mb default shm size is not big enough, disabling just uses /tmp instead
// '--no-xshm',
// Profile data dir setup
// chrome://profile-internals
`--user-data-dir=${CHROME_PROFILE_PATH}`,
`--profile-directory=${CHROME_PROFILE_USER}`,
'--password-store=basic', // use mock keychain instead of OS-provided keychain (we manage auth.json instead)
'--use-mock-keychain',
'--disable-cookie-encryption', // we need to be able to write unencrypted cookies to save/load auth.json
// '--disable-sync', // don't try to use Google account sync features
// Extensions
// chrome://inspect/#extensions
// `--load-extension=${CHROME_EXTENSIONS.map(({unpacked_path}) => unpacked_path).join(',')}`, // not needed when using existing profile that already has extensions installed
`--allowlisted-extension-id=${CHROME_EXTENSIONS.map(({ webstore_id }) => webstore_id).join(',')}`,
'--allow-legacy-extension-manifests',
// Browser window and viewport setup
// chrome://version
// `--user-agent="${DEFAULT_USER_AGENT}"`,
// `--window-size=${DEFAULT_VIEWPORT.width},${DEFAULT_VIEWPORT.height}`,
'--window-position=0,0',
'--hide-scrollbars', // hide scrollbars because otherwise they show up in screenshots
'--install-autogenerated-theme=169,32,85', // red border makes it easier to see which chrome window is archivebox's
'--autoplay-policy=no-user-gesture-required', // auto-start videos so they trigger network requests + show up in outputs
'--disable-gesture-requirement-for-media-playback',
'--lang=en-US,en;q=0.9',
// DANGER: JS isolation security features (to allow easier tampering with pages during archiving)
// chrome://net-internals
// '--disable-web-security', // <- WARNING, breaks some sites that expect/enforce strict CORS headers (try webflow.com)
// '--disable-features=IsolateOrigins,site-per-process', // useful for injecting JS, but some very strict sites can panic / show error pages when isolation is disabled (e.g. webflow.com)
// '--allow-running-insecure-content', // Breaks CORS/CSRF/HSTS etc., useful sometimes but very easy to detect
// '--allow-file-access-from-files', // <- WARNING, dangerous, allows JS to read filesystem using file:// URLs
// // DANGER: Disable HTTPS verification
// '--ignore-certificate-errors',
// '--ignore-ssl-errors',
// '--ignore-certificate-errors-spki-list',
// '--allow-insecure-localhost',
// IO: stdin/stdout, debug port config
// chrome://inspect
'--log-level=2', // 1=DEBUG 2=WARNING 3=ERROR
'--enable-logging=stderr',
'--remote-debugging-address=0.0.0.0',
`--remote-debugging-port=${CHROME_DEBUG_PORT}`,
// GPU, canvas, text, and pdf rendering config
// chrome://gpu
'--enable-webgl', // enable web-gl graphics support
'--font-render-hinting=none', // make rendering more deterministic by ignoring OS font hints, may also need css override, try: * {text-rendering: geometricprecision !important; -webkit-font-smoothing: antialiased;}
'--force-color-profile=srgb', // make rendering more deterministic by using consitent color profile, if browser looks weird, try: generic-rgb
'--disable-partial-raster', // make rendering more deterministic (TODO: verify if still needed)
'--disable-skia-runtime-opts', // make rendering more deterministic by avoiding Skia hot path runtime optimizations
'--disable-2d-canvas-clip-aa', // make rendering more deterministic by disabling antialiasing on 2d canvas clips
// '--disable-gpu', // falls back to more consistent software renderer
// // '--use-gl=swiftshader', <- DO NOT USE, breaks M1 ARM64. it makes rendering more deterministic by using simpler CPU renderer instead of OS GPU renderer bug: https://groups.google.com/a/chromium.org/g/chromium-dev/c/8eR2GctzGuw
// // '--disable-software-rasterizer', <- DO NOT USE, harmless, used in tandem with --disable-gpu
// // '--run-all-compositor-stages-before-draw', <- DO NOT USE, makes headful chrome hang on startup (tested v121 Google Chrome.app on macOS)
// // '--disable-gl-drawing-for-tests', <- DO NOT USE, disables gl output (makes tests run faster if you dont care about canvas)
// // '--blink-settings=imagesEnabled=false', <- DO NOT USE, disables images entirely (only sometimes useful to speed up loading)
// Process management & performance tuning
// chrome://process-internals
'--disable-lazy-loading', // make rendering more deterministic by loading all content up-front instead of on-focus
'--disable-renderer-backgrounding', // dont throttle tab rendering based on focus/visibility
'--disable-background-networking', // dont throttle tab networking based on focus/visibility
'--disable-background-timer-throttling', // dont throttle tab timers based on focus/visibility
'--disable-backgrounding-occluded-windows', // dont throttle tab window based on focus/visibility
'--disable-ipc-flooding-protection', // dont throttle ipc traffic or accessing big request/response/buffer/etc. objects will fail
'--disable-extensions-http-throttling', // dont throttle http traffic based on runtime heuristics
'--disable-field-trial-config', // disable shared field trial state between browser processes
'--disable-back-forward-cache', // disable browsing navigation cache
// '--in-process-gpu', <- DONT USE THIS, makes headful startup time ~5-10s slower (tested v121 Google Chrome.app on macOS)
// '--disable-component-extensions-with-background-pages', // TODO: check this, disables chrome components that only run in background (could lower startup time)
// uncomment to disable hardware camera/mic/speaker access + present fake devices to websites
// (faster to disable, but disabling breaks recording browser audio in puppeteer-stream screenrecordings)
// '--use-fake-device-for-media-stream',
// '--use-fake-ui-for-media-stream',
// '--disable-features=GlobalMediaControls,MediaRouter,DialMediaRouteProvider',
// // Output format options (PDF, screenshot, etc.)
'--export-tagged-pdf', // include table on contents and tags in printed PDFs
'--generate-pdf-document-outline',
// Suppress first-run features, popups, hints, updates, etc.
// chrome://system
'--no-pings',
'--no-first-run',
'--no-default-browser-check',
'--disable-default-apps',
'--ash-no-nudges',
'--disable-infobars',
'--disable-search-engine-choice-screen',
'--disable-session-crashed-bubble',
'--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"',
'--hide-crash-restore-bubble',
'--suppress-message-center-popups',
'--disable-client-side-phishing-detection',
'--disable-domain-reliability',
'--disable-component-update',
'--disable-datasaver-prompt',
'--disable-hang-monitor',
'--disable-session-crashed-bubble',
'--disable-speech-synthesis-api',
'--disable-speech-api',
'--disable-print-preview',
'--safebrowsing-disable-auto-update',
'--deny-permission-prompts',
'--disable-external-intent-requests',
'--disable-notifications',
'--disable-desktop-notifications',
'--noerrdialogs',
'--disable-popup-blocking',
'--disable-prompt-on-repost',
'--silent-debugger-extension-api',
'--block-new-web-contents',
'--metrics-recording-only',
'--disable-breakpad',
// other feature flags
// chrome://flags chrome://components
`--disable-features=${CHROME_DISABLED_COMPONENTS.join(',')}`,
'--enable-features=NetworkService',
]
const CHROME_ARGS_EXTRA = []
const CHROME_LAUNCH_OPTIONS = {
CHROME_PROFILE_PATH,
CHROME_PROFILE_USER,
CHROME_EXTENSIONS,
CHROME_DEBUG_PORT,
CHROME_DISABLED_COMPONENTS,
DEFAULT_VIEWPORT,
CHROME_ARGS_DEFAULT,
CHROME_ARGS_EXTRA,
}
/* Chrome CLI Args Documentation
- https://github.com/GoogleChrome/chrome-launcher/blob/main/docs/chrome-flags-for-tools.md
- https://chromium.googlesource.com/chromium/chromium/+/master/content/public/common/content_switches.cc
- https://jtway.co/optimize-your-chrome-options-for-testing-to-get-x1-25-impact-4f19f071bf45
- https://peter.sh/experiments/chromium-command-line-switches/
- https://www.chromium.org/developers/how-tos/run-chromium-with-flags/
- https://github.com/manoj9788/Chrome-Driver-arguments/blob/master/README.md
*/
const getChromeArgs = ({CHROME_ARGS_DEFAULT, CHROME_ARGS_EXTRA,
CHROME_PROFILE_PATH, CHROME_PROFILE_USER,
CHROME_EXTENSIONS,
CHROME_DEBUG_PORT,
CHROME_DISABLED_COMPONENTS,
DEFAULT_VIEWPORT}=CHROME_LAUNCH_OPTIONS) =>
[
...CHROME_ARGS_DEFAULT,
`--user-data-dir=${CHROME_PROFILE_PATH}`,
`--profile-directory=${CHROME_PROFILE_USER}`,
`--load-extension=${CHROME_EXTENSIONS.map(({unpacked_path}) => unpacked_path).join(',')}`,
`--allowlisted-extension-id=${CHROME_EXTENSIONS.map(({id}) => id).join(',')}`,
`--window-size=${DEFAULT_VIEWPORT.width},${DEFAULT_VIEWPORT.height}`,
`--remote-debugging-port=${CHROME_DEBUG_PORT}`,
`--disable-features=${CHROME_DISABLED_COMPONENTS.join(',')}`,
...CHROME_ARGS_EXTRA,
]
/******************** Chrome Extension Management *****************************/
function getExtensionId(unpacked_path) {
const manifest_path = path.join(unpacked_path, 'manifest.json')
if (!fs.existsSync(manifest_path)) return null
// chrome uses a SHA256 hash of the unpacked extension directory path to compute a dynamic id
const hash = crypto.createHash('sha256');
hash.update(Buffer.from(unpacked_path, 'utf-8'));
const detected_extension_id = Array.from(hash.digest('hex'))
.slice(0, 32) // Convert each hexadecimal character to a character in the range 'a'-'p'
.map(i => String.fromCharCode(parseInt(i, 16) + 'a'.charCodeAt(0)))
.join('');
return detected_extension_id
}
async function installExtension(extension) {
const manifest_path = path.join(extension.unpacked_path, 'manifest.json')
// Download extensions using:
// curl -fsSL 'https://clients2.google.com/service/update2/crx?response=redirect&prodversion=1230&acceptformat=crx3&x=id%3D$EXTENSION_ID%26uc' > extensionname.crx
// unzip -d extensionname extensionname.zip
if (!fs.existsSync(manifest_path) && !fs.existsSync(extension.crx_path)) {
console.log("[🛠️] Downloading missing extension", extension.name, extension.webstore_id, '->', extension.crx_path);
// Download crx file from ext.crx_url -> ext.crx_path
const response = await fetch(extension.crx_url) as Response
const crx_file = fs.createWriteStream(extension.crx_path);
if (response.headers.get("content-length") && response.body) {
// @ts-ignore
const crx_stream = Readable.fromWeb(response.body)
await finished(crx_stream.pipe(crx_file))
} else {
console.warn('[⚠️] Failed to download extension', extension.name, extension.webstore_id)
}
}
var {stdout, stderr} = {stdout: '', stderr: ''}
// Unzip crx file from ext.crx_url -> ext.unpacked_path
await fs.promises.mkdir(extension.unpacked_path, {recursive: true})
try {
var {stdout, stderr} = await exec(`/usr/bin/unzip ${extension.crx_path} -d ${extension.unpacked_path}`)
} catch(err1) {
try {
await unzip(extension.crx_path, extension.unpacked_path)
} catch(err2) {
// console.error(`[❌] Failed to install ${extension.crx_path}: could not unzip crx`, err1, err2)
// return false
}
}
if (!fs.existsSync(manifest_path))
console.error(`[❌] Failed to install ${extension.crx_path}: could not find manifest.json in unpacked_path`, stdout, stderr)
return fs.existsSync(manifest_path)
}
async function loadOrInstallExtension(ext) {
if (!(ext.webstore_id || ext.unpacked_path))
throw 'Extension must have either {webstore_id} or {unpacked_path}'
// Set statically computable extension metadata
ext.webstore_id = ext.webstore_id || ext.id
ext.name = ext.name || ext.webstore_id
ext.webstore_url = ext.webstore_url || `https://chromewebstore.google.com/detail/${ext.webstore_id}`
ext.crx_url = ext.crx_url || `https://clients2.google.com/service/update2/crx?response=redirect&prodversion=1230&acceptformat=crx3&x=id%3D${ext.webstore_id}%26uc`
ext.crx_path = ext.crx_path || path.join(CHROME_EXTENSIONS_DIR, `${ext.webstore_id}__${ext.name}.crx`)
ext.unpacked_path = ext.unpacked_path || path.join(CHROME_EXTENSIONS_DIR, `${ext.webstore_id}__${ext.name}`)
const manifest_path = path.join(ext.unpacked_path, 'manifest.json')
ext.read_manifest = () => JSON.parse(fs.readFileSync(manifest_path, 'utf-8'))
ext.read_version = () => fs.existsSync(manifest_path) && ext.read_manifest()?.version || null
// if extension is not installed, download and unpack it
if (!ext.read_version()) {
await installExtension(ext)
}
// autodetect id from filesystem path (unpacked extensions dont have stable IDs)
ext.id = getExtensionId(ext.unpacked_path)
ext.version = ext.read_version()
if (!ext.version) {
console.warn('[❌] Unable to detect ID and version of installed extension', prettyPath(ext.unpacked_path))
} else {
console.log(`[➕] Installed extension ${ext.name} (${ext.version})...`.padEnd(82), prettyPath(ext.unpacked_path))
}
return ext
}
async function isTargetExtension(target) {
let target_type
let target_ctx
let target_url
try {
target_type = target.type()
target_ctx = (await target.worker()) || (await target.page()) || null
target_url = target.url() || target_ctx?.url() || null
} catch(err) {
if (String(err).includes('No target with given id found')) {
// because this runs on initial browser startup, we sometimes race with closing the initial
// new tab page. it will throw a harmless error if we try to check a target that's already closed,
// ignore it and return null since that page is definitely not an extension's bg page anyway
target_type = 'closed'
target_ctx = null
target_url = 'about:closed'
} else {
throw err
}
}
const target_is_bg = ['service_worker', 'background_page'].includes(target_type)
const target_is_extension = target_url?.startsWith('chrome-extension://')
const extension_id = (target_is_extension && target_url.split('://')[1].split('/')[0]) || null
const manifest_version = target_type === 'service_worker' ? '3' : '2'
return {
target_type,
target_ctx,
target_url,
target_is_bg,
target_is_extension,
extension_id,
manifest_version,
}
}
async function loadExtensionFromTarget(extensions, target) {
const {
target_is_bg,
target_is_extension,
target_type,
target_ctx,
target_url,
extension_id,
manifest_version,
} = await isTargetExtension(target)
if (!(target_is_bg && extension_id && target_ctx))
return null
const manifest = await target_ctx.evaluate(() =>
// @ts-ignore
chrome.runtime.getManifest())
const { name, version, homepage_url, options_page, options_ui } = manifest
if (!version || !extension_id)
return null
const options_url = await target_ctx.evaluate(
(options_page) => chrome.runtime.getURL(options_page),
options_page || options_ui?.page || 'options.html',
)
const commands = await target_ctx.evaluate(async () =>
(await new Promise((resolve, reject) => {
if (chrome.commands)
chrome.commands.getAll(resolve)
else
resolve({})
}))
)
// console.log(`[+] Found Manifest V${manifest_version} Extension:`, extension_id, name, target_url, Object.keys(commands).length)
let dispatchEval = async (...args) =>
await target_ctx.evaluate(...args)
let dispatchPopup = async () =>
await target_ctx.evaluate('chrome.action?.openPopup() || chrome.tabs.create({url: chrome.runtime.getURL("popup.html")})')
let dispatchAction
let dispatchMessage
let dispatchCommand
if (manifest_version === '3') {
dispatchAction = async (tab) => {
// https://developer.chrome.com/docs/extensions/reference/api/action#event-onClicked
return await target_ctx.evaluate(async (tab) => {
tab = tab || (await new Promise((resolve) =>
chrome.tabs.query({currentWindow: true, active: true}, ([tab]) => resolve(tab))))
// @ts-ignore
return await chrome.action.onClicked.dispatch(tab)
}, tab)
}
dispatchMessage = async (message, options) => {
// https://developer.chrome.com/docs/extensions/reference/api/runtime
return await target_ctx.evaluate(async (extension_id, message, options) => {
return await chrome.runtime.sendMessage(extension_id, message, options)
}, extension_id, message, options)
}
dispatchCommand = async (command, tab) => {
// https://developer.chrome.com/docs/extensions/reference/api/commands#event-onCommand
return await target_ctx.evaluate(async (command, tab) => {
// @ts-ignore
return await chrome.commands.onCommand.dispatch(command, tab)
}, command, tab)
}
} else if (manifest_version === '2') {
dispatchAction = async (tab) => {
// https://developer.chrome.com/docs/extensions/mv2/reference/browserAction#event-onClicked
return await target_ctx.evaluate(async (tab) => {
tab = tab || (await new Promise((resolve) =>
chrome.tabs.query({currentWindow: true, active: true}, ([tab]) => resolve(tab))))
// @ts-ignore
return await chrome.browserAction.onClicked.dispatch(tab)
}, tab)
}
dispatchMessage = async (message, options) => {
// https://developer.chrome.com/docs/extensions/mv2/reference/runtime#method-sendMessage
return await target_ctx.evaluate(async (extension_id, message, options) => {
return await new Promise((resolve) =>
chrome.runtime.sendMessage(extension_id, message, options, resolve)
)
}, extension_id, message, options)
}
dispatchCommand = async (command, tab) => {
// https://developer.chrome.com/docs/extensions/mv2/reference/commands#event-onCommand
return await target_ctx.evaluate(async (command, tab) => {
return await new Promise((resolve) =>
// @ts-ignore
chrome.commands.onCommand.dispatch(command, tab, resolve)
)
}, command, tab)
}
}
const existing_extension = extensions.filter(({id}) => id === extension_id)[0] || {}
const new_extension = {
...existing_extension,
id: extension_id,
webstore_name: name,
target,
target_ctx,
target_type,
target_url,
manifest_version,
manifest,
version,
homepage_url,
options_url,
dispatchEval, // run some JS in the extension's service worker context
dispatchPopup, // open the extension popup
dispatchAction, // trigger an extension menubar icon click
dispatchMessage, // send a chrome runtime message in the service worker context
dispatchCommand, // trigger an extension keyboard shortcut command
}
console.log(`[➕] Loaded extension ${name.substring(0, 32)} (${version}) ${target_type}...`.padEnd(82), target_url)
Object.assign(existing_extension, new_extension)
return new_extension
}
async function getChromeExtensionsFromPersona({CHROME_EXTENSIONS, CHROME_EXTENSIONS_DIR}) {
console.log('*************************************************************************')
console.log(`[⚙️] Installing ${CHROME_EXTENSIONS.length} chrome extensions from CHROME_EXTENSIONS...`)
try {
// read extension metadata from filesystem (installing from Chrome webstore if extension is missing)
for (const extension of CHROME_EXTENSIONS) {
Object.assign(extension, await loadOrInstallExtension(extension))
}
// for easier debugging, write parsed extension info to filesystem
await overwriteFile(
CHROME_EXTENSIONS_JSON_PATH.replace('.json', '.present.json'),
CHROME_EXTENSIONS,
)
} catch(err) {
console.error(err)
}
console.log('*************************************************************************')
return CHROME_EXTENSIONS
}
let _EXTENSIONS_CACHE = null
async function getChromeExtensionsFromCache({browser, extensions=CHROME_EXTENSIONS, extensions_dir=CHROME_EXTENSIONS_DIR}) {
if (_EXTENSIONS_CACHE === null) {
console.log(`[⚙️] Loading ${CHROME_EXTENSIONS.length} chrome extensions from CHROME_EXTENSIONS...`)
// find loaded Extensions at runtime / browser launch time & connect handlers
// looks at all the open targets for extension service workers / bg pages
for (const target of browser.targets()) {
// mutates extensions object in-place to add metadata loaded from filesystem persona dir
await loadExtensionFromTarget(extensions, target)
}
_EXTENSIONS_CACHE = extensions
// write installed extension metadata to filesystem extensions.json for easier debugging
await overwriteFile(
CHROME_EXTENSIONS_JSON_PATH.replace('.json', '.loaded.json'),
extensions,
)
await overwriteSymlink(
CHROME_EXTENSIONS_JSON_PATH.replace('.json', '.loaded.json'),
CHROME_EXTENSIONS_JSON_PATH,
)
}
return _EXTENSIONS_CACHE
}
async function setup2CaptchaExtension({browser, extensions}) {
let page = null
try {
// open a new tab to finish setting up the 2captcha extension manually using its extension options page
page = await browser.newPage()
new jsonl cli interface
2025-12-30 16:12:53 -08:00
const { options_url } = extensions.filter(ext => ext.name === 'twocaptcha')[0]
wip major changes
2025-12-24 20:09:51 -08:00
await page.goto(options_url)
await wait(2_500)
await page.bringToFront()
// type in the API key and click the Login button (and auto-close success modal after it pops up)
await page.evaluate(() => {
const elem = document.querySelector("input[name=apiKey]") as HTMLInputElement
elem.value = ""
})
await page.type('input[name=apiKey]', API_KEY_2CAPTCHA, { delay: 25 })
// toggle all the important switches to ON
await page.evaluate(() => {
const checkboxes = Array.from(document.querySelectorAll<HTMLInputElement>('input#isPluginEnabled, input[name*=enabledFor], input[name*=autoSolve]'));
for (const checkbox of checkboxes) {
if (!checkbox.checked) checkbox.click()
}
})
let dialog_opened = false
page.on('dialog', async (dialog) => {
setTimeout(async () => {
await dialog.accept();
dialog_opened = true
}, 500);
})
await page.click('button#connect')
await wait(2_500)
if (!dialog_opened) {
throw `2captcha extension login confirmation dialog never opened, please check its options page manually: ${options_url}`
}
console.log('[🔑] Configured the 2captcha extension using its options page...')
} catch(err) {
console.warn(`[❌] Failed to configure the 2captcha extension using its options page!`, err)
}
if (page) await page.close()
}
async function speedtest({browser, page, measureUpload=true, timeout=25000}: {browser?: Browser, page?: Page, measureUpload?: boolean, timeout?: number}) {
// run a speedtest using fast.com, printing results once per second
browser = browser || await page.browser()
page = page || await browser.newPage()
// save one speedtest_<date>.json result per day
const today = versionStrFromDate(new Date(), {withDate: true, withTime: false})
const SPEEDTEST_PATH = path.join(SPEEDTESTS_DIR, `speedtest_${today}.json`)
// check if we've already run one today, if so return earlier results and skip running again
try {
return JSON.parse(await fs.promises.readFile(SPEEDTEST_PATH, 'utf-8'))
} catch(err) {
// otherwise speedtest does not exist yet for today, continue onwards...
}
console.log('[🚤] Running Speedtest using Fast.com...'.padEnd(82), prettyPath(SPEEDTEST_PATH))
await page.goto('https://fast.com', {timeout, waitUntil: 'domcontentloaded'});
await page.waitForSelector('#speed-value', {timeout})
let result = null
let loop_idx = 0
while (loop_idx < 100) {
result = await page.evaluate(() => {
const $ = document.querySelector.bind(document);
return {
downloadSpeed: Number($('#speed-value').textContent),
downloadUnit: $('#speed-units').textContent.trim(),
downloaded: Number($('#down-mb-value').textContent.trim()),
uploadSpeed: Number($('#upload-value').textContent),
uploadUnit: $('#upload-units').textContent.trim(),
uploaded: Number($('#up-mb-value').textContent.trim()),
latency: Number($('#latency-value').textContent.trim()),
bufferBloat: Number($('#bufferbloat-value').textContent.trim()),
userLocation: $('#user-location').textContent.trim(),
userIp: $('#user-ip').textContent.trim(),
isDone: Boolean($('#speed-value.succeeded') && $('#upload-value.succeeded')),
};
})
if (result.downloadSpeed > 0) {
// console.log(JSON.stringify(result).replaceAll('"', '').replaceAll(',', ' ').replaceAll('{', '').replaceAll('}', ''))
}
if (result.isDone || (!measureUpload && result.uploadSpeed)) {
break
}
await wait(500)
loop_idx++
}
await Promise.allSettled([
page.close(),
overwriteFile(SPEEDTEST_PATH, result)
])
return result
}
/******************************************************************************/
/******************************************************************************/
const ALREADY_ARCHIVED = new Set(['', 'about:blank', 'chrome://newtab', 'chrome://version'])
const TASKS_PER_RUN_LIMIT = 200
async function botArchiveTask({page, data, url=''}) {
url = url || data // puppeteer-cluster passes in the url value via the data: arg
const is_unarchivable_url = URL_SCHEMES_IGNORED.includes(url.split(':')[0])
const is_already_archived = ALREADY_ARCHIVED.has(url.slice(0, 4096))
if (is_unarchivable_url || is_already_archived) return null
ALREADY_ARCHIVED.add(url.slice(0, 4096))
if (ALREADY_ARCHIVED.size > TASKS_PER_RUN_LIMIT) {
console.warn('[❌] Hit maximum URLs archived per browser session, exiting to free memory.')
console.warn(' Run this process again to continue with the next batch...')
process.exit(21)
}
const browser = await page.browser()
const client = await page.target().createCDPSession()
const extensions = await getChromeExtensionsFromCache({browser})
const browser_version = await browser.version()
const original_url = url.toString()
const start_time = (new Date())
console.log('[0/4]-------------------------------------------------------------------------')
const snapshot_dir = await setupSnapshotDir({original_url, start_time})
const snapshot = await setupSnapshotDB({original_url, start_time, snapshot_dir})
console.log('[1/4]-------------------------------------------------------------------------')
console.log(`[🪟] Starting page & viewport setup (${browser_version} ${DEFAULT_VIEWPORT.isMobile ? 'mobile' : 'desktop'} ${DEFAULT_VIEWPORT.width}x${DEFAULT_VIEWPORT.height}px)...`)
const page_state = {
// global static state
browser,
client,
browser_version,
extensions,
// per-page static metadata
original_url,
snapshot,
snapshot_dir,
start_time: start_time.toISOString(),
start_ts: Number(start_time),
version: versionStrFromDate(start_time),
// per-page mutable archiving state
main_response: null,
recorder: null,
console_log: [],
traffic_log: {},
redirects: {},
}
page._original_url = original_url
try {
// run all page setup functions in parallel
const results = await Promise.allSettled([
// loadAuthStorage(page, page_state, { apply: true }),
startMetadataRecording(page, page_state),
setupURLRewriting(page, page_state),
// setupViewport(page, page_state),
setupModalAutoClosing(page, page_state),
loadCloudflareCookie(page, page_state),
startResponseSaving(page, page_state),
saveYTDLP(page, page_state),
saveGALLERYDL(page, page_state),
// saveSourceMaps(page, page_state),
// TODO: someday setup https://github.com/osnr/TabFS ?
]);
// run all page setup functions in parallel
const rejected = results
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason);
if (rejected.length) console.warn('[⚠️] Partial failures during page setup:', rejected);
} catch(err) {
console.error('[❌] PAGE SETUP ERROR', JSON.stringify(err, null, 4))
return
}
console.log('[2/4]-------------------------------------------------------------------------')
console.log('[➡️] NAVIGATION[INI]', ANSI.blue + url + ANSI.reset)
const startrecording_promise = startScreenrecording(page, page_state)
page_state.main_response = await page.goto(url, {waitUntil: 'load', timeout: 40_000})
try {
const results = await Promise.allSettled([
startrecording_promise,
page.bringToFront(),
page.waitForNetworkIdle({concurrency: 0, idleTime: 900, timeout: 20_000}),
])
const rejected = results
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason)
if (rejected.length) console.warn('[⚠️] Parial failures during page load:', rejected)
} catch(err) {
console.error('[❌] ERROR DURING PAGE LOAD', JSON.stringify(err, null, 4))
return
}
if (page_state.main_response === null) {
page_state.main_response = await page.waitForResponse(() => true)
}
assert(page_state.main_response)
if (page_state.main_response.status() == 429) {
throw `[⚠️] Got 429 rate-limit response, skipping this URL for now...`
}
// emulate human browsing behavior
// await disableAnimations(page, page_state);
await jiggleMouse(page, page_state);
await solveCaptchas(page, page_state);
await blockRedirects(page, page_state);
await scrollDown(page, page_state);
// await expandComments(page, page_state);
await submitForm(page, page_state);
// await blockJSExecution(page, page_state);
console.log('[3/4]-------------------------------------------------------------------------')
// stop tampering with page requests & JS / recording metadata / traffic log
await stopMetadataRecording(page, page_state)
// do all synchonous archiving steps that need exclusive use of the whole page while doing stuff
const saveScreenrecording_promise = saveScreenrecording(page, page_state);
await saveScreenshot(page, page_state);
await savePDF(page, page_state);
console.log('[4/4]-------------------------------------------------------------------------')
// do all async archiving steps that can be run at the same time
await inlineShadowDOM(page, page_state);
const results = await Promise.allSettled([
saveTitle(page, page_state),
saveSEO(page, page_state),
saveFavicon(page, page_state),
saveSSL(page, page_state),
saveRequests(page, page_state),
saveRedirects(page, page_state),
saveHeaders(page, page_state),
saveRaw(page, page_state),
saveDOM(page, page_state),
saveBodyText(page, page_state),
// savePandoc(page, page_state),
saveReadability(page, page_state),
saveAccessibility(page, page_state),
saveOutlinks(page, page_state),
// saveAuthStorage(page, page_state),
saveAIQualityAssuranceResult(page, page_state),
]);
// do all sync archiving steps that require browser extensions at the very end (they are the buggiest)
const bg_results = Promise.allSettled([
saveScreenrecording_promise,
saveSinglefile(page, page_state),
// saveArchiveWebPage(page, page_state),
// savePocket(page, page_state),
])
const {duration} = await saveMetrics(page, page_state);
const rejected = results
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason) // not sure why this has a ts-error, .reason does exist on rejected promises
if (rejected.length)
console.warn('[⚠️] Parial failures during archiving:', rejected)
// Start an interactive REPL here with the `page` instance.
// https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-repl
// await page.repl()
// await page.browser().repl()
console.log(`[✅] ${ANSI.blue}Finished archiving in ${duration/1000}s.${ANSI.reset}`)
try {
const rejected = (await bg_results)
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason) // not sure why this has a ts-error, .reason does exist on rejected promises
if (rejected.length)
console.warn('[⚠️] Parial failures during wrap-up tasks:', rejected)
console.log('[🗑️] Resetting to about:blank to ensure memory is freed...')
await page.goto('about:blank')
await page.close()
} catch(err) {
console.log(err)
}
// symlink the best results from across all the versions/ into the snapshot dir root
await symlinkBestSnapshotResults(snapshot_dir)
// display latest version screenshot GIF
console.log()
try {
const latest_version_gif = path.join(snapshot_dir, 'versions', page_state.version, path.basename(SCREENRECORDGIF_PATH(page)))
const dirent = await blockUntilExists(latest_version_gif, {min_bytes: 100, timeout: 15_000})
child_process.spawn('/Users/squash/.iterm2/imgcat', [dirent.abspath], {stdio: [null, 'inherit', 'inherit']})
} catch(err) {
console.warn('[⚠️] Failed to display screenrecording.gif...', err)
console.log()
}
// determine whether task succeeded or failed based on AI QA score
const latest_version_aiqa = path.join(snapshot_dir, 'versions', page_state.version, path.basename(AIQA_PATH(page)))
const qa_results = JSON.parse((await fs.promises.readFile(latest_version_aiqa)).toString())
if (qa_results.pct_visible < 50) {
throw `[❌] Task completed with problems, got AI QA score of ${qa_results.pct_visible}%! ${qa_results.warnings.join(', ')} ${qa_results.error_text || ''}`
} else {
console.log(`[💫] Task completed succesfully: ${qa_results.pct_visible}% ${qa_results.warnings.join(', ') || ''}`)
console.log(` Summary: ${(qa_results.main_content_title || qa_results.description || 'No title/description detected').substring(0, 80)}... ${qa_results.main_content_author || ''} ${qa_results.main_content_date || ''}`)
return true
}
}
async function passiveArchiveTask({browser, page, url}) {
// archive passively (e.g. a tab that was opened already by a human), without changing the active page
const is_unarchivable_url = URL_SCHEMES_IGNORED.includes(url.split(':')[0])
const is_already_archived = ALREADY_ARCHIVED.has(url.slice(0, 4096))
if (is_unarchivable_url || is_already_archived) return null
ALREADY_ARCHIVED.add(url.slice(0, 4096))
// these have to be as early as possible because we're racing with the page load (we might even be too late)
// jk nevermind, we now re-open a new bg tab for every tab that's created to re-capture the initial request
// await page.setRequestInterception(true);
// await page.setCacheEnabled(false);
const original_url = url.toString()
const start_time = (new Date())
const browser_version = await browser.version()
console.log('------------------------------------------------------------------------------')
console.log('[➕] Starting archive of new tab opened in driver browser...', await browser.version())
const snapshot_dir = await setupSnapshotDir({original_url, start_time})
const snapshot = await setupSnapshotDB({ original_url, start_time, snapshot_dir })
console.log('------------------------------------------------------------------------------')
console.log(`[🪟] Starting page & viewport setup (${browser_version} ${DEFAULT_VIEWPORT.isMobile ? 'mobile' : 'desktop'} ${DEFAULT_VIEWPORT.width}x${DEFAULT_VIEWPORT.height}px)...`)
// create a new page in the background for archiving
const old_page = page
page = await browser.newPage()
await old_page.bringToFront()
const client = await page.target().createCDPSession()
const extensions = await getChromeExtensionsFromCache({ browser })
const page_state = {
// global static state
browser,
client,
browser_version,
extensions,
// per-page static metadata
original_url,
snapshot,
snapshot_dir,
start_time: start_time.toISOString(),
start_ts: Number(start_time),
version: versionStrFromDate(start_time),
// per-page mutable archiving state
main_response: null,
recorder: null,
console_log: [],
traffic_log: {},
redirects: {},
}
page._original_url = original_url
try {
// run all page setup functions in parallel
const results = await Promise.allSettled([
// loadAuthStorage(page, page_state, {apply: true}),
startMetadataRecording(page, page_state),
setupURLRewriting(page, page_state),
startResponseSaving(page, page_state),
saveYTDLP(page, page_state),
saveGALLERYDL(page, page_state),
// saveSourceMaps(page, page_state),
]);
const rejected = results
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason)
if (rejected.length) console.warn('[⚠️] Parial failures during page setup:', rejected)
} catch(err) {
console.warn('[❌] ERROR DURING PAGE SETUP', JSON.stringify(err, null, 4))
return
}
// load the url in the background page, then switch to it once its loaded and close the original tab
console.log('[➡️] NAVIGATION[INI]', ANSI.blue + url + ANSI.reset)
const startrecording_promise = startScreenrecording(page, page_state)
page_state.main_response = await page.goto(url, {waitUntil: 'load', timeout: 40_000})
// for debugging
globalThis.page = page
globalThis.page_state = page_state
// start loading the page, start screenrecording, close the old page, and wait for loading to finish (all at once, fine for these to race)
try {
const results = await Promise.allSettled([
startrecording_promise,
page.bringToFront(),
old_page.close(),
page.waitForNetworkIdle({concurrency: 0, idleTime: 900, timeout: 20_000}),
])
const rejected = results
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason)
if (rejected.length) console.warn('[⚠️] Parial failures during [age load:', rejected)
} catch(err) {
console.warn('[❌] ERROR DURING PAGE LOAD', JSON.stringify(err, null, 4))
return
}
if (page_state.main_response === null) {
page_state.main_response = await page.waitForResponse(() => true)
}
assert(page_state.main_response)
if (page_state.main_response.status() == 429) {
throw `[⚠️] Got 429 rate-limit response, skipping this URL for now...`
}
// resume page if paused by waitForDebuggerOnStart/dev tools debugger/backgrounding
try {
await client.send('Page.enable');
await client.send('Page.setWebLifecycleState', {state: 'active'});
await client.send('Runtime.runIfWaitingForDebugger')
} catch(err) { /* console.warn(err) */ }
// wait a couple seconds for page to finish loading
await wait(5_000)
// emulate human browsing behavior
// await disableAnimations(page, page_state);
// await jiggleMouse(page, page_state);
await solveCaptchas(page, page_state);
// await blockRedirects(page, page_state);
// await scrollDown(page, page_state);
// await expandComments(page, page_state);
await submitForm(page, page_state);
// await blockJSExecution(page, page_state);
await stopMetadataRecording(page, page_state) // stop tampering with page requests & JS
console.log('[3/4]-------------------------------------------------------------------------')
// do all synchonous archiving steps that need exclusive use of the whole page while doing stuff
const saveScreenrecording_promise = saveScreenrecording(page, page_state);
await saveScreenshot(page, page_state);
await savePDF(page, page_state);
console.log('[4/4]-------------------------------------------------------------------------')
// do all async archiving steps that can be run at the same time
await inlineShadowDOM(page, page_state);
const results = await Promise.allSettled([
saveTitle(page, page_state),
saveSEO(page, page_state),
saveFavicon(page, page_state),
saveSSL(page, page_state),
saveRequests(page, page_state),
saveRedirects(page, page_state),
saveHeaders(page, page_state),
saveRaw(page, page_state),
saveDOM(page, page_state),
saveBodyText(page, page_state),
// savePandoc(page, page_state),
saveReadability(page, page_state),
saveAccessibility(page, page_state),
saveOutlinks(page, page_state),
// saveAuthStorage(page, page_state),
saveAIQualityAssuranceResult(page, page_state),
]);
// do all sync archiving steps that require browser extensions at the very end (they are the buggiest)
const bg_results = Promise.allSettled([
saveScreenrecording_promise,
saveSinglefile(page, page_state),
// saveArchiveWebPage(page, page_state),
// savePocket(page, page_state),
])
const {duration} = await saveMetrics(page, page_state);
const rejected = results
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason)
if (rejected.length)
console.warn('[⚠️] Parial failures during page archiving:', rejected)
// Start an interactive REPL here with the `page` instance.
// https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-repl
// await page.repl()
// await page.browser().repl()
console.log(`[✅] Finished archiving in ${duration/1000}s.`,)
// await page.tracing.stop();
try {
const rejected = (await bg_results)
.filter(result => result.status === 'rejected')
.map(result => (result as PromiseRejectedResult).reason)
if (rejected.length)
console.warn('[⚠️] Parial failures during page wrap-up tasks:', rejected)
} catch(err) {
console.log(err)
}
await symlinkBestSnapshotResults(snapshot_dir)
}
/******************************************************************************/
/************************* Page Setup Tasks ***********************************/
async function setupSnapshotDir({original_url, start_time, snapshot_dir=null}) {
// setup archive/<id> snapshot output folder, move old files into versions/<date>/* + clear any existing symlinks
const snap_dir = snapshot_dir || TASK_PATH(original_url)
console.log()
console.log()
console.log(ANSI.blue + original_url + ANSI.reset)
console.log(ANSI.black + snap_dir + ANSI.reset)
console.log()
console.log('[📂] Setting up Snapshot output directory...'.padEnd(82), prettyPath(snap_dir))
// check for existing data at old legacy paths e.g. ./data/archive/1999999999.1723425
const hacky_dir = path.join(ARCHIVE_DIR, `1999999999.${hashCode(original_url)}`)
const known_dir = SNAPSHOT_DIRS_BY_URL[original_url]
const known_dir_exists = fs.existsSync(known_dir)
const hacky_dir_exists = fs.existsSync(hacky_dir)
if (snap_dir == hacky_dir) {
if (known_dir_exists) {
throw `Tried to create snapshot in ${snap_dir} but potential duplicate exists in ${known_dir}!`
}
} else if (snap_dir == known_dir) {
if (hacky_dir_exists) {
throw `Tried to create snapshot in ${snap_dir} but potential duplicate exists in ${hacky_dir}!`
}
} else {
if (known_dir_exists) {
throw `Tried to create snapshot in ${snap_dir} but potential duplicate exists in ${known_dir}!`
} else if (hacky_dir_exists) {
throw `Tried to create snapshot in ${snap_dir} but potential duplicate exists in ${hacky_dir}!`
} else {
throw `Tried to create snapshot in ${snap_dir} but its not a recognized snapshot dir path:\n - ${known_dir}\n - ${hacky_dir}`
}
}
// mkdir -p ./data/archive/<snap_id>/versions && cd ./data/archive/<snap_id>
await fs.promises.mkdir(path.join(snap_dir, 'versions'), {recursive: true})
process.chdir(snap_dir)
// clear any /data/archive/<snap_id>/*.* symlinks pointing to existing ./versions/<versionid>/*.* files
await clearSnapshotDirSymlinks(snap_dir)
// move /data/archive/<snap_id>/*.* loose output files from any prior run into ./versions/<versionid>/*.*
await collectSnapshotDirVersionFiles(snap_dir)
// update /data/indexes/<index_name>/* to include references to /data/archive/<snap_id> as-needed
await updateSnapshotDirIndexes(snap_dir, {original_url, start_time})
// assert /data/archive/<snap_id>/ contains no invalid/partial files + is empty/ready to receive new files
await assertSnapshotDirIsValid(snap_dir, {is_empty: true})
return snap_dir
}
// ./index/<index_name> : index_getter(page_state) => "<index_key_str>"
const INDEXES = {
snapshots_by_day: ({start_time}) =>
versionStrFromDate(start_time, {withDate: true, withTime: false}),
snapshots_by_domain: ({original_url}) =>
(new URL(original_url)).hostname || '', // hostname does not include :port
}
async function updateSnapshotDirIndexes(snap_dir, page_state, indexes=INDEXES, indexes_dir=INDEXES_DIR) {
assert(indexes)
console.log(`[🔎] Linking Snapshot in indexes (${Object.keys(indexes).join(', ')})...`)
// const {snapshot_dir, original_url, start_ts} = page_state
for (const [index_name, index_key_getter] of Object.entries(indexes)) {
const index_entry = await indexSnapshotDir(snap_dir, {index_name, index_key_getter, indexes_dir}, page_state)
}
}
async function indexSnapshotDir(snap_dir, {index_name, index_key_getter, indexes_dir=INDEXES_DIR}, page_state) {
// place symlinks to this snapshot in any /indexes/<index_name/ indexes as-needed
// const snap_id = snap_dir.split('/').at(-1)
const index_dir = path.join(indexes_dir, index_name) // /data/index/snapshots_by_day
await fs.promises.mkdir(index_dir, {recursive: true})
// calculate the index key, e.g. "200101231" or "example.com"
assert(index_name && index_key_getter)
assert(page_state)
const index_key = String(index_key_getter(page_state)) // '20010131'
assert(index_key)
const snap_id = path.parse(snap_dir).base // '19999999.23423523'
assert(snap_id)
const index_entries_dir = path.join(index_dir, index_key) // /data/index/snapshots_by_day/20010131
await fs.promises.mkdir(index_entries_dir, {recursive: true})
const symlink_path = path.join(index_entries_dir, snap_id) // /data/index/snapshots_by_day/20010131/19999999.23423523
// create symlink index/snapshots_by_day/<YYYYMMDD>/<snap id> -> ./archive/<snap_id> symlink
const {symlink_abspath} = await overwriteSymlink(snap_dir, symlink_path, {relative: true, mkdirs: false})
}
async function collectSnapshotDirVersionFiles(snap_dir) {
// move archive/<id>/*.* snapshot output files into archive/<id>/versions/<date>/* dated version folder
// detect start time / version info from previous result metrics.json
const snap_id = snap_dir.split('/archive/').at(-1)
const existing_metrics = path.join(snap_dir, 'metrics.json')
let {start_time, VERSION} = {start_time: '1970-01-01T00:00:00.000Z', VERSION: '19700101000000'}
try {
;({start_time, VERSION} = JSON.parse(await fs.promises.readFile(existing_metrics, 'utf-8')));
} catch(err) {
// continue normally, overwriting existing files is fine if they're broken to begin with
}
// create new version folder based on metrics.json start_time (or epoch time as fallback for legacy output)
const version_dir_name = VERSION || versionStrFromDate(start_time)
const version_dir = path.join(snap_dir, 'versions', version_dir_name)
await fs.promises.mkdir(version_dir, {recursive: true})
// move all result files from snapshot_dir root into version folder
const existing_snapshot_files =
(await fs.promises.readdir(snap_dir, {withFileTypes: true}))
.filter(dirent => {
if (dirent.name.startsWith('.')) return false // ignore hidden files, dont version them
if (dirent.name == 'versions') return false // dont try to move versions folder into itself
if (dirent.isSymbolicLink()) return false // skip existing symbolic links
return (dirent.isFile() || dirent.isDirectory()) // dont try to version sockets/FIFOs/devs etc.
})
if (existing_snapshot_files.length) {
console.log(`[📅] Moving snapshot results into version dir: ./data/archive/${snap_id}/* ->`.padEnd(82), `./data/archive/${snap_id}/versions/${VERSION}/`)
}
const snapshot_files = await getDirInfo(snap_dir, {withRoot: false, filter: ({relpath}) => !relpath.startsWith('versions')})
const version_files = await getDirInfo(version_dir, {withRoot: false})
for (const {name} of existing_snapshot_files) {
const snapdir_entry_abspath = path.join(snap_dir, name)
const versioned_entry_abspath = path.join(version_dir, name)
const snapshot_entry = snapshot_files[name]
const version_entry = version_files[name]
if (snapshot_entry && version_entry) {
// a conflicting file/dir already exists in the destination path
// we have a few options here, we can try to merge them, or we can create a new version
if (snapshot_entry.sha256 == version_entry.sha256) {
// both are the same already, delete the duplicate (leaving the copy inside the version dir)
// if (snapshot_entry.is_dir) {
// await fs.promises.rmdir(snapshot_entry.abspath, {recursive: true})
// } else {
// await fs.promises.unlink(snapshot_entry.abspath)
// }
// console.warn(`[!] Found harmless exact duplicate files, leaving as is: ${snapshot_entry.summary} and ${version_entry.summary}`)
} else {
// both are different,
if (snapshot_entry.num_bytes > version_entry.num_bytes) {
// snapshot entry is bigger, keep it and delete version entry?
} else {
// version entry is bigger, keep it and delete snapshot entry
}
console.warn(' ', snapshot_entry.summary)
console.warn(' ', version_entry.summary)
// throw `Found conflicting duplicate files with different contents: ${name}`
}
} else {
// mv ./data/archive/<snap_id>/example.txt -> ./data/archive/<snap_id>/versions/<version_id>/example.txt
await fs.promises.rename(snapdir_entry_abspath, versioned_entry_abspath)
console.log(` ↣ ${prettyPath(snapdir_entry_abspath)} ->`.padEnd(82), prettyPath(versioned_entry_abspath))
}
}
}
// Extractor definition
// {
// phase: setup | load | sync1 | async1 | sync2 | close
// name: 'media' | 'photos', 'wget', 'singlefile'
//
// shouldRun(page, page_state)
// pageSetup
// pageLoad
// pageInteraction clicking around/scrolling
// archivePhase1 sync
// archivePhase2 async
// archivePhase3 async
// pageClose
// execute(page, page_state)
// validateResult(page, page_state)
// }
async function clearSnapshotDirSymlinks(snap_dir) {
// delete all archive/<id>/* symlinks in preparation for new snapshot output to be placed there
const existing_symlinks =
(await fs.promises.readdir(snap_dir, {withFileTypes: true}))
.filter(dirent => {
if (dirent.name.startsWith('.')) return false // ignore hidden files, dont version them
if (dirent.name == 'versions') return false // dont try to move versions folder into itself
return dirent.isSymbolicLink()
})
for (const {name: existing_symlink} of existing_symlinks) {
await fs.promises.unlink(path.join(snap_dir, existing_symlink))
// if symlinks are not cleared before starting, it can cause issues with outputs writing into previous versions folders
// e.g. screerecording saves to ./media which could be pointing to previous version's ./versions/<olddate>/media
}
}
async function symlinkBestSnapshotResults(snap_dir) {
// move any existing files into versions/<date> folder (clear out main folder)
// symlink latest files from versions/<date>/* into main folder
await fs.promises.mkdir(path.join(snap_dir, 'versions'), {recursive: true})
process.chdir(snap_dir)
const metrics_file = path.join(snap_dir, 'metrics.json')
// if (!fs.existsSync(metrics_file) || (await fs.promises.lstat(metrics_file)).isSymbolicLink()) {
// console.warn('[⚠️] Warning, found partial dirty snapshot state (did the snapshot get interrupted?)', snap_dir)
// }
// move output files into versioned folder
await collectSnapshotDirVersionFiles(snap_dir)
// clear any existing symlinks
await clearSnapshotDirSymlinks(snap_dir)
// assert task dir is empty and contains no bare files that might get overwritten, also asserts version dirs are valid
await assertSnapshotDirIsValid(snap_dir, {is_empty: true})
const version_dirs = (await fs.promises.readdir(path.join(snap_dir, 'versions'))).sort() // earliest to latest
const most_recent = version_dirs.at(-1)
// for each version dir in versions/ (oldest -> newest)
for (const version_dir of version_dirs) {
if (version_dir.startsWith('.')) continue
const version_dir_abspath = path.join(snap_dir, 'versions', version_dir)
const version_dir_files = (
(await fs.promises.readdir(version_dir_abspath))
.filter(filename => !filename.startsWith('.')))
// iterate through all the files/folders in the version dir
for (const filename of version_dir_files) {
const snapdir_entry = path.join(snap_dir, filename) // ./data/archive/<snapid>/filename
const versiondir_entry = path.join(snap_dir, 'versions', version_dir, filename) // ./data/archive/<snapid>/versions/<versionid>/filename
if (fs.existsSync(snapdir_entry)) {
// if an entry already exists in the snapshot root for this filename
if ((await fs.promises.lstat(snapdir_entry)).isSymbolicLink()) {
// if a symlink already exists in the root with the same name,
// check if the version file we're looking at is a better candidate to replace it
const existing_abspath = await fs.promises.realpath(snapdir_entry)
const desired_abspath = path.join(version_dir_abspath, filename)
if (existing_abspath != desired_abspath) {
// check if the new candidate is larger or if the existing symlink is larger (largest file = most likely to be highest quality capture data)
const largest_path = await getLargestPath(existing_abspath, desired_abspath)
if (largest_path != (await fs.promises.realpath(existing_abspath))) {
const larger_version = path.basename(path.dirname(largest_path))
const larger_abspath = path.join(snap_dir, 'versions', larger_version, filename)
// console.log(' - swapping for larger file:', filename, '->', larger_abspath.split('/archive/').at(-1))
await overwriteSymlink(larger_abspath, snapdir_entry, {search_limit: snap_dir})
} else {
// console.log(' - leaving larger file:', largest_path.split('/archive/').at(-1))
}
} else {
// leave existing symlink pointing to current version file, nothing to change
// console.log(' - leaving current file:', existing_abspath.split('/archive/').at(-1))
}
} else {
// clearSnapshotDirSymlinks() should have already cleared these files out!
throw `Non-symlink file found in root of snapshot dir! Refusing to overwrite: ${prettyPath(snapdir_entry)}`
}
} else {
// no entry exists in the snapshot root for this filename, create one by linking to the version file
await overwriteSymlink(versiondir_entry, snapdir_entry, {search_limit: snap_dir})
}
// if (version_dir == most_recent) {
// // only log most recent links even though we link older ones too (otherwise its too noisy)
// console.log(` 🔗 ./${filename} -> ./${versiondir_entry} linking...`)
// }
}
}
return snap_dir
}
async function assertSnapshotDirIsValid(snap_dir, {is_empty=false}={}) {
process.chdir(snap_dir)
console.log()
console.log(`[☑️] Checking that snapshot records are valid...`)
// get all directory entries in archive/<snapshot_id>/*
const snapshot_dir_entries =
(await fs.promises.readdir(snap_dir, {withFileTypes: true}))
.filter(dirent => {
if (dirent.name.startsWith('.')) return false
if (dirent.name == 'versions') return false
})
// assert versions folder exists and is not a symbolic link
const versions_dir = path.join(snap_dir, 'versions')
assert(fs.existsSync(versions_dir))
assert(!(await fs.promises.lstat(versions_dir)).isSymbolicLink())
// if it should be empty, check that no loose files exist
if (is_empty) {
assert(!snapshot_dir_entries.length, `Found loose files in snapshot-dir that shouldn't be there! ${snap_dir}`)
}
// assert all non-hidden files in snapshot dir are symbolic links to actual data in versions/<date>/*
for (const snapshot_dir_entry of snapshot_dir_entries) {
if (snapshot_dir_entry.name.startsWith('.')) continue
if (snapshot_dir_entry.name == 'versions') continue
assert(snapshot_dir_entry.isSymbolicLink(), `Found non-symbolic link in root of snapshot dir! ${snap_dir}/${snapshot_dir_entry.name}`)
assert(fs.existsSync(snapshot_dir_entry.name), `Found broken symbolic link in root of snapshot dir! ${snap_dir}/${snapshot_dir_entry.name}`)
}
const version_entries = (
(await fs.promises.readdir(versions_dir))
.filter(foldername => !foldername.startsWith('.'))
.sort())
console.log(` √ ${prettyPath(versions_dir)}`, version_entries.length)
for (const version_dir of version_entries) {
await assertVersionDirIsValid(path.join(versions_dir, version_dir))
}
// write snapshot dir file listing w/ sizes & hashes to .files.json
const directory_info = await getDirInfo(snap_dir, {withRoot: true, withHelpers: false, maxdepth: 3})
await overwriteFile(path.join(snap_dir, '.files.json'), directory_info)
}
async function assertVersionDirIsValid(version_dir) {
const dirname = path.parse(version_dir).name
assert(fs.existsSync(version_dir), `Version dir does not exist: ${prettyPath(version_dir)}`)
const dirent = await fs.promises.lstat(version_dir)
assert(dirent.isDirectory() && !dirent.isSymbolicLink(), `Found non-directory in versions dir! ${prettyPath(version_dir)}`)
const unix_epoch = '19700101000000'
const is_name_valid_datestr = /^\d+$/.test(dirname) && (dirname.length == 14) && (dirname.startsWith('2') || dirname == unix_epoch) && parseVersionDateStr(dirname)
assert(is_name_valid_datestr, `Version directories must be a 14-character long date string like 20251231235959! ${dirname}`)
// get all directory entries in archive/<snapshot_id>/versions/<version_id>/*
const version_dir_entries = (
(await fs.promises.readdir(version_dir, {withFileTypes: true}))
.filter((dirent) => !dirent.name.startsWith('.')))
// assert version dir contains only actual snapshot output files (not-symbolic links or other version dirs)
for (const version_dir_entry of version_dir_entries) {
assert(version_dir_entry.name != 'versions', `Version dir cannot contain another versions folder! ${prettyPath(version_dir)}/versions`)
assert(!version_dir_entry.isSymbolicLink(), `Version dir cannot contain symbolic link! ${prettyPath(version_dir)}/${version_dir_entry.name}`)
}
// color highlight the unix epoch version in black, and any version created today in blue
let pretty_dirname = dirname
if (dirname == unix_epoch) {
pretty_dirname = ANSI.black + unix_epoch + ANSI.reset
}
const today = versionStrFromDate(new Date(), {withDate: true, withTime: false})
if (dirname.startsWith(today)) {
pretty_dirname = ANSI.blue + dirname + ANSI.reset
}
// write version dir file listing w/ sizes & hashes to .files.json
const directory_info = await getDirInfo(version_dir, { withRoot: true, withHelpers: false, maxdepth: 3 })
await overwriteFile(path.join(version_dir, '.files.json'), directory_info)
console.log(` √ ./versions/${pretty_dirname} contains`, version_dir_entries.length, 'results')
}
async function setupSnapshotDB({ original_url, start_time, snapshot_dir }) {
// setup Snapshot database row, finding it if it already exists or creating a new one
const timestamp = snapshot_dir.split('/').at(-1)
const search_attrs = { url: original_url, timestamp }
const update_attrs = { url: original_url, timestamp, added: start_time, title: null }
let snapshot = await Snapshot.findOne({ where: search_attrs });
let created = false
if (!snapshot) {
snapshot = await Snapshot.findOne({ where: {url: original_url} });
if (snapshot) {
// console.warn(`[X] Found DB Snapshot [${timestamp}](${original_url.substring(0, 30)}...) that has different timestamp from existing dir ${prettyPath(snapshot_dir)}!`)
// throw 'Snapshot DB record does not match filesystem path!'
} else {
console.log(`[+] Creating new DB Snapshot [${timestamp}](${original_url.substring(0, 30)}...) for ${prettyPath(snapshot_dir)}...`)
// ;([snapshot, created] = await Snapshot.findOrCreate({where: search_attrs, defaults: update_attrs }));
// throw 'Wanted to create new Snapshot but refusing to modify DB during testing!'
}
}
// assert(snapshot && (snapshot instanceof Snapshot))
return snapshot
}
async function setupViewport(page, _page_state) {
// setup viewport
await page.setViewport(DEFAULT_VIEWPORT);
await page.setGeolocation(DEFAULT_GEOLOCATION);
// await page.setBypassCSP(true); // bypass CSP restrictions (requires --disable-web-security)
page.setDefaultTimeout(DEFAULT_TIMEOUT);
// Optional: emulate a mobile device
// await page.emulate(puppeteer.devices['iPhone 6']);
// Configure light mode/dark mode & accessibility reduced motion preferences
await page.emulateMediaFeatures([
{name: 'prefers-color-scheme', value: DEFAULT_COLOR_SCHEME},
{name: 'prefers-reduced-motion', value: 'reduce'},
]);
// Setup headers & deterministically chose a random referrer based on URL
const rand_idx = hashCode(await page.url()) % DEFAULT_REFERRERS.length
await page.setExtraHTTPHeaders({
...DEFAULT_HEADERS,
referrer: DEFAULT_REFERRERS[rand_idx],
})
// Setup alert to trigger if site tries to sniff whether we are a bot
function sniffDetector() {
const userAgent = window.navigator.userAgent;
const platform = window.navigator.platform;
// @ts-ignore
window.navigator.__defineGetter__('userAgent', function () {
// @ts-ignore
window.navigator.sniffed = true;
return userAgent;
});
// @ts-ignore
window.navigator.__defineGetter__('platform', function () {
// @ts-ignore
window.navigator.sniffed = true;
return platform;
});
}
await page.evaluateOnNewDocument(sniffDetector);
// @ts-ignore
const was_sniffed = await page.evaluate(() => (!!window.navigator.sniffed))
if (was_sniffed) {
console.warn('[⚠️] Site tried to sniff if we are a bot! Site may be difficult to archive.')
}
return page
}
async function setupModalAutoClosing(page, page_state, {timeout=1_250}={}) {
page.on('dialog', (dialog) => {
console.log(`[👆] Auto-closing modal that popped up: ${dialog.message()}...`)
setTimeout(() => {try { dialog.accept() } catch(err) {}}, timeout);
})
// if you expect a file-upload dialog, use this to catch it instead:
// const [fileChooser] = await Promise.all([
// page.waitForFileChooser(),
// ]);
// await fileChooser.accept(['/tmp/myfile.pdf']);
page.on('close', () => {
try {
page.off('dialog')
} catch(err) {}
})
}
async function startScreenrecording(page, page_state, {duration_limit=60, codec='libx264'}={}) {
await fs.promises.mkdir(path.dirname(SCREENRECORDING_PATH(page)), {recursive: true})
// console.log(`[🎬] Starting screen-recording stream...`.padEnd(82), prettyPath(SCREENRECORDING_PATH(page)))
// alternative: interact with low-level puppeteer screencast API directly
// using puppeteer.page.screencast: https://pptr.dev/api/puppeteer.page.screencast
// const recorder = await page.screencast({path: SCREENRECORDING_PATH(page)});
// alternative: use puppeteer-stream for .webm/.mp4 screen recordings with tab audio included
// works sometimes but has a few issues, e.g.: https://github.com/SamuelScheit/puppeteer-stream/issues/8
// alternative: puppeteer-screen-recorder (most compatible/stable but doesn't include tab audio output)
const recorder = new PuppeteerScreenRecorder(page, {
followNewTab: false,
recordDurationLimit: duration_limit,
// fps: 25,
// ffmpeg_Path: '<path of ffmpeg_path>' || null,
// videoFrame: {
// width: 1024,
// height: 768,
// },
// videoCrf: 18,
videoCodec: codec,
// videoPreset: 'ultrafast',
// videoBitrate: 1000,
// autopad: {
// color: 'black' | '#35A5FF',
// },
// aspectRatio: '4:3',
});
page_state.recorder = recorder
await recorder.start(SCREENRECORDING_PATH(page))
page.on('close', async () => {await saveScreenrecording(page, page_state)});
return page_state
}
async function startResponseSaving(page, page_state) {
const dir = RESPONSES_PATH(page)
await fs.promises.mkdir(dir, {recursive: true})
console.log(`[🌄] Starting raw response bytes recording...`.padEnd(82), prettyPath(dir) + '/')
// Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, Manifest, SignedExchange, Ping, CSPViolationReport, Preflight, Other
const types_to_save = [
// 'document',
'script',
'stylesheet',
'font',
'image',
'media',
'xhr',
'websocket',
]
// reset responses index file to empty
const responses_log_path = path.join(dir, 'index.jsonl')
await overwriteFile(responses_log_path, '')
// add handler to save all image repsonses into output directory
page.on('response', async (response) => {
try {
const timestamp = versionStrFromDate(new Date(), {withDate: true, withTime: true, withSeconds: true, withMilliseconds: true})
if (!page_state.main_response && (response.request().url() == page_state.original_url)) {
// save first response as main page response (if we havent already caught it earlier)
page_state.main_response = response
}
const status = response.status()
if ((status >= 300) && (status < 500)) {
// console.log('Got bad response from', response.url(), 'to', response.headers()['location'])
return
}
const request = response.request()
const resourceType = request.resourceType()
const url_scheme = (response.url() || request.url()).split(':')[0].toLowerCase()
const method = (url_scheme === 'data') ? 'DATA' : request.method()
// console.log(' ', resourceType, response.url())
if (types_to_save.includes(resourceType)) {
// create ./responses/xhr/www.facebook.com/static/images/icons/ subdir based on hostname + path
const resource_type_dir = path.join(dir, resourceType)
const url = new URL(response.url())
let subdir = resource_type_dir
const url_path = (url.pathname || '').slice(0, 250).endsWith('/')
? (url.pathname || '').slice(0, 250)
: path.dirname((url.pathname || '').slice(0, 250))
// determine subdirectory based on url type (handles http:,https:,file:,data:,chrome-extension:,about:,etc.)
if (!URL_SCHEMES_IGNORED.includes(url_scheme)) {
// is a normal http:// or https:// url, use the domain + path to construct subdirectory
subdir = path.join(resource_type_dir, (url.hostname || 'data').slice(0, 250), url_path)
} else if (url_scheme == 'data') {
// is a data:... url, store in ./data subdirectory
subdir = path.join(resource_type_dir, 'data')
} else {
// is a chrome-extension:// or other special url, use the extension id + path to construct subdirectory
const url_path = path.dirname((url.pathname || '').slice(0, 999))
subdir = path.join(resource_type_dir, url_scheme, (url.hostname || 'data').slice(0, 250), url_path)
}
// write response to responses/all/1716861056899__https%3A%2F%2Fwww.instagram.com%2Fgraphql%2Fquery.json
let abspath = null
let resp_mimetype = null
let extension = ''
let uniq_filename = null
let uniq_abspath = null
let symlink_abspath = null
let responseSha256 = null
try {
await fs.promises.mkdir(path.join(dir, 'all'), {recursive: true})
try {
await fs.promises.mkdir(subdir, {recursive: true})
} catch(err) {
subdir = subdir + '.dir' // TODO: apply this workaround to parent path entries too
try {
await fs.promises.mkdir(subdir, {recursive: true})
} catch(err) {
subdir = path.join(resource_type_dir, 'data')
await fs.promises.mkdir(subdir, {recursive: true})
}
}
;({abspath: symlink_abspath, resp_mimetype, extension} = await detectFilename({page, response, dir: subdir, resourceType}))
// responses/all/1716861056899__https%3A%2F%2Fwww.instagram.com%2Fgraphql%2Fquery.json
uniq_filename = `${timestamp}__${method}__` + [encodeURIComponent(url.href).slice(0, 64).replaceAll('/', '_').replace(new RegExp(`.${extension}$`), ''), extension].filter(s => s.length).join('.')
uniq_abspath = path.join(dir, 'all', uniq_filename)
let bytesBuffer = null
try {
bytesBuffer = await response.buffer()
} catch(err) {
if (String(err).includes("Cannot read properties of undefined (reading 'body')")) {
// not sure why it's happening but seems to be too late to caputre body sometimes? possible race condition
} else {
console.warn('[⚠️] Failed to save response bytes for:', response.request().url(), err)
}
}
if (bytesBuffer) {
// write response data into ./all/<TS>__<METHOD>__<URL>.<EXT>
await overwriteFile(uniq_abspath, bytesBuffer)
responseSha256 = crypto.createHash('sha256').update(bytesBuffer).digest('hex')
// write symlink file to ./<TYPE>/<DOMAIN>/...<PATH>/<FILENAME>.<EXT> -> ./all/<TS>__<METHOD>__<URL>.<EXT>
await overwriteSymlink(uniq_abspath, symlink_abspath, {relative: dir, mkdirs: true, search_limit: dir})
}
// console.log(' ->', symlink_abspath)
} catch(err) {
// dont do anything for redirectresponses, error responses, etc.
console.warn(err)
}
const urlSha256 = crypto.createHash('sha256').update(String(request.url())).digest('hex')
// const headersSha256 = crypto.createHash('sha256').update(String(request.headers())) // someday we may want to save headers hashes too
const truncated_url = (method == 'DATA') ? request.url().slice(0, 128) : request.url() // don't duplicate bytes in data: urls (we already saved them in the file)
// this is essentially replicating the functionality of a WARC file, but in directory + index.jsonl form
await fs.promises.appendFile(
responses_log_path,
JSON.stringify({
ts: timestamp,
method,
url: truncated_url,
urlSha256,
postData: request.postData(),
response_url: ((method != 'DATA') && (url.href != request.url())) ? url.href : undefined,
status,
resourceType,
mimeType: resp_mimetype,
responseSha256,
path: uniq_abspath?.replace(dir, '.'),
symlink_path: symlink_abspath?.replace(dir, '.'),
extension,
}) + '\n',
'utf-8',
)
}
} catch(err) {
// we should never throw hard errors here because there's nothing above us to catch it
// and we dont want to crash the entire CDP session / browser / main node process
console.warn('[❌] Error in response handler (set in startResponseSaving):', err)
}
});
// handled by stopMetadataRecording():
// page.on('close', () => {
// page.off('response')
// })
}
function dedupeCookies(cookies) {
const len_before = cookies.length
const allowed_cookie_attrs = ['domain', 'path', 'name', 'value', 'expires', 'sameSite', 'sourceScheme', 'url', 'priority', 'secure', 'httpOnly']
const deduped_cookies = {}
for (const cookie of cookies) {
try {
const unique_id = `${cookie.domain}${cookie.path}${cookie.name}`
deduped_cookies[unique_id] = {
...(deduped_cookies[unique_id] || {}),
...cookie,
expires: 2147483640, // max allowed expiry time (2038-01-18)
session: false, // make sure cookies dont expire at browser close time
secure: false, // make cookie restrictions more lax (for archiving scripts)
httpOnly: false, // make it easier to tamper with cookies from JS (for archiving scripts)
// "path": "/",
// "expires": 2147483641,
// "size": 194,
// "httpOnly": false,
// "secure": false,
// "session": false,
// "priority": "High",
// "sameParty": false,
// "sourceScheme": "Secure",
// "sourcePort": 443
// and more... https://pptr.dev/api/puppeteer.cookieparam
} as Cookie
if (!deduped_cookies[unique_id].value) {
delete deduped_cookies[unique_id]
continue
}
if (deduped_cookies[unique_id].name.startsWith('__')) {
// cookies that start with __ must be secure, see https://github.com/puppeteer/puppeteer/issues/6806
deduped_cookies[unique_id].secure = true
deduped_cookies[unique_id].sourceScheme = 'Secure'
}
if (deduped_cookies[unique_id].domain.startsWith('.')) {
deduped_cookies[unique_id].sameParty = false
deduped_cookies[unique_id].domain = deduped_cookies[unique_id].domain.slice(1)
}
for (const key of Object.keys(deduped_cookies[unique_id])) {
if (!allowed_cookie_attrs.includes(key)) {
delete deduped_cookies[unique_id][key]
}
}
} catch(err) {
console.error('[❌] Failed to parse cookie during deduping', cookie)
throw err
}
}
// console.log(`[🍪] Deduped ${len_before} cookies to ${Object.keys(deduped_cookies).length}...`)
return Object.values(deduped_cookies) as Cookie[]
}
async function loadCookiesTxt() {
const cookies = [] as Cookie[]
return cookies // write-only from chrome -> files for now
if (fs.existsSync(COOKIES_TXT_PATH)) {
// console.log(`[🍪] Loading cookies/localStorage/sessionStorage from ${COOKIES_TXT_PATH}...`)
// Read from to cookies.txt file using tough-cookie + @root/file-cookie-store
const cookies_store = new FileCookieStore(COOKIES_TXT_PATH, {auto_sync: false, lockfile: false});
cookies_store.getAllCookiesAsync = util.promisify(cookies_store.getAllCookies);
const exported_cookies = await cookies_store.getAllCookiesAsync()
for (const cookie of exported_cookies) {
const cookie_from_tough = cookie.toJSON()
const domain = cookie_from_tough.hostOnly ? `.${cookie_from_tough.domain}` : cookie_from_tough.domain
const cookie_for_puppeteer: Cookie = {
domain,
name: cookie_from_tough.key,
path: cookie_from_tough.path,
value: cookie_from_tough.value,
secure: cookie_from_tough.secure || false,
httpOnly: cookie_from_tough.httpOnly || false,
session: false,
expires: (new Date(cookie_from_tough.expires)).valueOf()/1000,
size: undefined,
}
// console.log('COOKIE_FROM_TOUGH_TXT', cookie_from_tough, cookie_for_puppeteer)
cookies.push(cookie_for_puppeteer)
}
}
}
type AuthJSON = {
cookies: Cookie[],
sessionStorage: any,
localStorage: any,
}
async function loadAuthStorage(page, {client}, {apply=true}={}) {
var {
cookies,
sessionStorage,
localStorage,
}: AuthJSON = {cookies: [], sessionStorage: {}, localStorage: {}}
if (!LOAD_AUTH_STORAGE) {
// dont read auth from filesystem auth.json/cookies.txt, just rely on existing cookies in chrome profile
return {cookies, sessionStorage, localStorage}
}
if (fs.existsSync(COOKIES_TXT_PATH)) {
try {
cookies = await loadCookiesTxt()
} catch(err) {
console.warn('[⚠️] Loaded invalid cookies.txt, moved it to cookies.txt.corrupted (did two processes try to change it at the same time?)')
await fs.promises.rename(COOKIES_TXT_PATH, COOKIES_TXT_PATH + '.corrupted')
}
// console.log(`[🍪] Loading cookies from cookies.txt...`, cookies.length)
}
if (fs.existsSync(AUTH_JSON_PATH)) {
try {
var {
cookies: auth_json_cookies,
sessionStorage,
localStorage,
} = JSON.parse(await fs.promises.readFile(AUTH_JSON_PATH, 'utf-8'));
cookies = [...cookies, ...auth_json_cookies]
// console.log(`[🍪] Loading cookies from auth.json...`, auth_json_cookies.length)
} catch(err) {
console.warn('[⚠️] Loaded invalid auth.json, moved it to auth.json.corrupted (did two processes try to change it at the same time?)')
await fs.promises.rename(AUTH_JSON_PATH, AUTH_JSON_PATH + '.corrupted')
}
}
cookies = dedupeCookies(cookies)
if (apply) {
console.log(`[🍪] Loading stored cookies/localStorage/sessionStorage into session...`, cookies.length)
// if (cookies?.length) {
// try {
// // try setting all at once first (much faster)
// await page.setCookie(...cookies)
// } catch(err) {
// // if any errors, fall back to setting one-by-one so that individual error can be caught
// for (const cookie of cookies) {
// try {
// await page.setCookie(cookie);
// } catch(err) {
// console.error('[❌] Failed to set cookie', cookie)
// throw err
// }
// }
// }
// }
const origin = await page.evaluate(() => window.location.origin)
await page.evaluate((savedSessionStorage) => {
for (const [key, value] of Object.entries(savedSessionStorage)) {
sessionStorage[key] = value;
}
}, sessionStorage[origin] || {});
await page.evaluate((savedLocalStorage) => {
for (const [key, value] of Object.entries(savedLocalStorage)) {
localStorage[key] = value;
}
}, localStorage[origin] || {});
// origin/auth context changes when we do page.goto so we have to hook pageload and apply it then as well
// https://stackoverflow.com/questions/51789038/set-localstorage-items-before-page-loads-in-puppeteer
await page.evaluateOnNewDocument(({sessionStorage, localStorage}) => {
const origin = window.location.origin;
for (const [key, value] of Object.entries(sessionStorage[origin] || {})) {
window.sessionStorage.setItem(key, value as string)
}
for (const [key, value] of Object.entries(localStorage[origin] || {})) {
window.localStorage.setItem(key, value as string)
}
}, {sessionStorage, localStorage});
}
return {cookies, sessionStorage, localStorage}
}
async function loadCloudflareCookie(page, {original_url}, {timeout=20_000}={}) {
// make request to FlareSolverr server to get magic cookies that let us bypass cloudflare bot detection
// docker run -p 8191:8191 -e LOG_LEVEL=info ghcr.io/flaresolverr/flaresolverr
// alternatives if this stops working:
// - https://github.com/omkarcloud/botasaurus
// - https://github.com/ultrafunkamsterdam/nodriver
// - https://github.com/Akmal-CloudFreed/CloudFreed-CloudFlare-bypass
// - https://github.com/VeNoMouS/cloudscraper
const query = { url: original_url, cmd: "request.get", maxTimeout: timeout }
try {
const response = await fetch(FLARESOLVERR_API_ENDPOINT, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(query),
});
const data = await response.json();
const new_cookies = (data?.solution?.cookies || []).map(cookie => ({
...cookie,
'expires': 2147483640, // overwrite expiration to 32bit maximum timestamp (2038-01-18)
'secure': false, // cookie value is plain text (not encrypted/encoded)
}))
if (new_cookies.length) {
console.log(`[☑️] Got Cloudflare bypass cookies (${new_cookies.length}) from FlareSolverr API...`)
await page.setCookie(...new_cookies);
return new_cookies
} else {
const error_str = JSON.stringify(data?.message || data, null, 4)
throw `Bad FlareSolverr Response: ${error_str}`
}
} catch (error) {
if (JSON.stringify(error).includes('Challenge not detected')) {
console.log('[☑️] Page is accessible without FlareSolverr Cloudflare bypass.')
} else {
console.warn('[❌] Failed to get Cloudflare bypass cookies from FlareSolverr API.', error)
}
}
return []
}
async function setupURLRewriting(page, page_state) {
await page.setRequestInterception(true);
const rewrites = URL_REWRITES.sort((a, b) => (a.idx || 0) - (b.idx || 0))
page.on('request', interceptedRequest => {
if (interceptedRequest.isInterceptResolutionHandled()) return;
const original_url = interceptedRequest.url()
// apply all the rewrites in order to the request URL
let url = original_url
for (const rewrite of rewrites) {
const new_url = url.replace(rewrite.pattern, rewrite.replacement)
// console.log(rewrite, url, new_url)
// if url is rewritten to an emptystring, abort the request
if (!new_url) {
console.warn('[🟥] Request blocked', rewrite.pattern, ':', url)
interceptedRequest.abort()
return
}
else if (new_url && new_url != url) {
// console.warn('[📳] Request rewritten', rewrite.pattern, rewrite.replacement, ':', url, '->', new_url)
console.warn('[📳] Request rewritten', rewrite.pattern, ':', new_url)
url = new_url
}
}
if (url == original_url) {
// if url is unchanged, continue request flow as-is
interceptedRequest.continue()
} else {
// otherwise redirect the browser to our rewritten version
interceptedRequest.respond({
status: 302,
headers: {
location: url,
'x-redirect-by': 'ArchiveBox.setupURLRewriting',
},
})
}
});
// handled by stopMetadataRecording():
// page.on('close', () => {
// page.off('request')
// page.setRequestInterception(false)
// })
}
async function startMetadataRecording(page, {original_url, version, client, traffic_log, console_log, redirects}) {
// update helper state on page
page._original_url = (original_url || (await page.url())).toString()
// DEBUGGING: helpers for repl() debugging, dont rely on these (global state is badd mmkay)
// page._client = client || page._client || await page.target().createCDPSession()
// page._redirects = redirects
// page._traffic_log = traffic_log
// add initial entry to page redirect log
redirects[original_url] = {
idx: 0,
url: original_url,
src: null,
type: 'Initial',
wallTime: Date.now()/1000,
frameId: page.mainFrame()._id,
requestId: null,
initiator: {type: "user"},
isMainFrame: true,
}
// DEBUGGING: record optional chrome debug trace with screenshots (heavy)
// try {
// await page.tracing.stop()
// await wait(200)
// } catch(err) {}
// try {
// await page.tracing.start({path: TRACE_PATH(page), screenshots: true});
// } catch(err) {}
let last_main_frame_url = original_url
// setup network request intercepts handler
const addCDPRequestDataListener = (eventName) => {
client.on(eventName, event => {
try {
// save any HTTP/JS redirects to redirects for saveRedirects(page) to use later on
const new_url = event.documentURL
const http_status = event.redirectResponse?.status || 0
const is_new_url = (new_url !== original_url) && !redirects[new_url]
const is_main_frame_navigation = (event.frameId == page.mainFrame()._id)
const is_http_redirect = (300 < http_status) && (http_status < 400)
if (new_url && is_new_url && (is_main_frame_navigation || is_http_redirect) && event.type == 'Document') {
const new_redirect_entry = {
url: new_url,
src: event.redirectResponse?.url || last_main_frame_url,
type: http_status || 'JS',
wallTime: Date.now()/1000,
frameId: event.frameId,
requestId: event.requestId,
initiator: event.initiator,
idx: Object.keys(redirects).length,
isMainFrame: is_main_frame_navigation,
}
redirects[new_url] = new_redirect_entry
if (is_main_frame_navigation) {
ALREADY_ARCHIVED.add(new_redirect_entry.url.slice(0, 4096)) // we're already archiving this tab as it redirects, dont create a duplicate archive for the destination
console.warn(`[➡️] NAVIGATION[${new_redirect_entry.type}]${ANSI.blue} ${last_main_frame_url} ${ANSI.reset}\n ->${ANSI.blue} ${new_redirect_entry.url} ${ANSI.reset}`)
last_main_frame_url = new_url
}
}
if (event.loaderId) {
traffic_log[event.loaderId] = traffic_log[event.loaderId] || {} // make sure loader is also in requests list first
// sometimes it's not in the list if we start archiving too late / after a page's initial request was already made
}
// save to traffic_log as {8BC2087A2CCEF28017099C0E10E87440: {Network.eventWillBeSent: {eventId,loaderId, request|response, ...}}
// https://stackoverflow.com/questions/47078655/missing-request-headers-in-puppeteer?noredirect=1&lq=1
traffic_log[event.requestId] = traffic_log[event.requestId] || {}
Object.assign(traffic_log[event.requestId], { [eventName]: event })
// DEBUGGING: log page visits and navigation events to console
// if (event?.response?.status) {
// // if we're expecting an HTML response, then we assume it's a page visit & log it to console
// const acceptMimeType = traffic_log[event.requestId]['Network.requestWillBeSentExtraInfo']?.headers?.accept
// if (acceptMimeType && acceptMimeType.includes('text/html')) {
// // log any HTML page responses (less noisy)
// console.log(`[>] GOT ${event.documentURL}: ${event.response.status} ${event.response.url} (${event.response.mimeType})`)
// } else {
// // log ALL responses, inclusing JS,CSS,Images,etc. (very noisy)
// // console.log(` > ${event.response.status} ${event.response.url} (${event.response.mimeType})`)
// }
// }
} catch(err) {
console.warn('[X] Error during request/response handler (startMetadataRecording.addCDPRequestDataListener)')
console.warn(err)
}
})
}
addCDPRequestDataListener('Network.requestWillBeSent')
addCDPRequestDataListener('Network.requestWillBeSentExtraInfo')
addCDPRequestDataListener('Network.responseReceived')
addCDPRequestDataListener('Network.responseReceivedExtraInfo')
// clear any existing log entries
const consolelog_info = {
TYPE: 'console',
VERSION: version,
URL: original_url,
}
await overwriteFile(CONSOLELOG_PATH(page), JSON.stringify(consolelog_info) + '\n')
// record console logs from page
const appendConsoleLog = async (line) => {
if (!line) return
console_log.push(line)
await fs.promises.appendFile(
CONSOLELOG_PATH(page),
line + '\n',
'utf-8',
)
}
page.on('console', async(message) =>
await appendConsoleLog(`${message.type().toUpperCase()} ${message.location()} ${JSON.stringify(message.text())}`))
page.on('pageerror', async (error) =>
await appendConsoleLog(error.message || JSON.stringify(error)))
page.on('requestfailed', async (request) =>
await appendConsoleLog(`${request.failure()?.errorText} ${request.url() || JSON.stringify(request)}`))
// set puppeteer options on page
await client.send('Network.enable') // enable network tampering API
await client.send('Emulation.clearDeviceMetricsOverride'); // clear timing statistics
await client.send('Page.setDownloadBehavior', {
behavior: 'allow',
downloadPath: CHROME_DOWNLOADS_DIR,
})
// handled by stopMetadataRecording():
// page.on('close', () => {
// try {
// page.off('request')
// page.off('console')
// page.off('pageerror')
// page.off('requestfailed')
// page.setRequestInterception(false)
// } catch(err) {
// // some versions of puppeteer have had race conditions here where page is already closed by now
// console.warn('[X] Error in page close handler', err)
// }
// })
return {original_url, client, redirects, traffic_log, console_log}
}
async function stopMetadataRecording(page, _page_state) {
console.log('[🪝] Stopping CDP event hooks and request interception...')
try {
page.off('request')
page.off('response')
page.off('console')
page.off('pageerror')
page.off('requestfailed')
page.off('hashchange')
page.setRequestInterception(false)
// page.tracing.stop()
} catch(err) {
// some versions of puppeteer have had race conditions here where page is already closed by now
console.warn('[X] Error in page close handler', err)
}
}
/********************** Human Behavior Emulation ******************************/
async function solveCaptchas(page, page_state, {timeout=90_000}={}) {
// using puppeteer-extra-plugin-recaptcha auto-solver
// await page.solveRecaptchas()
// using 2captcha-solver extension auto-solver
try {
// console.log('[🕑] Waiting for CAPTCHA to appear...')
await page.waitForSelector('.captcha-solver', {timeout: 5_000})
console.log('[🤖] CAPTCHA challenge found, submitting to 2Captcha for solving...')
await page.click('.captcha-solver')
console.log(`[🧠] Waiting up to ${timeout/1000}s for CAPTCHA to be solved...`)
await page.waitForSelector(`.captcha-solver[data-state="solved"]`, {timeout})
console.log('[🔓] CAPTCHA solution retrieved from 2captcha.')
} catch(err) {
console.log('[☑️] No CATPCHA challenges found, site thinks we are human.')
}
}
async function jiggleMouse(page, page_state, {timeout=600}={}) {
console.log(`[🐁] Moving mouse around randomly for ${timeout/1000}s...`)
const randomPoint = await getRandomPagePoint(page)
const cursor = createCursor(page, randomPoint, true)
cursor.toggleRandomMove(true)
await wait(timeout/2);
await cursor.moveTo({x: DEFAULT_VIEWPORT.width/2, y: DEFAULT_VIEWPORT.height/2});
await wait(timeout/2);
cursor.toggleRandomMove(false)
}
async function blockRedirects(page, {original_url}) {
page.on('request', req => {
if (req.isInterceptResolutionHandled()) return;
// if it's a top-level navigation event to a new url
if (req.isNavigationRequest() && req.frame() === page.mainFrame() && req.url() !== original_url) {
req.abort('aborted');
console.warn('[🟥] Blocked page attempt to naviage to new URL', req.url())
} else {
req.continue();
}
});
// handled by stopMetadataRecording():
// page.on('close', () => {
// page.off('request')
// page.setRequestInterception(false)
// })
await page.setRequestInterception(true);
}
async function blockJSExecution(page, _page_state) {
console.warn('[🟥] Stopping all JS execution on page...')
await page.evaluate(() => {
debugger;
})
// OR alternatively this (more buggy, breaks many sites):
// const html = await page.content();
// page.setJavaScriptEnabled(false);
// await page.setContent(html, { waitUntil: 'networkidle0' }); // 4
}
async function scrollDown(page, _page_state, {timeout=120_000, scroll_delay=SCROLL_DELAY, scroll_distance=SCROLL_DISTANCE, scroll_limit=SCROLL_LIMIT}={}) {
const starting_height = await page.evaluate('document.body.scrollHeight');
let last_height = starting_height
let scroll_count = 0;
let scroll_position = scroll_count * scroll_distance
// await page.bringToFront()
// scroll to top
await page.evaluate(() => { window.scrollTo({ top: 0, left: 0, behavior: 'smooth' }); });
while ((scroll_count < scroll_limit) && ((scroll_delay * scroll_count) < timeout)) {
console.log(`[⬇️] Scrolling down ${scroll_count}x 1000px... (${scroll_position}/${last_height})`)
await page.evaluate((y_offset) => { window.scrollTo({ top: y_offset, left: 0, behavior: 'smooth' }); }, scroll_position);
scroll_count++
scroll_position = scroll_count * scroll_distance
// check if any new content was added / if we are infiniscrolling
let new_height = await page.evaluate('document.body.scrollHeight')
const added_px = new_height - last_height
if (added_px > 0) {
console.log('[✚] Detected infini-scrolling...', `${last_height}+${added_px} => ${new_height}`)
} else if (scroll_position >= new_height + scroll_distance) {
// we've reached the bottom, condition isn't true until we've tried to go n+1 past the end (which is fine)
if (scroll_count > 2)
break
}
last_height = new_height
// sleep 2s, perform the smooth scroll down by 1000px, and increment the counter
await wait(scroll_delay);
// facebook watch pages infiniscroll (more and more recommendations forever), stop them after 3 pages
if (page._original_url.startsWith('https://www.facebook.com/watch/?v') && scroll_count > 3) break
}
// scroll to bottom
if (scroll_position < last_height) {
await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, left: 0, behavior: 'smooth' }); });
await wait(scroll_delay)
await page.evaluate(() => { window.scrollTo({ top: document.body.scrollHeight, left: 0, behavior: 'smooth' }); });
}
// Always wait an additional 2sec at the end for scroll animations / loading / rendering to settle down
console.log('[📉] Reached bottom of the page.', `(${scroll_position}/${last_height})`)
await wait(scroll_delay);
await page.evaluate(() => { window.scrollTo({ top: 0, left: 0, behavior: 'smooth' }); });
await wait(scroll_delay);
return last_height
}
async function disableAnimations(page, _page_state) {
console.log(`[⛄️] Disabling all animations using CSS override...`)
// https://stackoverflow.com/questions/53167644/injecting-css-into-site-with-puppeteer
const css_override = `*, *::before, *::after {
-moz-animation: none !important;
-moz-transition: none !important;
animation: none !important;
transition: none !important;
caret-color: transparent !important;
}`
// inject override into current page
await page.addStyleTag({content: css_override});
// inject override into any subsequently navigated pages
await page.evaluateOnNewDocument((css_override) => {
const style_tag = document.createElement('style')
style_tag.type = 'text/css'
style_tag.innerHTML = css_override
document.getElementsByTagName('head')[0].appendChild(style_tag)
}, css_override);
}
async function expandComments(page, _page_state, {timeout=120_000, limit=15_000, delay=650}={}) {
console.log(`[🗃️] Expanding up to ${limit} comments every ${delay}ms...`)
// expand all <details> sections in Github READMEs, HedgeDoc pages, etc.
await page.$$eval('pierce/article details', elem => {elem.open = true}) // expand Github README details sections
await page.$$eval('pierce/div.js-discussion details:not(.details-overlay)', elem => {elem.open = true}) // expand Github issue discussion hidden comments
await page.$$eval('pierce/.markdown-body details', elem => {elem.open = true}) // expand HedgeDoc Markdown details sections
await page.exposeFunction('onHashChange', url => page.emit('hashchange', url));
await page.evaluateOnNewDocument(() => {
// @ts-ignore
addEventListener('hashchange', (e) => onHashChange(location.href));
});
// Listen for hashchange events in node Puppeteer code.
page.on('hashchange', url => console.log('Page tried to navigate to:', new URL(url)));
const num_expanded = await page.evaluate(async ({timeout, limit, delay}) => {
function getElementsByXPath(xpath, ctx?) {
var results = [];
var xpathResult = document.evaluate(
xpath, // e.g. //*[text()='"+text+"']
ctx || document,
null,
XPathResult.ORDERED_NODE_ITERATOR_TYPE,
null
);
var node;
while ((node = xpathResult.iterateNext()) != null) {
results.push(node);
}
return results;
}
let num_expanded = 0
const getLoadMoreLinks = () => [
// find all the buttons/links to expand collapsed/hidden/lazy-loaded content
...document.querySelectorAll('faceplate-partial[loading=action]'), // new reddit
...document.querySelectorAll('a[onclick^="return morechildren"]'), // old reddit show more replies
...document.querySelectorAll('a[onclick^="return togglecomment"]'), // old reddit show hidden replies
// ...document.querySelectorAll('a.js-show-link'), // stack overflow comments show more (TODO: make this only work on SO)
// ...document.querySelectorAll('a.morelink'), // HackerNews profile show more (TODO: make this only work on HN)
// ...getElementsByXPath("//*[text()~='View \d+ replies']"), // facebook comment expander
...getElementsByXPath("//*[text()='Show more replies']"), // twitter infiniscroll expander
...getElementsByXPath("//*[text()='Show replies']"), // twitter replies expander
]
const wait = (ms) => new Promise(res => setTimeout(res, ms))
let load_more_links = getLoadMoreLinks()
while (load_more_links.length) {
console.log('Expanding comments...', load_more_links.length)
for (const link of load_more_links) {
link.scrollIntoView({behavior: 'smooth'})
if (link.slot == 'children') {
continue
// patch new reddit "More replies" links that would open in a new window to display inline instead
// const comment_id = link.src.split('?')[0].split('/').at(-1)
// link.slot = `children-${comment_id}-0`
// link.__alwaysShowSlot = false
}
// click the "More replies" button
link.click()
num_expanded++
await wait(delay)
const time_elapsed = num_expanded * delay
if ((num_expanded > limit) || (time_elapsed > timeout))
return num_expanded
}
load_more_links = getLoadMoreLinks()
}
return num_expanded
}, {timeout, limit, delay});
page.off('hashchange')
if (num_expanded) {
console.log(`[🗃️] Expanded ${num_expanded} comments...`)
// scroll to bottom, then back up to top
const final_height = await page.evaluate('document.body.scrollHeight');
await page.evaluate((top) => { window.scrollTo({ top, left: 0, behavior: 'smooth' }); }, final_height + 1000);
await wait(delay);
await page.evaluate(() => { window.scrollTo({ top: 0, left: 0, behavior: 'smooth' }); });
await wait(delay);
}
}
async function submitForm(page, _page_state, {timeout=5_000}={}) {
try {
await page.waitForSelector('form button[type=submit]', {timeout: 1_500});
console.log('[☑️] Submitting form...')
await page.click('form button[type=submit]')
await page.waitForNavigation({timeout});
await page.goBack();
} catch (err) {
// no form found
}
}
// TODO: add an evasion to set navigator.connection.rtt = 365 (0 = detectable as headless)
/******************************************************************************/
/******************************************************************************/
/**************** Extension-Based Archive Output Tasks ************************/
async function saveSinglefile(page, {main_response, extensions}) {
const extension = extensions.filter(({name}) => name === 'singlefile')[0]
if (!extension.version) throw 'Could not find Singlefile extension ID, is it installed?'
const url = await page.url() || main_response.url()
if (URL_SCHEMES_IGNORED.includes(url.split(':')[0])) return null
// get list of existing past files in downloads/* to ignore
const files_before = new Set(
(await fs.promises.readdir(CHROME_DOWNLOADS_DIR))
.filter(fn => fn.endsWith('.html'))
);
const out_path = SINGLEFILE_PATH(page)
console.log(`[🛠️] Saving Singlefile HTML using extension (${extension.id})...`.padEnd(82+1), prettyPath(CHROME_DOWNLOADS_DIR))
await page.bringToFront() // action button acts on the foreground tab, so it has to be in front :(
await extension.dispatchAction()
let files_new = []
const check_delay = 3_000
for (const _try in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) {
await wait(check_delay)
const files_after = (await fs.promises.readdir(CHROME_DOWNLOADS_DIR)).filter(fn => fn.endsWith('.html'));
files_new = files_after.filter(file => !files_before.has(file))
if (files_new.length == 0) {
// console.warn(` ...waiting for Singlefile to write HTML into ${CHROME_DOWNLOADS_DIR}...`)
continue
}
// iterate through new downloads and find a matching .html containing our page's URL in the header
for (const file of files_new) {
const dl_path = path.join(CHROME_DOWNLOADS_DIR, file)
const dl_text = await fs.promises.readFile(dl_path, 'utf-8')
const dl_header = dl_text.split('meta charset')[0]
if (dl_header.includes(`url: ${url}`)) {
/// dont need this check anymore as now all output is versioned:
// if (fs.existsSync(out_path)) {
// const {size: existingSize} = await fs.promises.stat(out_path)
// const {size: newFileSize} = await fs.promises.stat(dl_path)
// if (newFileSize < existingSize) {
// console.log(`[🗑️] Discarding singlefile output (${file}) as it's smaller than existing ${out_path}...`)
// await fs.promises.rm(dl_path)
// return out_path
// }
// }
console.log(`[✍️] Moving Singlefile download from ${file}...`.padEnd(82), prettyPath(out_path))
await fs.promises.rename(dl_path, out_path)
return out_path
}
}
}
console.warn(`[❌] Couldn't find matching Singlefile HTML in ${CHROME_DOWNLOADS_DIR} after waiting ${(check_delay*10)/1000}s:`, files_new.join(', '))
return null
}
async function saveArchiveWebPage(page, {extensions}, {timeout=30_000}={}) {
// TODO: waiting on them to expose commands so we can generate .wacz easily
// https://github.com/webrecorder/archiveweb.page/issues/207
// ...
const browser = await page.browser()
const extension = extensions.filter(({name}) => name === 'archivewebpage')[0]
await page.bringToFront()
await extension.dispatchPopup()
await extension.dispatchAction()
const popup = await browser.waitForTarget(
target => target.url().toString().startsWith(`chrome-extension://${extension.id}/popup.html`),
{timeout: 5_000},
)
await page.bringToFront()
// await puppeteer.Locator.race([
// popup.locator('::-p-aria(Start With Autopilot)'),
// popup.locator('wr-popup-viewer >>>> input'),
// popup.locator(':scope >>> input')
// ])
// .setTimeout(timeout)
// .click({
// offset: {
// x: 7.7265625,
// y: 7.203125,
// },
// });
// @ts-ignore
await puppeteer.Locator.race([
popup.locator('wr-popup-viewer >>>> div.status-row > p'),
popup.locator(':scope >>> div.status-row > p'),
popup.locator('::-p-text(Recording: \n)')
]).setTimeout(timeout).click({
delay: 733.3000000007451,
offset: {
x: 293,
y: 13.5,
},
})
await wait(8_000)
// @ts-ignore
await puppeteer.Locator.race([
popup.locator('wr-popup-viewer >>>> div:nth-of-type(2) > button > span:nth-of-type(2)'),
popup.locator(':scope >>> div:nth-of-type(2) > button > span:nth-of-type(2)'),
popup.locator('::-p-text(Stop)')
]).setTimeout(timeout).click({
offset: {
x: 7.859375,
y: 23.203125,
},
});
return null
}
async function savePocket(page, {extensions}) {
const browser = await page.browser()
const extension = extensions.filter(({name}) => name === 'pocket')[0]
if (!extension.version) throw 'Could not find Pocket extension ID, is it installed?'
console.log(`[🛠️] Saving URL to Pocket API using extension (${extension.id})...`, 'https://getpocket.com/saves')
await page.bringToFront() // action button acts on the foreground tab, so it has to be in front
await extension.dispatchAction()
try {
const login_window = await browser.waitForTarget(
target => target.url().toString().startsWith('https://getpocket.com/'),
{timeout: 3_000},
)
// login window will open if pocket is not signed-in
if (login_window) return false
} catch(e) {
// no new window should open if it saves correctly
return true
}
}
/***************** Synchronous Archive Output Tasks ***************************/
async function saveScreenrecording(page, page_state, {save_gif=true}={}) {
if (page_state.recorder) {
const duration = Date.now() - page_state.start_ts
console.log(`[🎥] Saving screen-recording video (${duration/1000}s)...`.padEnd(82), prettyPath(SCREENRECORDING_PATH(page)))
const recorder = page_state.recorder
page_state.recorder = null
await recorder.stop()
// create symlink for legacy path
const snap_dir = page_state.snapshot_dir
const legacy_path = path.join(snap_dir, 'media', 'screenrecording.mp4')
await overwriteSymlink(SCREENRECORDING_PATH(page), legacy_path, {relative: snap_dir, search_limit: snap_dir})
// // remove duplicate frames (white frames at start while it loads + static image at end)
// const video_path = SCREENRECORDING_PATH(page)
// const short_path = video_path.replace('.mp4', '.short.mp4')
// try {
// await exec(
// // create a shortened video starting from 0:02s to 0:01s with duplicate frames removed (can look jumpy sometimes)
// `ffmpeg -ss 2 -sseof -1 -y -i ${video_path} -vf mpdecimate,setpts=N/FRAME_RATE/TB ${short_path}`
// )
// } catch(err) {
// console.log('[❌] Failed to shorten screenrecording.mp4')
// }
// convert video to GIF
if (save_gif) {
try {
const BIN_NAME = '/Volumes/NVME/Users/squash/bin/ffmpeg'
const child = child_process.spawn(
BIN_NAME,
[
'-hide_banner',
'-loglevel', 'error',
'-ss', '3',
'-t', '10',
'-y',
'-i', SCREENRECORDING_PATH(page),
'-vf', "fps=10,scale=1024:-1:flags=bicubic,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse",
'-loop', '0',
SCREENRECORDGIF_PATH(page),
],
{
cwd: path.dirname(SCREENRECORDING_PATH(page)),
timeout: 60_000,
// stdio: [null, 'pipe', 'pipe'],
stdio: 'ignore',
detached: true, // run in background, don't block on response
},
)
await blockUntilExists(SCREENRECORDGIF_PATH(page), {min_bytes: 100, timeout: 40_000})
console.log(`[🎥] Saved screen-recording GIF with ffmpeg pid=${child.pid} (${duration/1000}s)...`.padEnd(82), prettyPath(SCREENRECORDGIF_PATH(page)))
const snap_dir = page_state.snapshot_dir
const legacy_path = path.join(snap_dir, 'media', 'screenrecording.gif')
await overwriteSymlink(SCREENRECORDGIF_PATH(page), legacy_path, {relative: snap_dir, search_limit: snap_dir})
} catch(err) {
console.log('[❌] Failed to convert video to GIF:', err)
}
}
return SCREENRECORDING_PATH(page)
}
return null
}
async function saveScreenshot(page, _page_state, {aspect_ratio=SCREENSHOT_ASPECT_RATIO, width=null, height=null, jpg_width=1440, jpg_quality=90, timeout=30_000}={}) {
try {await fs.promises.unlink(SCREENSHOT_PATH(page))} catch(err) {}
// setup width and height
width = width || DEFAULT_VIEWPORT.width
assert((typeof width === 'number') && width > 200)
height = height || Math.floor(width/aspect_ratio)
assert((typeof height === 'number') && height > 200)
console.log(`[📸] Saving full-page screenshot (${width}x${height}px)...`.padEnd(82), prettyPath(SCREENSHOT_PATH(page)))
// set width, height, and deviceScale factor: https://github.com/puppeteer/puppeteer/issues/1576
await page.setViewport({ ...DEFAULT_VIEWPORT, width, height, deviceScaleFactor: 2})
await page.bringToFront()
await wait(1_250) // page takes a sec settle after foregrounding and viewport update
// take lossless fullpage screenshot of 1920x1440+px (4:3+) -> ./screenshot.png
await page.screenshot({ path: SCREENSHOT_PATH(page), fullPage: true, type: 'png' })
// wait for the screenshot to be created, then set the viewport to the next size
await blockUntilExists(SCREENSHOT_PATH(page), {min_bytes: 100, timeout})
await wait(6_000) // puppeteer takes a while to finish writing png data when fullPage: true
const jpg_height = Math.floor(jpg_width/aspect_ratio)
await page.setViewport({ ...DEFAULT_VIEWPORT, width: jpg_width, height: jpg_height, deviceScaleFactor: 2})
await wait(1_250) // page takes a sec settle after foregrounding and viewport update
// WARNING: make sure you never try to create two screenshots at the same time (especially not fullpage screenshots)
// thats why there are all these delays here.
// screenshot creation messes up the whole viewport while it's running,
// and it writes bad/white empty screenshots if you try to make more than one concurrently
// take compressed screenshot of jpg_width*jpg_height (4:3) -> ./screenshot.jpg
await page.screenshot({
path: SCREENSHOT_JPG_PATH(page),
type: 'jpeg',
quality: jpg_quality,
clip: {
x: 0,
y: 0,
width: jpg_width,
height: jpg_height,
},
captureBeyondViewport: false,
});
await blockUntilExists(SCREENSHOT_JPG_PATH(page), {min_bytes: 100, timeout: timeout/2})
console.log(`[📸] Saved screenshot as screenshot.jpg (${jpg_width}x${jpg_height}px)...`.padEnd(82), prettyPath(SCREENSHOT_JPG_PATH(page)))
// reset viewport back to defaults
await wait(1_250)
await page.setViewport(DEFAULT_VIEWPORT)
// ALTERNATIVE METHOD based on cropping fullpage png and converting to jpg manually:
// import {PNG} from 'pngjs';
// import jpeg from 'jpeg-js';
// setTimeout(async () => {
// try {
// const screenshot_png = SCREENSHOT_PATH(page);
// const screenshot_jpg = SCREENSHOT_JPG_PATH(page)
// const jpg_max_height = height
// const jpg_quality = quality; // Adjust the quality as needed (0-100)
// fs.createReadStream(screenshot_png)
// .pipe(new PNG())
// .on('parsed', function () {
// const width = this.width;
// const height = this.height;
// let cropped_height = height;
// if (height > jpg_max_height) {
// cropped_height = jpg_max_height;
// }
// const cropped_bytes = new Uint8Array(width * cropped_height * 4);
// for (let y = 0; y < cropped_height; y++) {
// for (let x = 0; x < width; x++) {
// const idx = (width * y + x) << 2;
// cropped_bytes[idx] = this.data[idx];
// cropped_bytes[idx + 1] = this.data[idx + 1];
// cropped_bytes[idx + 2] = this.data[idx + 2];
// cropped_bytes[idx + 3] = this.data[idx + 3];
// }
// }
// const jpeg_obj = {
// data: cropped_bytes,
// width: width,
// height: cropped_height,
// };
// const jpeg_bytes = jpeg.encode(jpeg_obj, jpg_quality);
// fs.writeFileSync(screenshot_jpg, jpeg_bytes.data);
// console.log(`[📸] Saved screenshot as screenshot.jpg (${width}x${jpg_max_height}px)...`.padEnd(82), prettyPath(SCREENSHOT_JPG_PATH(page)))
// });
// } catch(err) {
// console.error('[X] Error while generating JPG screenshot', SCREENSHOT_JPG_PATH(page), err)
// }
// }, DELAY_BEFORE_JPG_CONVERSION)
// ALTERNATIVE METHOD TO WRITE SCREENSHOT JPG:
// await wait(5_000) // puppeteer takes a while to finish writing png data when fullPage: true
// if ((await page.evaluate('document.body.scrollHeight')) > max_height) {
// // if page exceeds max_height, save additional cropped screenshot as screenshot.top.png
// // (needed b.c. uncropped screenshot may have insane 1:20+ aspect ratio that is hard to use elsewhere)
// await page.screenshot({ path: SCREENSHOT_JPG_PATH(page), type: 'jpg', quality: 100})
// await wait(1_000) // page takes a sec settle after a screenshot
// }
return SCREENSHOT_PATH(page)
}
async function savePDF(page, _page_state, {timeout=30_000}={}) {
const url = page.url() || 'about:blank'
if (URL_SCHEMES_IGNORED.includes(url.split(':')[0])) return null
const out_path = PDF_PATH(page)
console.log(`[📓] Saving print-as-PDF export...`.padEnd(82), prettyPath(out_path))
await page.bringToFront()
try {await fs.promises.unlink(PDF_PATH(page))} catch(err) {}
// await page.emulateMediaType('screen') // print as "@media(screen) instead of @media(print)"
// page.createPDFStream lets us to save larger PDFs than page.pdf() before crashing
// (streams to disk in chunks instead of all at once)
const pdf_stream = await page.createPDFStream({
timeout: timeout,
printBackground: true,
outline: true,
tagged: true,
format: 'A4',
displayHeaderFooter: false,
// margin: { top: '0.5cm', right: '1cm', bottom: '0.8cm', left: '1cm' },
})
const reader = pdf_stream.getReader()
// iterate through reader and append chunks to out_path
await fs.promises.rm(out_path, {force: true})
let num_bytes = 0
let error = '0 bytes written'
try {
while (true) {
const {done, value} = await reader.read()
if (done) break;
await fs.promises.appendFile(out_path, value)
num_bytes += value.length;
}
} catch(error) {
num_bytes = 0
}
if (!num_bytes) {
console.warn('[❌] Failed to save PDF', JSON.stringify(error, null, 4))
await fs.promises.rm(out_path, {force: true})
return null
}
return out_path
}
async function inlineShadowDOM(page, _page_state, {limit=100_000}={}) {
console.log(`[😎] Replacing Shadow DOM elements with inline HTML...`)
try {
const num_replaced = await page.evaluate((limit) => {
let num_replaced = 0
// Returns HTML of given shadow DOM.
const getShadowDomHtml = (shadowRoot) => {
let shadowHTML = '';
for (const el of shadowRoot.childNodes) {
shadowHTML += el.nodeValue || el.outerHTML;
}
return shadowHTML;
};
// Recursively replaces shadow DOMs with their HTML.
const replaceShadowDomsWithHtml = (rootElement) => {
if (num_replaced > limit) return
for (const el of rootElement.querySelectorAll('*')) {
if (el.shadowRoot) {
replaceShadowDomsWithHtml(el.shadowRoot);
el.innerHTML += getShadowDomHtml(el.shadowRoot);
}
}
num_replaced++
};
replaceShadowDomsWithHtml(document.body);
return num_replaced
}, limit)
// console.log(' √ replaced', num_replaced, 'Shadow DOM trees')
} catch(err) {
console.log('[⚠️] Inlining Shadow DOM failed', err)
}
}
async function saveAIQualityAssuranceResult(page, {original_url, version}) {
console.log(`[🧠] Analyzing screenshot with GPT-4o for QA checks...`.padEnd(82), prettyPath(AIQA_PATH(page)))
let screenshot_path = SCREENSHOT_PATH(page)
const screenshot_cropped_path = SCREENSHOT_JPG_PATH(page)
if (fs.existsSync(screenshot_cropped_path)) {
// screenshot is too tall to pass to openai, send cropped version instead
screenshot_path = screenshot_cropped_path
}
try {
await blockUntilExists(screenshot_path, {min_bytes: 100, timeout: 7_500})
} catch (err) {
console.warn('[❌] Failed to send screenshot to GTP-4o for analysis, no screenshot.{png,jpg} exists', err)
return null
}
var stdout = ''
var stderr = ''
let result = null
const PYTHON_BIN = path.join(__dirname, '.venv/bin/python')
const SCRIPT_PATH = path.join(__dirname, 'ai_qa.py')
await blockUntilExists(PYTHON_BIN, {min_bytes: 1, timeout: 250})
await blockUntilExists(SCRIPT_PATH, {min_bytes: 1, timeout: 250})
try {
var {stdout, stderr} = await exec(
`${PYTHON_BIN} ${SCRIPT_PATH} --attach '${screenshot_path}'`
)
result = JSON.parse(stdout.toString())
if (!result) throw 'Got empty result!'
result = {
TYPE: 'aiqa',
VERSION: version,
URL: original_url,
...result,
}
} catch(parse_err) {
console.warn('[❌] Failed to get OpenAI analysis for screenshot.png', parse_err, stderr)
}
if (!(result || stdout)) {
return null
}
await overwriteFile(
AIQA_PATH(page),
result || stdout.toString(),
)
return result
}
async function saveYTDLP(page, {original_url, version}, {max_size='750m'}={}) {
console.log(`[🎥] Saving media with YT-DLP (<=${max_size})...`.padEnd(82), prettyPath(YTDLP_PATH(page)))
await fs.promises.mkdir(YTDLP_PATH(page), {recursive: true})
const cwd = YTDLP_PATH(page)
const bin_name = 'yt-dlp'
const timeout = 300_000 // 5min timeout
const args = [
'--restrict-filenames',
'--trim-filenames', '128',
'--write-description',
'--write-info-json',
'--write-annotations',
'--write-thumbnail',
'--no-call-home',
'--write-sub',
'--write-auto-subs',
'--convert-subs=srt',
'--yes-playlist',
'--continue',
'--no-abort-on-error',
'--ignore-errors',
'--geo-bypass',
'--add-metadata',
`--format=(bv*+ba/b)[filesize<=${max_size}][filesize_approx<=?${max_size}]/(bv*+ba/b)`,
'--no-check-certificate',
'--no-progress',
// `--cookies=${COOKIES_TXT_PATH}`, // using logged in cookies actually makes it fail more often, not sure why
original_url,
]
const {getResult, ...exec_info} = await saveExecResult(bin_name, args, {original_url, version}, {cwd, timeout})
return {getResult, ...exec_info}
}
async function saveGALLERYDL(page, {original_url, version}) {
console.log(`[🎥] Saving photos with gallery-dl...`.padEnd(82), prettyPath(GALLERYDL_PATH(page)))
await fs.promises.mkdir(GALLERYDL_PATH(page), {recursive: true})
const cwd = GALLERYDL_PATH(page)
const bin_name = 'gallery-dl'
const timeout = 300_000 // 5min timeout
const args = [
'--verbose',
'--write-metadata',
'--write-infojson',
'--write-tags',
'--sleep=1.5-2.5',
`--cookies=${COOKIES_TXT_PATH}`,
// '--no-check-certificate',
// `--directory=media`,
original_url,
]
const {getResult, ...exec_info} = await saveExecResult(bin_name, args, {original_url, version}, {cwd, timeout})
return {getResult, ...exec_info}
}
// async function saveWget(page, {original_url, version}) {
// console.log(`[⎒] Saving wget site clone...`.padEnd(82), prettyPath(WGET_PATH(page)))
// const args = [
// // ...
// ]
// spawn(
// 'wget',
// [
// ...args,
// original_url,
// ],
// {
// cwd: WGET_PATH(page),
// detached: true, // run in background, don't block on response
// stdio: 'ignore',
// timeout: 300_000, // 5min timeout
// },
// )
// return {path: WGET_PATH(page)}
// }
/**************** Asynchronous Archive Output Tasks ***************************/
type FaviconCandidate = {
url: string,
basename: string,
extension: string,
expected_mimetype: string,
}
const faviconFromDomain = (url) => {
// https://auth:pass@t.co:1234/a/bc123 -> https://auth:pass@t.co:1234/favicon.ico
const url_origin = (new URL(url)).origin
return {
url: url_origin ? `${url_origin}/favicon.ico` : null,
basename: 'favicon',
extension: undefined, // auto-detect extension at download time in case it redirects us to a png
expected_mimetype: 'image/', // only accept image/* to avoid saving html/txt error reponses as icon
} as FaviconCandidate
}
const faviconFromGoogle = (url, size=256) => {
// https://auth:pass@t.co:1234/a/bc123 -> https://www.google.com/s2.favicons?domain=t.co
const domain = url && (new URL(url)).hostname
return {
url: domain?.includes('.') ? `https://www.google.com/s2/favicons?sz=${size},domain=${domain}` : null,
basename: 'google_favicon',
extension: 'png',
expected_mimetype: 'image/png', // google always provides PNGs in response
} as FaviconCandidate
}
const faviconFromHtml = async (page) => {
// <link rel="icon" src="https://example.com/static/images/favicon.png"/> -> https://example.com/static/images/favicon.png
let url
try {
url = await page.$eval('link[rel*="icon"]', (elem) => elem?.href)
if (!url || !url.includes('://'))
url = null
} catch(err) {
url = null
// console.warn('Failed to find favicon tag in html', JSON.stringify(err, null, 4))
}
return {
url,
basename: 'favicon',
extension: undefined, // auto-detect extension at download time
expected_mimetype: 'image/', // accept any image/* mimetype at download time
} as FaviconCandidate
}
type FaviconResult = {
url: string,
num_bytes: number,
abspath?: string,
dir?: string,
filename?: string,
mimeType?: string,
}
async function saveFavicon(page, {original_url, main_response, version}) {
const dir = path.dirname(FAVICON_PATH(page))
const response_url = main_response?.url()
const favicon_downloads_to_try: {[key: string]: FaviconCandidate} = unique([
await faviconFromHtml(page),
faviconFromDomain(response_url),
faviconFromDomain(original_url),
faviconFromGoogle(response_url),
faviconFromGoogle(original_url),
].filter(({url}) => url), 'url')
const browser = await page.browser()
// let logs = []
// let errors = []
let output_files: {[key: string]: FaviconResult} = {}
for (const download_options of Object.values(favicon_downloads_to_try)) {
let result: FaviconResult = {num_bytes: 0, url: download_options.url}
// {url, num_bytes, abspath, dir, filename, basename, extension, mimeType}
try {
// try getting it with node-fetch first
const response = await fetch(download_options.url) as Response
const file_options = await detectFilename({...download_options, response, dir})
if (response.headers.get("content-length")) {
const favicon_stream = Readable.fromWeb(response.body as any)
await overwriteFile(file_options.abspath, favicon_stream)
result = {
...file_options,
num_bytes: parseInt(response.headers.get("content-length") || '0'),
mimeType: response.headers.get("content-type"),
}
} else {
throw 'Failed to download favicon with fetch()'
}
} catch(err) {
// console.warn('[!] Failed to get favicon with node-fetch', err)
// fallback to getting it by opening a new browser tab
result = await download({...download_options, browser, dir, page})
}
// logs.push(...(result.logs || []))
// errors.push(...(result.errors || []))
if (result.num_bytes) {
console.log(`[🌠] Saving page favicon (${result.url.substring(0, 35)}... ${result.mimeType})...`.padEnd(82), prettyPath(result.abspath))
output_files[result.filename] = result
break // break here stops after the first successful download, comment out to keep going instead
}
}
const output_file = Object.values(output_files).sort(file => file.num_bytes).at(-1)
const favicon_info = {
TYPE: 'favicon',
VERSION: version,
URL: original_url,
succeeded: !!output_file,
// stdout: JSON.stringify(logs),
// stderr: JSON.stringify(errors),
favicon_url: output_file?.url,
favicon_urls: Object.keys(favicon_downloads_to_try),
favicon_files: Object.keys(output_files).map(fname => fname.replace(dir, '.')),
favicon_filename: output_file?.filename,
favicon_num_bytes: output_file?.num_bytes,
}
await overwriteFile(FAVICON_PATH(page), favicon_info)
return favicon_info
}
async function saveTitle(page, {original_url, version}) {
const title_from_browser = (await page.title()) || null
const title_from_js = await page.evaluate(() => document?.title || null)
const title_from_html = await page.evaluate(() => document?.querySelector('title')?.innerText || null)
const title_from_og = await page.evaluate(() => document?.querySelector('meta[property="og:title"]')?.getAttribute('content') || null)
// best guess at best title = longest title
const title = ([title_from_html, title_from_og, title_from_js, title_from_browser]
.filter(title => title)
.sort((a, b) => b.length - a.length)[0] || '')
.replaceAll('\n', ' ')
if (title?.length) {
console.log(`[📗] Saving page title (${title.substring(0, 40)})...`.padEnd(82), prettyPath(TITLE_PATH(page)))
await overwriteFile(TITLE_PATH(page), title)
}
const title_info = {
TYPE: 'title',
VERSION: version,
URL: original_url,
title,
title_from_html,
title_from_og,
title_from_js,
title_from_browser,
}
const title_json_path = TITLE_PATH(page).replace('.txt', '.json')
await overwriteFile(title_json_path, title_info)
return title_info
}
async function saveRaw(page, {main_response}) {
const response = main_response
if (!response) {
console.warn('[⚠️] Failed to save page RAW bytes, main_response is null', response)
}
const dir = RAW_PATH(page)
await fs.promises.mkdir(dir, {recursive: true})
const {url, abspath, mimeType} = await detectFilename({page, response, dir})
console.log(`[🔟] Saving raw response bytes (${mimeType})...`.padEnd(82), prettyPath(abspath))
await download({page, response, abspath})
return abspath
}
async function saveSourceMaps(page, {original_url, version}) {
console.log(`[🐛] Saving source maps to ./responses/all/*.{js,css}.map...`)
const response_index_path = path.join(RESPONSES_PATH(page), 'index.jsonl')
const response_index = await fs.promises.readFile(response_index_path, 'utf-8')
const urls_to_download = []
for (const response of response_index.split('\n')) {
try {
const {url, extension} = JSON.parse(response)
if (['css', 'js'].includes(extension?.toLowerCase())) {
urls_to_download.push(url + '.map')
}
} catch(err) { continue }
}
// TODO: fix this, it needs to both after stopSavingMetadata and before stopSavingMetadata
// fix is to use traffic_log to get response url list instead of waiting for index.jsonl to be created
await page.evaluate(async (urls_to_download) => {
const promises = []
for (const sourcemap_url in urls_to_download) {
promises.push(fetch(sourcemap_url))
}
return Promise.allSettled(promises)
}, urls_to_download)
return {
TYPE: 'sourcemaps',
URL: original_url,
VERSION: version,
sourcemaps: urls_to_download,
}
}
async function saveRequests(page, {original_url, version, traffic_log}) {
console.log(`[📼] Saving requests log (${Object.keys(traffic_log).length})...`.padEnd(82), prettyPath(REQUESTS_PATH(page)))
const requests_info = {
TYPE: 'requests',
VERSION: version,
URL: original_url,
requests: traffic_log,
}
await overwriteFile(REQUESTS_PATH(page), requests_info)
return requests_info
}
async function saveRedirects(page, {original_url, main_response, traffic_log, redirects, version}) {
const main_request_id = Object.keys(traffic_log).filter(id => !id.includes('.'))[0]
const main_response_traffic = traffic_log[main_request_id] || {}
const url_from_browser = await page.url() || null
const url_from_request = (
main_response?.request()?.url()
|| main_response_traffic['Network.requestWillBeSent']?.request?.url
|| null)
const url_from_response = (
main_response?.url()
|| main_response_traffic['Network.responseReceived']?.main_response?.url
|| null)
const http_redirects =
Object.values(traffic_log)
.filter(event => event['Network.requestWillBeSent']?.redirectResponse)
.map(event => event['Network.requestWillBeSent'])
.map(requestWillBeSent => ({
url: requestWillBeSent.request.url,
src: requestWillBeSent.redirectResponse.url,
status: requestWillBeSent.redirectResponse.status,
loaderId: requestWillBeSent.loaderId,
requestId: requestWillBeSent.requestId,
wallTime: requestWillBeSent.wallTime,
initiator: requestWillBeSent.initiator,
isMainFrame: (requestWillBeSent.loaderId == main_request_id),
}))
const url_parsed = new URL(url_from_response || url_from_request || url_from_browser)
const redirects_info = {
TYPE: 'redirects',
VERSION: version,
URL: original_url,
url_parsed,
url_from_request,
url_from_response,
url_from_browser,
redirects_from_browser: redirects,
redirects_from_http: http_redirects,
}
console.log(`[🔗] Saving page redirects log (${http_redirects.length})...`.padEnd(82), prettyPath(REDIRECTS_PATH(page)))
await overwriteFile(REDIRECTS_PATH(page), redirects_info)
return redirects_info
}
async function saveHeaders(page, {original_url, version, traffic_log}) {
const main_request_id = Object.keys(traffic_log).filter(id => !id.includes('.'))[0]
const main_response_traffic = traffic_log[main_request_id] || {}
// combine base request with browser-added request headers
const request = {...main_response_traffic['Network.requestWillBeSent']?.request}
const request_extra_headers = main_response_traffic['Network.requestWillBeSentExtraInfo']?.headers || {}
request.headers = {...request.headers, ...request_extra_headers}
// combine base response with browser-added response headers
const response = {...main_response_traffic['Network.responseReceived']?.response}
const response_extra_headers = main_response_traffic['Network.responseReceivedExtraInfo']?.headers || {}
response.headers = {...response.headers, ...response_extra_headers}
const headers_info = {
TYPE: 'headers',
VERSION: version,
URL: original_url,
request,
response,
}
const num_headers = Object.keys({...request.headers, ...response.headers}).length
if (num_headers) {
console.log(`[👾] Saving main request & response headers (${num_headers})...`.padEnd(82), prettyPath(HEADERS_PATH(page)))
await overwriteFile(HEADERS_PATH(page), headers_info)
}
return headers_info
}
async function saveSSL(page, {original_url, version, traffic_log}) {
const main_request_id = Object.keys(traffic_log).filter(id => !id.includes('.'))[0]
const main_response_traffic = traffic_log[main_request_id] || {}
const relevant_response_keys = [
'url',
'status',
'mimeType',
'connectionReused',
'remoteIPAddress',
'remotePort',
'fromServiceWorker',
'encodedDataLength',
'protocol',
'alternateProtocolUsage',
'securityState',
'securityDetails',
]
let ssl_info = Object.entries(main_response_traffic['Network.responseReceived']?.response || {})
.reduce((obj, [key, val]) => {
if (relevant_response_keys.includes(key)) {
obj[key] = val
}
return obj
}, {}) as any
// TODO: parse SSL certificate sha256 hash from chrome://system/#chrome_root_store
// const ssl_certificate = await client.send('Network.getCertificate', {origin: original_url})
// ssl_info.sslCertSha256 = '<unknown>'
ssl_info = {
TYPE: 'ssl',
VERSION: version,
URL: original_url,
...ssl_info,
}
if (Object.keys(ssl_info).length-3) {
console.log(`[🔏] Saving page SSL details (${ssl_info?.securityDetails?.protocol})...`.padEnd(82), prettyPath(SSL_PATH(page)))
await overwriteFile(SSL_PATH(page), ssl_info)
}
return ssl_info
}
async function saveDOM(page, {original_url, version}) {
const html = await page.content();
console.log(`[📖] Saving DOM dump (${html.length})...`.padEnd(82), prettyPath(DOM_PATH(page)))
const html_with_header =
`<!-- Saved by ArchiveBox TYPE=dom VERSION=${version} URL=${original_url} -->\n${html}`
await overwriteFile(DOM_PATH(page), html_with_header)
return DOM_PATH(page)
}
async function saveBodyText(page, _page_state) {
const innerText = await page.evaluate(() => document?.body?.innerText);
if (innerText?.length) {
console.log(`[📃] Saving body text (${innerText.length})...`.padEnd(82), prettyPath(BODYTEXT_PATH(page)))
await overwriteFile(BODYTEXT_PATH(page), innerText)
}
// // alternative method: emulate Ctrl+A, Ctrl+C (sometimes gets more than body.innerText)
// const innerText = await page.$eval('*', (el) => {
// const selection = window.getSelection();
// const range = document.createRange();
// range.selectNode(el);
// selection.removeAllRanges();
// selection.addRange(range);
// return window.getSelection().toString();
// });
return innerText
}
async function savePandoc(page, { original_url, version }) {
console.log(`[📒] Converting DOM HTML to markdown with Pandoc...`.padEnd(82), prettyPath(PANDOC_PATH(page)))
let dom_paths = [DOM_PATH(page), SINGLEFILE_PATH(page)].filter(fs.existsSync)
if (!dom_paths) return null
const dom_path = dom_paths[0]
var stdout: string = ''
var stderr: string = ''
let result: any = null
const BIN_NAME = 'pandoc'
// pandoc --from html --to markdown_github --citeproc --wrap=none --highlight-style=kate
const args = [
BIN_NAME,
'--from=html',
'--to=markdown_github',
'--wrap=none',
'--citeproc',
'--highlight-style=kate',
`--output='${PANDOC_PATH(page)}'`,
dom_path,
]
try {
;({ stdout, stderr } = await exec(args.join(' ')));
stdout = stdout.toString().trim()
if (!stdout) throw 'Got empty result!'
result = {
TYPE: 'pandoc',
VERSION: version,
URL: original_url,
cmd: args,
markdown_file: PANDOC_PATH(page),
}
} catch (parse_err) {
console.warn('[❌] Failed to run Pandoc HTML to MD conversion', parse_err, stderr)
}
if (!stdout) {return null}
await overwriteFile(
PANDOC_PATH(page),
stdout,
)
// pandoc --from markdown_github --to html --citeproc --wrap=none --highlight-style=kate
const reverse_conversion_args = [
BIN_NAME,
'--from=markdown_github',
'--to=html',
'--wrap=none',
'--citeproc',
'--highlight-style=kate',
`--output='${PANDOC_PATH(page).replace('.md', '.html')}'`,
PANDOC_PATH(page),
]
try {
; ({ stdout, stderr } = await exec(reverse_conversion_args.join(' ')));
stdout = stdout.toString().trim()
if (!stdout) throw 'Got empty result!'
result = {
...result,
html_file: PANDOC_PATH(page).replace('.md', '.html'),
}
} catch (parse_err) {
console.warn('[❌] Failed to run Pandoc MD to HTML conversion', parse_err, stderr)
}
if (!result) { return null }
await overwriteFile(
PANDOC_PATH(page).replace('.md', '.html'),
result,
)
return result
}
async function saveReadability(page, {original_url, version}) {
const url = await page.url()
let html = ''
let article = null
try {
html = await page.content()
if (html.length > 14_000_000) {
console.warn('[⚠️] Truncating readability article text because html is too long...', html.length)
html = html.substring(0, 13_900_000)
}
const virtualConsole = new VirtualConsole()
const dom = new JSDOM(html, {url, virtualConsole})
const reader = new Readability(dom.window.document);
article = reader.parse()
} catch(err) {
console.warn(`[❌] Failed to get readability article text`)
return null
}
if (article) {
console.log(`[📜] Saving readability article text (${article.textContent?.length})...`.padEnd(82), prettyPath(READABILITY_PATH(page)))
const {content, textContent, ...metadata} = article
if (content.trim()) {
await overwriteFile(READABILITY_PATH(page).replace('.json', '.html'), content);
}
if (textContent.trim()) {
await overwriteFile(READABILITY_PATH(page).replace('.json', '.txt'), textContent);
}
const readability_info = {
TYPE: 'readability',
VERSION: version,
URL: original_url,
...metadata,
}
await overwriteFile(READABILITY_PATH(page), readability_info)
return readability_info
}
return null
}
async function saveAccessibility(page, {original_url, version}) {
// get accessibility tree
const accessibility_tree = await page.accessibility.snapshot({interestingOnly: true});
// console.log(accessibility_tree);
// get iframe tree
const iframes = []
function dumpFrameTree(frame, indent='>') {
iframes.push(indent + frame.url());
for (const child of frame.childFrames()) {
dumpFrameTree(child, indent + '>');
}
}
dumpFrameTree(page.mainFrame(), '');
// console.log(iframes)
// generate simple table-of-contents of all the key html elements (e.g. h1, h2, h3, article, main, etc.)
const outline = await page.evaluate(() => {
const headings = []
for (const elem of [...document.querySelectorAll("h1, h2, h3, h4, h5, h6, a, header, footer, article, main, aside, nav, section, figure, summary, table, form, iframe")] as HTMLElement[]) {
// skip a tags that aren't named anchors
if (elem.tagName.toLowerCase() == 'a' && !(elem as HTMLAnchorElement).name) continue
// e.g. article #main-article
const elem_id = ((typeof elem.id === 'string' && elem.id) || (elem as HTMLAnchorElement).name || elem.ariaLabel || elem.role || '')
const elem_classes = elem.className.trim().split(' ').slice(0, 3).join(' .') || ''
const elem_action = (elem as any).action?.split('/')?.slice(-1)?.join('/')
const summary = elem.innerText.length > 128
? `${elem.innerText?.slice(0, 128)}...`
: elem.innerText
let prefix = ''
let title = (elem_id ? `#${elem_id}` : '')
if (!title && elem_classes) title = `.${elem_classes}`
if (elem_action) title = `${title} /${elem_action}`
if (summary) title = `${title}: ${summary}`
// if elem is a header, prepend a #### prefix based on its level
const level = Number(elem.tagName.toLowerCase().replace('h', ''))
if (!isNaN(level)) {
prefix = '#'.repeat(level)
title = elem.innerText || elem_id || elem_classes
} else {
// set prefix to element's breadcrumb path
let node = elem
const parents = [elem.tagName?.toLowerCase().trim()]
while (node) {
// add each parent element's name to the path
// const elem_type = node.tagName?.toLowerCase().trim() || ''
// if (elem_type && !['div', 'span', 'p', 'body', 'html'].includes(elem_type)) {
// parents.unshift(elem_type);
// }
parents.unshift('') // add emptystring to abbreviate path as >>>> istead of main>article>header>div>...
node = node.parentNode as HTMLElement
}
prefix = parents.join('>')
}
// strip all repeated whitespace and newlines
title = title.replaceAll('\n', ' ').replace(/\s+/g, ' ').trim()
if (prefix) {
headings.push(`${prefix} ${title}`)
}
}
// console.log(headings.join('\n'))
return headings
})
console.log(`[🩼] Saving accessibility outline (${Object.keys(accessibility_tree).length})...`.padEnd(82), prettyPath(ACCESIBILITY_PATH(page)))
// console.log(outline.filter(line => line.startsWith('#')).join('\n'))
const accessibility_info = {
TYPE: 'accessibility',
VERSION: version,
URL: original_url,
iframes,
headings: outline,
tree: accessibility_tree,
}
await overwriteFile(
ACCESIBILITY_PATH(page),
accessibility_info,
)
return accessibility_info
}
async function saveSEO(page, {original_url, version}) {
// collect all <meta name="title" property="og:title" content="Page Title for SEO | Somesite.com"> tags into dict
const seo_vars = await page.evaluate(() =>
[...document.querySelectorAll('meta')]
.map(tag => ({key: tag.getAttribute('name') || tag.getAttribute('property') || '', value: tag.getAttribute('content') || ''}))
.filter(obj => obj.key && obj.value)
.sort((a, b) => a.value.length - b.value.length)
.reduce((acc, node) => {acc[node.key] = node.value; return acc}, {})
)
const seo_info = {
TYPE: 'seo',
VERSION: version,
URL: original_url,
...seo_vars,
}
const num_vars = Object.keys(seo_vars).length
if (num_vars) {
console.log(`[🔎] Saving page SEO metadata (${num_vars})...`.padEnd(82), prettyPath(SEO_PATH(page)))
await overwriteFile(SEO_PATH(page), seo_info)
}
return seo_info
}
async function saveOutlinks(page, {original_url, version}) {
// TODO: slow to iterate over all elements so many times, perhaps we can collapse everything down into one loop
// Regular expression that matches syntax for a link (https://stackoverflow.com/a/3809435/117030):
const LINK_REGEX = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/gi;
const filterW3Urls = (urls) =>
urls.filter(url =>
url && !url.startsWith('http://www.w3.org/'))
const filterDataUrls = (urls) =>
urls.filter(url =>
url && !url.startsWith('data:'))
const html = await page.content();
const raw = html?.match(LINK_REGEX) || [];
const hrefs = await page.$$eval(
"pierce/a[href]",
elems => elems
.map(elem => elem.href)
.filter(url => url),
);
const links = await page.$$eval(
"pierce/link[href]",
elems => elems
.map(({rel, href}) => ({rel, href}))
.filter(({rel, href}) => rel !== 'stylesheet')
.reduce((collection, entry) => {
const {rel, href} = entry
const non_empty_rel = collection[href]?.rel || rel
collection[href] = {rel: non_empty_rel, href}
return collection
}, {})
);
const iframes = await page.$$eval(
"pierce/iframe[src]",
elems => elems.map(iframe => iframe.src).filter(url => url)
);
const images = await page.$$eval(
"pierce/img[src]",
elems => elems.map(img => img.src).filter(url => url && !url.startsWith('data:'))
);
const css_images = await page.$$eval(
"pierce/*",
elems => elems
.map(elem => {
const css_url_ptn = /url\(\s*?['"]?\s*?(\S+?)\s*?["']?\s*?\)/i;
const bg_img = window.getComputedStyle(elem, null).getPropertyValue('background-image')
const bg_url = css_url_ptn.exec(bg_img)
return bg_url ? bg_url[1] : null
})
)
const css_stylesheets = await page.$$eval(
"pierce/link[rel=stylesheet]",
elems => elems.map(elem => elem.href).filter(url => url)
);
const js_scripts = await page.$$eval(
"pierce/script[src]",
elems => elems.map(elem => elem.src).filter(url => url)
);
const outlinks_info = {
TYPE: 'outlinks',
VERSION: version,
URL: original_url,
raw: [...new Set(filterDataUrls(filterW3Urls(raw)))],
hrefs: [...new Set(filterDataUrls(hrefs))],
links: [...Object.values(links)],
iframes: [...new Set(iframes)],
images: [...new Set(filterDataUrls(images))],
css_images: [...new Set(filterDataUrls(css_images))],
css_stylesheets: [...new Set(filterDataUrls(css_stylesheets))],
js_scripts: [...new Set(filterDataUrls(js_scripts))],
}
if (raw?.length || hrefs?.length || links?.length || iframes?.length) {
console.log(`[🖇️] Saving page outgoing links (${raw?.length || hrefs?.length})...`.padEnd(82+1), prettyPath(OUTLINKS_PATH(page)))
await overwriteFile(OUTLINKS_PATH(page), outlinks_info)
}
return outlinks_info
}
async function saveAuthStorage(page, {client, version, original_url}) {
const url = original_url || await page.url()
if (URL_SCHEMES_IGNORED.includes(url.split(':')[0])) return null
if (!SAVE_AUTH_STORAGE) return null
// const cookies = JSON.stringify(await page.cookies()); // doesnt include httponly cookies
const auth_from_browser = {
cookies: (await client.send('Network.getAllCookies')).cookies,
localStorage: {},
sessionStorage: {},
}
// attempt to load localStorage and sessionStorage from browser (may fail in some cases https://github.com/puppeteer/puppeteer/issues/921)
try {
auth_from_browser.localStorage = (await page.evaluate(() =>
JSON.parse(JSON.stringify({[window.location.origin]: window.localStorage}))))
} catch(err) {
throw `Failed to get page window.localStorage! ${err}`
}
try {
auth_from_browser.sessionStorage = (await page.evaluate(() =>
JSON.parse(JSON.stringify({[window.location.origin]: window.sessionStorage}))))
} catch(err) {
throw `Failed to get page window.sessionStorage! ${err}`
}
// WARNING: small TOCTTOU gap between this read-before-write and the write below
// can possibly overwrite changes made by other processes in this gap
const auth_on_disk = await loadAuthStorage(page, {client}, {apply: false})
const cookies = dedupeCookies([...auth_on_disk.cookies, ...auth_from_browser.cookies])
const auth_info = {
TYPE: 'auth',
VERSION: version,
URL: original_url,
cookies: cookies,
sessionStorage: merge(auth_on_disk.sessionStorage, auth_from_browser.sessionStorage),
localStorage: merge(auth_on_disk.localStorage, auth_from_browser.localStorage),
}
// console.log(`[⛙] Merged ${auth_on_disk.cookies.length} existing + ${auth_from_browser.cookies.length} new -> ${auth_info.cookies.length} cookies`)
console.log(`[🍪] Saving cookies/localStorage/sessionStorage (${auth_info.cookies.length})...`.padEnd(82), prettyPath(AUTH_JSON_PATH));
await overwriteFile(AUTH_JSON_PATH, auth_info);
// Write to cookies.txt file using tough-cookie + @root/file-cookie-store
await saveCookiesTxt(cookies)
return auth_info
}
async function saveCookiesTxt(cookies) {
const cookies_store = new FileCookieStore(COOKIES_TXT_PATH, {auto_sync: false, lockfile: false})
const cookie_jar = new ToughCookie.CookieJar(cookies_store)
cookie_jar.setCookieAsync = util.promisify(cookie_jar.setCookie)
cookies_store.saveAsync = util.promisify(cookies_store.save)
for (const cookie of cookies) {
const cookie_for_tough = {
domain: cookie.domain,
path: cookie.path,
key: cookie.name,
value: cookie.value,
expires: (new Date(cookie.expires * 1000)).toISOString(),
hostOnly: cookie.domain.startsWith('.'),
secure: cookie.secure,
}
// console.log('COOKIE_FOR_TOUGH_TXT', cookie_for_tough)
const parsed_cookie = ToughCookie.Cookie.fromJSON(cookie_for_tough)
// console.log('COOKIE_FOR_TOUGH_TXT_TO_DUMP', parsed_cookie)
try {
// assemble a fake URL just to satisfy ToughCookieJar's requirement of having a URL at set time
let url = cookie.secure ? 'https://' : 'http://'
if (cookie.domain.startsWith('.')) {
url = url + cookie.domain.slice(1)
} else {
url = url + cookie.domain
}
if (cookie.sourcePort && ![80, 443].includes(cookie.sourcePort)) {
url = `${url}:${cookie.sourcePort}`
}
url = `${url}${cookie.path || ''}`
await cookie_jar.setCookieAsync(parsed_cookie, url, {ignoreError: true})
} catch(err) {
console.error('[❌] Failed to dump browser cookie for cookies.txt...', cookie_for_tough, '->', parsed_cookie, err)
}
}
console.log(`[🍪] Saving cookies TXT (${cookies.length})...`.padEnd(82), prettyPath(COOKIES_TXT_PATH));
await cookies_store.saveAsync()
}
async function saveMetrics(page, {original_url, version, start_time, start_ts, traffic_log, redirects}) {
const end_time = (new Date()).toISOString()
const end_ts = Date.now()
const metrics_info = {
TYPE: 'metrics',
VERSION: version,
URL: original_url,
...(await page.metrics()),
start_time,
start_ts,
end_time,
end_ts,
duration: (end_ts - start_ts),
num_requests: traffic_log.length,
num_redirects: Object.keys(redirects).length -1,
}
console.log(`[🏎️] Saving final summary + timing metrics...`.padEnd(82+1), prettyPath(METRICS_PATH(page)))
await overwriteFile(METRICS_PATH(page), metrics_info)
return metrics_info
}
/******************************************************************************/
/******************************************************************************/
/**************************** Utility Helpers *********************************/
function hashCode(str) {
// get a simple integer hash for a given string (based on java String#hashCode)
// useful only for throwaway nonces / easy deterministic random identifiers, not a replacement for sha256
let hash = 0;
for (let i=0; i<str.length; i++) {
hash = str.charCodeAt(i) + ((hash << 5) - hash);
}
return Math.abs(hash)
}
function unique(iter, key: string | ((any, number) => string)='id') {
// uniqueify an array of objects by a value within them, key can be name of attr or getter function
// > iter = [{id: 1}, {id: 2}, {id: 1}]
// > Object.entries(iter) = [
// [ '0', { id: 1 } ],
// [ '1', { id: 2 } ],
// [ '2', { id: 1 } ] ]
// > unique(iter, 'id') => {1: {id: 1}, 2: {id: 2}}
// > iter = {a1: {id: 1}, b2: {id: 2}, a3: {id: 1}}
// > Object.entries(iter) = [
// [ 'a1', { id: 1 } ],
// [ 'b2', { id: 2 } ],
// [ 'a3', { id: 1 } ]
// ]
// > unique(iter, 'id') => {1: {id: 1}, 2: {id: 2}}
const key_type = (typeof key)
if (!['function', 'string'].includes(key_type))
throw 'key must be either a string lookup key or a function (obj, idx) => return unique_id'
const key_func = (key_type === 'string')
? (entry_obj, idx) => entry_obj[(key as string)]
: (entry_obj, idx) => (key as Function)(entry_obj, idx) // otherwise key is a callback func
const seen = {}
for (const [idx, entry_obj] of Object.entries(iter)) {
const unique_id = key_func(entry_obj, idx)
if (seen[unique_id] === undefined) {
seen[unique_id] = entry_obj
}
}
return seen
}
const wait = (ms: number) => new Promise(res => {
if (ms > 10_000) {
console.debug(`[⏲️] Waiting ${Math.round(ms/1000)}s...`)
}
setTimeout(res, ms)
})
const TimeoutError = Symbol()
const withTimeout = (promise, ms) => {
// run a promise with a time limit, raises a TimeoutError if it fails
let timer
return Promise.race([
promise,
new Promise((_r, reject) =>
timer = setTimeout(reject, ms, TimeoutError)
),
]).finally(() => clearTimeout(timer))
}
const MAX_VALID_DATE = new Date('2150-01-01T00:00:00.000Z')
const MIN_VALID_DATE = new Date('2010-01-01T00:00:00.000Z')
const UNIX_EPOCH_DATE = new Date(0)
const validateDate = (date, {min=MIN_VALID_DATE, max=MAX_VALID_DATE, singleton=UNIX_EPOCH_DATE}={}) => {
assert((date instanceof Date), `Got invalid type for Date: ${typeof date} ${date} (expected Date)`)
assert(String(date) !== 'Invalid Date', `Got invalid value for Date: ${typeof date} ${date}`)
if (Number(date) === Number(singleton)) return date // epoch singleton is always valid
assert(date < max, `Got Date that was higher than MAX_VALID_DATE=${max}`)
assert(date > min, `Got Date that was lower than MIN_VALID_DATE=${min}`)
return date
}
const parseVersionDateStr = (yyyymmddtime) => {
// YYYYMMDDhhmmssxxx or YYYYMMDDhhmmss or YYYYMMDDhhmm or YYYYMMDD -> Date
const is_only_numbers = /^\d+$/.test(yyyymmddtime.replace('.', ''))
assert(is_only_numbers, `Non-numeric characters in YYYYMMDD date are not allowed: ${yyyymmddtime} (while trying YYYYMMDDhhmmssxxx format)`)
const num_digits = String(yyyymmddtime).split('.')[0].length
assert([17, 14, 12, 8].includes(num_digits), `Got invalid number of digits (${num_digits}) in YYYYMMDD date: ${yyyymmddtime} (while trying YYYYMMDDhhmmssxxx format)`)
const [_all, yyyy, mm, dd, hr, min, sec, ms] = /^(\d{4})(\d{2})(\d{2})(\d{2})?(\d{2})?(\d{2})?(\d{3})?$/.exec(yyyymmddtime)
assert(yyyy && mm && dd, `Could not find YYYYMMDD`)
const time_error_msg = `Detected YYYYMMDD[hhmm[ss[xxxx]]] but time segment is invalid ${hr}:${min || '__'}:${ms || '___'}`
if (ms) assert(hr && min && sec, time_error_msg)
if (sec) assert(hr && min, time_error_msg)
if (min) assert(hr, time_error_msg)
if (hr) assert (min, time_error_msg)
const iso_str = `${yyyy}-${mm}-${dd}T${hr || '00'}:${min || '00'}:${sec || '00'}.${ms || '00'}Z`
const parsed_date = new Date(iso_str)
return validateDate(parsed_date) // 1970-01-01T00:00:00.000Z (ISO format)
}
const parseTimestampDateStr = (timestamp) => {
// 1709724291000 or 1709724291000.000 or 1709724291 or 1709724291.000 -> Date
timestamp = String(timestamp)
const is_only_numbers = /^\d+$/.test(timestamp.replace('.', ''))
assert(is_only_numbers, `Got invalid characters in timstamp: ${timestamp} (while trying xxxxxxxxxxxxx format)`)
const num_digits = String(timestamp).split('.')[0].length
assert([13, 10, 1].includes(num_digits), `Got invalid number of digits (${num_digits}) in timestamp: ${timestamp} (while trying xxxxxxxxxxxxx format)`)
let parsed_date = null
if (num_digits === 13) {
parsed_date = new Date(Number(timestamp)) // 1709724291000 (unix timestamp w/ milliseconds)
} else if (num_digits === 10) {
parsed_date = new Date(Number(timestamp) * 1000) // 1709724291 (unix timestamp w/ seconds)
} else if (num_digits === 1) {
assert(String(timestamp) === '0', `Got invalid single-digit timestamp: ${timestamp} (while trying xxxxxxxxxxxxx format or 0 for UNIX epoch)`)
parsed_date = UNIX_EPOCH_DATE
}
return validateDate(parsed_date)
}
const parseISODateStr = (iso_str) => {
// 1970-01-01T00:00:00.000Z -> Date
const num_digits = String(iso_str).length
assert([24, 19, 16, 10].includes(num_digits), `Got invalid number of digits (${num_digits}) in ISO date: ${iso_str} (while trying 1970-01-01T00:00:00.000Z format)`)
const parsed_date = new Date(iso_str)
return validateDate(parsed_date)
}
const parseDate = (date) => {
// date === undefined => use today/now
// date === null => use unix epoch 0 aka 1970-01-01T00:00:00.000Z
// date *= YYYYMMDDHHMMSS => use a version date string (e.g. 20010131235958)
// date *= 1234567... => use a timestmap (e.g. 1709724291000)
// date *= 1970-01-01T... => use iso datetime (e.g. 1970-01-01T00:00:00.000Z)
// returns -> Date
if (date === undefined) {
return (new Date()) // today (2024-05-29T22:02:34.682Z) aka timestamp=1717020154682
}
if (date === null || date == 0) {
return UNIX_EPOCH_DATE // unix epoch (1970-01-01T00:00:00.000Z) aka timestamp=0
}
if (date instanceof Date) {
return validateDate(date) // JS date Date('1970-01-01T00:00:00.000Z')
}
if ((typeof date) === 'number') {
date = String(date) // unix timestamp e.g. 1717020154682
}
assert((typeof date) === 'string', `Tried to parse date but got unsupported type ${(typeof date)}: ${date}`)
const errors = [`Failed to parse Date from string: ${date}`]
try {
return parseVersionDateStr(date)
} catch(err) { errors.push(err) }
try {
return parseTimestampDateStr(date)
} catch(err) { errors.push(err) }
try {
return parseISODateStr(date)
} catch(err) { errors.push(err) }
throw errors.join('\n')
}
const versionStrFromDate = (date, {withDate=true, withTime=true, withSeconds=true, withMilliseconds=false}={}) => {
// takes Date, returns YYYYMMDDHHMMSSXXX or YYYYMMDDHHMMSS or YYYYMMDDHHMM or YYYYMMDD
const parsed_date = parseDate(date)
const [date_iso, time_iso] = parsed_date.toISOString().split('T') // ['2001-01-31', '23:59:58.090Z']
const components_to_use = []
if (withDate) {
components_to_use.push(date_iso.replaceAll('-', '')) // '20010131'
}
if (withTime) {
const [hr, min, sec, ms] = time_iso.replace('Z', '').replace('.', ':').split(':') // ['23', '59', '58', '090']
components_to_use.push(hr)
components_to_use.push(min)
if (withSeconds) {
components_to_use.push(sec)
if (withMilliseconds) {
components_to_use.push(ms)
}
}
}
assert(components_to_use.length, 'At least one of {withDate, withTime} must be set.')
const final_str = components_to_use.join('') // 20010131235958
assert(parseVersionDateStr(final_str)) // sanity check to make sure it parses correctly
return final_str
}
// test date functions:
// console.log(parseDate('20120131'))
// console.log(versionStrFromDate(parseDate('20120131')))
// console.log(versionStrFromDate(parseDate('0')))
// console.log(versionStrFromDate(parseDate(0)))
// console.log(versionStrFromDate(parseDate(null)))
// console.log(versionStrFromDate())
// console.log(versionStrFromDate(parseDate('20120131235859090')))
// console.log(versionStrFromDate(parseDate('1970-01-01T00:00:00.000Z')))
// console.log(versionStrFromDate(parseDate('2024-12-01T00:00')))
// console.log(versionStrFromDate(parseDate('2024-12-01'), {withTime: false}))
const prettyPath = (path) => {
// return a pretty-printable path where the abspath of the data dir is replaced with /data for brevity/privacy
return path.replace(DATA_DIR, './data')
}
const pathIsHidden = (relpath) => {
// check if a path or any of the directories above it are hidden (e.g. ./some/.dir/abc or ./.DS_Store)
// make sure test path behaves like an abspath (avoids edge-cases messing up relpaths on '' or '.' or './')
let test_path = relpath
if (test_path.startsWith('./'))
test_path = test_path.substring(2)
if (!test_path.startsWith('/'))
test_path = path.join('/', test_path)
// iterate through parents, checking if any parent is hidden until we reach /
while (test_path !== '/') {
const basename = path.basename(test_path)
if (basename.startsWith('.')) {
// console.log('PATH IS HIDDEN', relpath)
return true
}
// otherwise set test_path to parent dir and repeat
test_path = path.dirname(test_path)
}
return false
}
const pathDepth = (child_path, relative_to='.') => {
// get the number of directory hops deep a child path is relative to '.' (or a given parent)
if (child_path.startsWith('/') && !relative_to.startsWith('/')) {
// if child_path is absolute, then relative_to must be absolute as well otherwise depth will be depth all the way to the / root
relative_to = fs.realpathSync(relative_to)
}
if (relative_to.startsWith('/') && !child_path.startsWith('/')) {
// same deal, either both paths have to be relative, or both have to be absolute
child_path = fs.realpathSync(child_path)
}
const relative_path_to_root = path.relative(relative_to, child_path)
const num_hops_down = relative_path_to_root.split('/').length
return num_hops_down
}
interface DirentWithExtras extends fs.Dirent {
relpath: string,
abspath: string,
reldepth: number,
}
async function getDirEntries(dir_path, {pwd=null, recursive=true, includeHidden=false, includeFiles=true, includeDirs=true, includeLinks=false, filter=null, maxdepth=-1}={}) {
// get the list of all sub-paths under a given path recursively
// console.log('GETTING DIRECTORY ENTRIES', {dir_path, pwd, recursive, includeHidden, includeFiles, includeDirs, maxdepth})
pwd = pwd || dir_path
let dir_abspath = dir_path
if (!dir_abspath.startsWith(pwd)) {
dir_abspath = path.join(pwd, dir_abspath)
}
assert(fs.existsSync(dir_abspath), `Tried to get directory listing for dir that doesn't exist! ${prettyPath(dir_abspath)}`)
return (await fs.promises.readdir(dir_abspath, { recursive, withFileTypes: true }))
.map((dirent: DirentWithExtras) => {
// filter combined with map because relpath is re-used in both operations
const relpath = path.join(path.relative(pwd, dirent.parentPath), dirent.name)
// console.log('CALCULATED RELATIVE PATH', relpath)
const abspath = path.join(dir_abspath, relpath)
const basename = path.basename(dirent.name)
if (!includeLinks && dirent.isSymbolicLink()) return null
if (!includeFiles && dirent.isFile()) return null
if (!includeDirs && dirent.isDirectory()) return null
if (!includeHidden && pathIsHidden(relpath)) return null
dirent.relpath = relpath
dirent.abspath = abspath
dirent.reldepth = pathDepth(relpath)
// console.log('RELATIVE DEPTH MEASURED', prettyPath(dir_abspath), prettyPath(relpath), dirent.reldepth)
if (maxdepth >= 0) {
if ((dirent.reldepth-1) > maxdepth) return null
}
if ((typeof filter) === 'function') {
const should_keep = filter({abspath, relpath, basename, dirent})
if (!should_keep) {
// console.log('FILTER EXCLUDED RESULT', {abspath, relpath, basename, dirent})
return null
}
}
return relpath
})
.filter(Boolean)
.sort() as string[]
}
async function getTotalSize(dir_or_file_path, {pwd=null, _cache=null, filter=null, subfiles=null}={}) {
// get the total size in bytes of a file or directory (recursively adds up file sizes within directory)
// check _cache first
if (_cache && (dir_or_file_path in _cache))
return _cache[dir_or_file_path]
// make sure dir_or_file_path is under pwd
pwd = pwd || path.dirname(dir_or_file_path)
let abspath = dir_or_file_path
if (!dir_or_file_path.startsWith(pwd)) {
abspath = path.join(pwd, dir_or_file_path)
}
// if it's a file, stat it and return the size
// console.log('CALCUALTED ABSPATH', {abspath, dir_or_file_path, pwd})
const dirent = await fs.promises.stat(abspath)
if (dirent.isFile()) {
// console.log('CALCULATING FILE SIZE subfile=', prettyPath(abspath))
return dirent.size
}
// if it's not a file and not a directory, give up, dont try to size special files like FIFO/socket/etc.
if (!dirent.isDirectory()) return 0
// if it's a directory, size is the sum of all the sizes of files within
// console.log('CALCULATING SUBDIR SIZE subdir=', prettyPath(abspath))
let total_bytes = 0
const files_within = subfiles || await getDirEntries(dir_or_file_path, {
pwd,
recursive: true,
includeDirs: false,
includeFiles: true,
filter,
})
for (const subpath of files_within) {
total_bytes += await getTotalSize(subpath, {pwd, _cache, filter})
}
return total_bytes
}
async function getDirSizes(dir_path, {pwd=null, subfiles=null, withRoot=true, filter=null, maxdepth=-1}={}) {
// get the size of a directory and all the files within (recursively) as a number of bytes
// dir_path: path absolute or relative path of the directory you want size info for
// pwd: path (optional) absolute path of the directory you want to interpret dir_path relative to
// subfiles: dirent[] (optional) instead of reading disk, you can manually provide a getDirEntries results list to use
// withRoot: bool include a summary entry for the root dir_path dir in the list as '.'
// filter: function (optional) provide a filter func for dir entries ({abspath, relpath, basename, dirent}) => true/false
// maxdepth: number (optional) does not affect actual calculations, but hides entries below a certain depth in the returned output for brevity
assert((await fs.promises.stat(dir_path)).isDirectory(), `Tried to calculate directory sizes but path is not a directory! ${dir_path}`)
pwd = pwd || dir_path
// {'.': 246, 'example.json': 123, 'example2.txt': 123}
const sizes = {}
// first collect the list of all sub-files recursively and calculate their sizes individually
const files_within = subfiles || await getDirEntries(dir_path, {
pwd,
recursive: true,
includeDirs: false,
includeFiles: true,
// dont pass maxdepth here, we need the entire file listing to accurately calculate parent dir sizes
// it never makes sense to ignore subfiles beyond a certain depth for size calculation
filter, // filter is allowed though, useful to calculcate size of some subset of files that match a pattern
})
for (const subpath of files_within) {
sizes[subpath] = await getTotalSize(subpath, {pwd, _cache: sizes, filter})
}
// then calculate the top-level directory total as the sum of all the file sizes under it
const total_size = Object.values(sizes).reduce((a: number, b: number) => a + b, 0)
// then calculate the subtotals of all the sub-directories
const subdirs_within = await getDirEntries(dir_path, {pwd, recursive: true, includeDirs: true, includeFiles: false, filter, maxdepth})
for (const subpath of subdirs_within) {
sizes[subpath] = await getTotalSize(subpath, {pwd, _cache: sizes, filter}) // uses _cache to avoid re-computing
}
// if maxdepth is passed, filter results to only include paths shallower than max depth
if (maxdepth >= 0) {
for (const subpath of Object.keys(sizes)) {
if (pathDepth(subpath) > maxdepth) {
delete sizes[subpath]
}
}
}
// set total_size last so it appears at the bottom of the object in logs for convenience
if (withRoot) {
sizes['.'] = total_size
}
return sizes
}
async function getLargestPath(path_a, path_b) {
// compare two files/directories and return the largest one of the two (calculating size recursively)
path_a = await fs.promises.realpath(path_a)
path_b = await fs.promises.realpath(path_b)
const size_a = await getTotalSize(path_a)
const size_b = await getTotalSize(path_b)
// console.log('COMPARING', prettyPath(path_a), size_a, ' ', prettyPath(path_b), size_b)
if (size_a > size_b) return path_a
return path_b
}
async function findCommonAncestor(target_abspath, symlink_abspath, {relative=true, search_limit=DATA_DIR}: {relative?: boolean | string, search_limit?: string}={}) {
// given a target path and a symlink path, find the common ancestor path they both share
// (searches recursively through absolute path parent directories until a common dir is found, up to search_limit)
search_limit = await fs.promises.realpath(search_limit)
let relative_dir = search_limit
if ((typeof relative) === 'boolean') {
// if start dir is default, set it to symlinks directory path
if (relative) {
relative_dir = path.dirname(symlink_abspath)
} else {
relative_dir = search_limit
}
} else if ((typeof relative) === 'string') {
// if start dir is a string, get its absolute path
relative_dir = relative as string
} else {
throw `Got invalid type for relative path during common ancestor search: ${relative}`
}
if ((await fs.promises.stat(relative_dir)).isFile()) {
// if start dir is a file, set it to its parent dir path
relative_dir = path.dirname(relative_dir)
}
assert(
(await fs.promises.stat(relative_dir)).isDirectory(),
`Tried to find common ancestor starting from invalid search directory:\n 🔗 ${prettyPath(symlink_abspath)}\n -> ${prettyPath(target_abspath)}\n Error: search dir does not exist or is not a directory: ❌ ${prettyPath(relative_dir)}`,
)
const symlink_filename = path.basename(symlink_abspath)
const target_filename = path.basename(target_abspath)
const symlink_parent_abspath = await fs.promises.realpath(path.dirname(symlink_abspath))
const target_parent_abspath = await fs.promises.realpath(path.dirname(target_abspath))
const search_dir_abspath = await fs.promises.realpath(relative_dir)
let closest_common_ancestor = search_dir_abspath
const isAncestorCommon = (ancestor) => (
target_parent_abspath.startsWith(ancestor)
&& symlink_parent_abspath.startsWith(ancestor))
// check if both src and target start with the same ancestor path
while (closest_common_ancestor !== search_limit) {
if (isAncestorCommon(closest_common_ancestor)) break
else {
// otherwise go up one directory and try again
// console.log(' ...going up a directory', prettyPath(closest_common_ancestor)+'/..')
closest_common_ancestor = path.dirname(closest_common_ancestor)
}
}
assert(
isAncestorCommon(closest_common_ancestor),
`Tried to create relative symlink but could not find common ancestor:\n 🔗 ${prettyPath(symlink_abspath)}\n -> ${prettyPath(target_abspath)}\n Error: target path and symlink path are not both under:\n ❌ ${prettyPath(closest_common_ancestor)}`,
)
const symlink_to_ancestor_relpath = path.relative(symlink_parent_abspath, closest_common_ancestor) // ../../..
const target_from_ancestor_relpath = path.join(path.relative(closest_common_ancestor, target_parent_abspath), target_filename) // 'archive/19999999.23423523'
const symlink_to_target_relpath = path.join(symlink_to_ancestor_relpath, target_from_ancestor_relpath) // '../../../archive/19999999.23423523'
return {
closest_common_ancestor,
search_dir_abspath,
target_abspath,
target_filename,
target_from_ancestor_relpath,
symlink_abspath,
symlink_filename,
symlink_to_ancestor_relpath,
symlink_to_target_relpath,
}
}
interface StatsWithExtras extends fs.Stats {
abspath: string
relpath?: string
reldepth?: number
}
async function blockUntilExists(file_path, {timeout=7_500, min_bytes=0}={}) {
// wait up to timeout seconds until file we expect to exist appears on the filesystem
// (used to handle eventual consistency in network filesystems where we need a delay after writing before reads show up)
const interval = 250
const max_tries = timeout / interval
let tries = 0
let abspath = null
while (tries < max_tries) {
try {
const abspath = await fs.promises.realpath(file_path)
assert(fs.existsSync(abspath))
const dirent = await fs.promises.stat(abspath) as StatsWithExtras
dirent.abspath = abspath
if (min_bytes && (dirent.size < min_bytes)) {
assert(dirent.size >= 1)
// this is a valid warning but unfortunately its too common to bother showing:
// console.warn(`[⚠️] Expected file to be >=${Math.round(min_bytes/1000)}kb but was only ${dirent.size/1000}kb:`, prettyPath(file_path))
}
return dirent
} catch(err) {
const waited = (tries * interval)
if (waited === 5_000) {
console.warn(`[⚠️] Waited >${waited/1000}s for file to appear (is filesystem or bg task running slow?):`, prettyPath(file_path))
}
await wait(interval)
tries++
}
}
throw `Expected file does not exist after ${timeout/1000}s: ${prettyPath(file_path)}`
}
async function overwriteSymlink(target_path, symlink_path, {relative=true, mkdirs=false, search_limit=DATA_DIR, timeout=5_000}: {relative?: boolean | string, mkdirs?: boolean, search_limit?: string, timeout?: number}={}) {
// create a symlink from symlink_path -> target_path
// relative: true => symlink is created as a relative link by default (it will auto-find the closest common ancestor dir, often DATA_DIR)
// mkdirs: true => optionally creates symlink parent dirs automatically)
// make sure target file actually exists first
let target_dirent
try {
target_dirent = await blockUntilExists(target_path, {timeout})
} catch(err) {
throw `Tried to create symlink pointing to file that does not exist:\n 🔗 ${prettyPath(symlink_path)}\n -> ❌ ${prettyPath(target_path)}\n ${err}`
}
const target_abspath = target_dirent.abspath
const target_filename = path.basename(target_abspath)
const target_parent_abspath = path.dirname(target_abspath)
// make sure target is a valid file or directory and not a special character/block device/other weird file
const target_is_dir = target_dirent.isDirectory()
const target_is_file = target_dirent.isFile()
assert(target_is_dir || target_is_file, `Tried to create symlink to an unsupported file type:\n 🔗 ${prettyPath(symlink_path)}\n -> ❌ ${prettyPath(target_path)} (expected file or directory)`)
// create symlink file parent directories if needed
const symlink_filename = path.basename(symlink_path)
const symlink_parent_dir = path.dirname(symlink_path)
if (mkdirs) {
await fs.promises.mkdir(symlink_parent_dir, {recursive: true})
}
try {
assert((await fs.promises.stat(symlink_parent_dir)).isDirectory())
} catch(err) {
throw `Tried to create symlink in a directory that doesn't exist:\n 🔗 ${symlink_parent_dir}❌/${symlink_filename}\n -> ${target_path}\n ${err}`
}
const symlink_parent_abspath = await fs.promises.realpath(symlink_parent_dir)
const symlink_abspath = path.join(symlink_parent_abspath, symlink_filename)
// determine nearest common ancestor between symlink dir and target dir
const {
closest_common_ancestor,
symlink_to_ancestor_relpath,
target_from_ancestor_relpath,
symlink_to_target_relpath,
} = await findCommonAncestor(target_abspath, symlink_abspath, {relative, search_limit})
// set final target path to abspath or relative path depending on {relative} options
let target_path_final
if (relative) {
// make symlink into relative link (based on closest common ancestor dir between symlink_abspath and target_abspath)
target_path_final = symlink_to_target_relpath
// console.log(' 🔗', prettyPath(symlink_abspath), '->', prettyPath(target_abspath), `(as relative link: ${target_path_final})`)
} else {
// make symlink into an absolute path (verbatim passed target_path)
target_path_final = target_path
// console.log(' 🔗', prettyPath(symlink_abspath), '->', prettyPath(target_abspath), '(as absolute path)')
}
// remove any existing symlink at destination if there is already one there
const random_nonce = crypto.randomBytes(16).toString('hex').substring(0, 8)
const symlink_temp_path = `${symlink_abspath}.${random_nonce}.dup`
try { await fs.promises.unlink(symlink_abspath) } catch(err) {}
try { await fs.promises.unlink(symlink_temp_path) } catch(err) {}
// create the symlink and check that it works after creation
let created_symlink = null
try {
created_symlink = symlink_temp_path
await fs.promises.symlink(target_path_final, symlink_temp_path)
created_symlink = symlink_abspath
await fs.promises.rename(symlink_temp_path, symlink_abspath)
} catch(err) {
if (String(err).includes('EISDIR')) {
// console.warn('[⚠️] Tried to create symlink on top of existing directory', prettyPath(symlink_abspath))
// no real recourse in this situation, and its too noisy to log every time this happens
// it's also not always safe to move the dir out of the way, so better to just fail silently here, leaving:
// ${symlink_abspath}.${random_nonce}.dup
} else {
console.warn('[⚠️] Failed to create symlink', prettyPath(created_symlink), err)
}
}
let dirent
try {
dirent = await blockUntilExists(created_symlink, {timeout, min_bytes: 0})
// best we can do here is just check that it exists ^, trying to check that it has the exact expected abspath that we set is bad, because its a race condition:
// assert(dirent.abspath == target_abspath) // its often already overwritten by later activity, so final abspath may already be different
} catch(err) {
throw `Symlink created but does not seem to resolve to intended file:\n 🔗 ${symlink_path}\n -> ❌ ${target_path}\n actual=${dirent?.abspath}\n expected=${target_abspath}\n ${err}`
}
return {
symlink_path,
symlink_abspath: created_symlink,
symlink_filename: path.basename(created_symlink),
symlink_parent_abspath,
symlink_to_ancestor_relpath,
symlink_to_target_relpath,
target_path,
target_abspath,
target_filename,
target_parent_abspath,
target_from_ancestor_relpath,
target_path_final,
target_is_dir,
target_is_file,
target_is_relative: Boolean(relative),
closest_common_ancestor,
}
}
// test symlink and common ancestor finding
// console.log(await findCommonAncestor('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/archive/1709724410.19269/seo.json', '/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/archive/1709724410.19269/seo2.json'))
// console.log(await findCommonAncestor('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/archive/1709724410.19269', '/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/index/snapshots_by_domain/twitter.com/1709724410.19269', {relative: true, search_limit: '/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/'}))
// console.log(await overwriteSymlink('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/archive/1709724410.19269', '/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/index/snapshots_by_domain/twitter.com/1709724410.19269'))
// console.log(await overwriteSymlink('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/index/snapshots_by_domain/twitter.com/1709724410.19269', '/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/index/favorite_snapshots/1709724410.19269', {relative: false, mkdirs: true, search_limit: '/Volumes/NVME/Users/squash/Local/Code/archiveboxes/archivebox-spreadsheet-bot/data/'}))
async function overwriteDir(path) {
// delete any existing folder at the destination path (important otherwise we may create a folder inside an existing folder/symlink)
try {
await fs.promises.rm(path, { recursive: true, force: true });
} catch(err) {}
await fs.promises.mkdir(path, {recursive: true})
return path
}
async function overwriteFile(path, contents, options={encoding: 'utf8', flag: 'w', flush: false, block: true}) {
// write any JS value to a fresh file (e.g. String, Buffer, WritableStream, etc. anything JSON-serializable)
const block_until_created = options.block || true
delete options.block
try {
// delete any existing symlink/file present at the destination path
// (important otherwise we may write into an existing symlink by accident)
await fs.promises.unlink(path)
} catch(err) {}
try {
let nonce = 1
while ((await fs.promises.stat(path)).isDirectory()) {
// if we try to write a file to a path that already has a directory in that location
// (common when trying to write response JSON e.g. http://www.instagram.com/api/graphql returns json and www.instagram.com/api/graphql/abc returns json)
path = path.replace(`.${nonce-1}`, '') + `.${nonce}`
nonce++;
if (nonce > 20) throw `Too many conflicting files while trying to write to ${prettyPath(path)}`
}
} catch(err) {
if (!String(err).includes('no such file or directory')) {
console.warn('[⚠️] Warning: Problem with conflicting directory at while trying to write file', err)
}
}
// refuse writing undefined/null/function because its likely an error and not intended
const content_is_null = (contents === null) || (contents === undefined)
const content_is_func = (typeof contents === 'function')
if (content_is_null || content_is_func) {
throw `Cannot write ${typeof contents} ${contents} to file: ${path}`
}
// Numbers, BigInts, and Booleans can be cast to strings, then wrt
const content_is_primitive = ['number', 'bigint', 'boolean'].includes(typeof contents)
if (content_is_primitive) {
contents = String(contents)
await fs.promises.writeFile(path, contents, options as any)
if (block_until_created) await blockUntilExists(path, {min_bytes: Buffer.byteLength(contents)})
return path
}
// Strings and Buffers can be written directly to file
const content_is_string = (typeof contents === 'string' || contents instanceof String)
const content_is_buffer = Buffer.isBuffer(contents)
if (content_is_string || content_is_buffer) {
await fs.promises.writeFile(path, contents, options as any)
if (block_until_created) await blockUntilExists(path, {min_bytes: Buffer.byteLength(contents)})
return path
}
// WritableStream objects can be piped into file
const content_is_stream = (contents?.pipe)
if (content_is_stream) {
const stream_byte_length = contents.writableLength
const dest_file = fs.createWriteStream(path);
await finished(contents.pipe(dest_file))
if (block_until_created) await blockUntilExists(path, {min_bytes: stream_byte_length})
return path
}
// Objects and Arrays can be JSON-stringified then written into file
const content_is_obj = (Array.isArray(contents) || typeof contents === 'object')
if (content_is_obj) {
contents = JSON.stringify(contents, null, 4)
await fs.promises.writeFile(path, contents, options as any)
if (block_until_created) await blockUntilExists(path, {min_bytes: Buffer.byteLength(contents)})
return path
}
throw `Cannot write contents of type ${typeof contents} to file: ${path} < ${contents}`
}
async function saveExecResult(bin, args=null, {original_url, version}, {cwd='.', timeout=60_000, ...spawn_options}={}) {
assert(bin)
assert(original_url && original_url.includes('://'))
assert(version)
const BIN_NAME = bin // 'yt-dlp'
const ARGS = args || [] // ['--some-arg', '--some-other-arg']
const CWD = cwd || process.cwd() // '.'
const TIMEOUT = 300_000 // 5min timeout
const PATH = process.env.PATH
await fs.promises.mkdir(cwd, {recursive: true})
// quick-n-dirty dump of cmd to bash script, but this might be better: https://github.com/nodejs/node/issues/34840#issuecomment-677402567
const cmd_log_str = `#!/usr/bin/env bash
TYPE="${BIN_NAME}"
URL="${original_url}"
VERSION="${version}"
TIMEOUT=${TIMEOUT}
CWD="${CWD}"
PATH="${PATH}:$PATH"
${BIN_NAME} ${ARGS.map(arg => JSON.stringify(arg)).join(' ')}
`
const cmd_log = path.join(cwd, 'cmd.sh')
await overwriteFile(cmd_log, cmd_log_str)
const stdout_log = fs.createWriteStream(path.join(cwd, 'stdout.log'))
const stderr_log = fs.createWriteStream(path.join(cwd, 'stderr.log'))
const start_date = new Date()
const start_ts = Number(start_date)
const start_time = start_date.toISOString()
const child = child_process.spawn(
BIN_NAME,
ARGS,
{
cwd: CWD,
timeout: TIMEOUT, // 5min timeout
stdio: [null, 'pipe', 'pipe'], // </dev/null >./stdout.log 2>./stderr.log
// detached: true, // run in background, don't block on response
...(spawn_options || {}),
},
)
child.stdout.setEncoding('utf8')
child.stdout.pipe(stdout_log)
child.stderr.setEncoding('utf8')
child.stderr.pipe(stderr_log)
const exec_info = {
TYPE: BIN_NAME,
URL: original_url,
VERSION: version,
bin_name: BIN_NAME,
args: ARGS,
timeout: TIMEOUT,
hostname: os.hostname(),
bin_paths: PATH,
ppid: process.pid,
pid: child.pid,
start_ts,
start_time,
end_time: null,
end_ts: null,
duration: null,
returncode: null,
log_files: {},
output_files: {},
}
// promise that resolves when the command is finished executing
// TODO: refactor to use withTimeout
const getResult = (timeout=TIMEOUT) =>
new Promise((resolve, reject) => {
const loop = setInterval(() => {
if (exec_info.end_time) {
clearInterval(loop)
clearTimeout(timer)
resolve(exec_info)
}
}, 100)
const timer = setTimeout(() => {
clearInterval(loop)
if (!exec_info.end_time) {
reject(new Error(`Process ${BIN_NAME} did not finish within TIMEOUT=${TIMEOUT}`))
}
}, timeout);
})
const logFilesFilter = ({relpath}) =>
['cmd.sh', 'stdout.log', 'stderr.log'].includes(relpath)
const outputFilesFilter = ({relpath}) =>
!['cmd.sh', 'stdout.log', 'stderr.log', 'index.json'].includes(relpath)
const getOutputFiles = async (filter=outputFilesFilter) => {
return await getDirInfo(CWD, {filter, withHelpers: false, withRoot: false, maxdepth: 6})
}
child.on('close', async (returncode) => {
const end_date = new Date()
exec_info.returncode = returncode
exec_info.pid = child.pid
exec_info.end_ts = Number(end_date)
exec_info.end_time = end_date.toISOString()
exec_info.duration = exec_info.end_ts - exec_info.start_ts
exec_info.log_files = await getOutputFiles(logFilesFilter)
exec_info.output_files = await getOutputFiles(outputFilesFilter)
const end_metadata = `
# END_TIME="${exec_info.end_time}"
# DURATION=${exec_info.duration}
# RETURNCODE=${exec_info.returncode }
`
await fs.promises.appendFile(cmd_log, end_metadata)
// write exec_info json (which includes file list) to CWD/index.json
await overwriteFile(path.join(CWD, 'index.json'), exec_info)
})
// child.unref() // dont wait for child process to close
const start_metadata = `
#################### LAST RUN LOG ####################
# HOSTNAME="${exec_info.hostname}"
# PPID=${exec_info.ppid}
# PID=${exec_info.pid}
# START_TIME="${exec_info.start_time}"
`
await fs.promises.appendFile(cmd_log, start_metadata)
return {
...exec_info,
getResult,
}
}
const HASH_CACHE = {}
async function sha256File(file_path: string, {pwd=null}: {pwd?: string}={}) {
return new Promise((resolve, reject) => {
pwd = pwd || path.dirname(file_path);
if (!file_path.startsWith(pwd)) {
file_path = path.join(pwd, file_path);
}
const dirent = fs.statSync(file_path);
const abspath = fs.realpathSync(file_path);
const cache_key = `${abspath}:${dirent.size}:${dirent.mtimeMs}`; // PATH:SIZE:LAST_MODIFIED_TIME
if (cache_key in HASH_CACHE) {
resolve(HASH_CACHE[cache_key]);
}
const hash = crypto.createHash('sha256');
const rs = fs.createReadStream(abspath);
rs.on('error', reject);
rs.on('data', chunk => hash.update(chunk));
rs.on('end', () => {
const final_hash = hash.digest('hex');
HASH_CACHE[cache_key] = final_hash;
resolve(final_hash);
});
}) as Promise<string>
}
async function getDirSha256(dir_path, {pwd=null, withRoot=true, filter=null, maxdepth=-1, subfiles=null}={}) {
// console.log('CALCULATING SHA256 OF FILES IN DIR', dir_path, {withRoot, filter, maxdepth})
// dir_path: path absolute or relative path of the directory you want the merkle sha256 for
// pwd: path (optional) absolute path of the directory you want to interpret dir_path relative to
// withRoot: bool include a summary entry for the root dir_path dir in the list as '.'
// filter: function (optional) provide a filter func for dir entries ({abspath, relpath, basename, dirent}) => true/false
// maxdepth: number (optional) does not affect actual calculations, but hides entries below a certain depth in the returned output for brevity
// subfiles: dirent[] (optional) instead of reading disk, you can manually provide a getDirEntries results list to use
pwd = pwd || dir_path
if (!dir_path.startsWith(pwd)) {
dir_path = path.join(pwd, dir_path)
}
const dirent = await fs.promises.stat(dir_path)
assert(dirent.isDirectory(), `Tried to compute sha256 of path but missing or not a directory! ${dir_path}`)
assert((maxdepth >= -1), `maxdepth must be -1, 0, or 1, 2, 3, etc... (got ${maxdepth})`)
// assert(!(filter && withRoot), `Cannot generate root hash (consistently) when a custom filter is provided!`)
// get the sha256 of every file in a directory recursively (excluding hidden files and symlinks)
// EQUIVALENT TO: find . -type f -not -path '*/.*' -print0 | sort -z | xargs -0 sha256sum
const all_subfiles = (subfiles as string[]) || await getDirEntries(dir_path, {
pwd,
recursive: true,
includeFiles: true,
includeDirs: false,
// ~~maxdepth,~~ // dont pass maxdepth here, we need the entire file listing to accurately calculate parent dir hashes.
// it never makes sense to ignore subfiles beyond a certain depth for hash calculation. Hashes are
// only useful IDs if they are consistent+repeatable, hashing to an arbitrary depth will produce
// many different hashes for the same directory, which is not something we need/want polluting the hash space.
filter, // we do however allow passing a manual filter funcs which does actually affect the hash
// this is useful to allow quick checks to see whether a certain subset of files has changed or not
})
const hashes: {[key: string]: string} = {}
let hashable_summary_str = ''
for (const subfile of all_subfiles) {
// {'versions/20240413144307/screen recording.mp4': '1df4d9c3aca8b36f1f73e327d56038f80a35db407a298edb16c72576d7dd894e', ...}
hashes[subfile] = await sha256File(subfile, {pwd})
const relpath = path.relative(await fs.promises.realpath(dir_path), await fs.promises.realpath(path.join(pwd, subfile)))
hashable_summary_str += `${hashes[subfile]} ./${relpath}\n`
}
// console.log('CALCULATED HASHES FOR ALL SUBFILES IN DIR', dir_path, Object.keys(hashes).length)
// get list of subdirectories and recursively hash every subdirectory
// EQUIVALENT TO: find . -type d -not -path '*/.*' -maxdepth ${maxdepth} -print | sort
const subdirs = await getDirEntries(dir_path, {pwd, recursive: true, includeHidden: false, includeDirs: true, includeFiles: false, filter, maxdepth})
// for each subdirectory, get its hash recursively and store it in the hash list
for (const subdir of subdirs) {
// console.log('GETTING SUBDIR HASH', subdir)
// a directory's hash is defined as the hash of all the *files* within (excluding dirs/symlinks/hidden)
const subdir_hashes = await getDirSha256(
subdir,
{pwd, withRoot: true, filter, maxdepth: 0},
)
hashes[subdir] = subdir_hashes['.']
}
// console.log('CALCULATED HASHES FOR ALL SUBDIRS IN DIR', dir_path, subdirs.length)
// filter results if maxdepth is provided
if (maxdepth >= 0) {
for (const subpath of Object.keys(hashes)) {
if (pathDepth(subpath) > maxdepth) {
delete hashes[subpath]
}
}
}
// console.log('LIMITED OUTPUT DUE TO MAXDEPTH', maxdepth, Object.keys(hashes).length)
// calculate the hash of the root '.' folder by hashing all of hashes of its contents
// EQUIVALENT TO: find . -type f -not -path '*/.*' -print0 | sort -z | xargs -0 sha256sum | sha256sum
if (withRoot) {
// pass the first command's output containing the file list + hashes into another sha256
// to get the final hash of the whole directory combined
// console.log('CALCULATING FINAL ROOT HASH for ', dir_path)
// console.log(hashable_summary_str)
hashes['.'] = crypto.createHash('sha256').update(hashable_summary_str).digest('hex') as string
// console.log('--->', hashes['.'])
}
return hashes
}
async function getDirInfo(dir_path, {pwd=null, withRoot=true, withHelpers=true, filter=null, maxdepth=-1, subfiles=null}={}) {
// get a detailed JSON/dumpable index of a directory's contents, w/ merkle sha256's, sizes, and mimeTypes
// dir_path: path absolute or relative path of the directory you want size info for
// pwd: path (optional) absolute path of the directory you want to interpret dir_path relative to
// withRoot: bool include a summary entry for the root dir_path dir in the list as '.'
// withHelpers: bool attach many extra helper attrs/funcs to results (beyond JSON-serializable core data)
// filter: function (optional) provide a filter func for dir entries ({abspath, relpath, basename, dirent}) => true/false
// maxdepth: number (optional) does not affect actual calculations, but hides entries below a certain depth in the returned output for brevity
// subfiles: dirent[] (optional) instead of reading disk, you can manually provide a getDirEntries results list to use
// {
// ...
// 'example.txt': { ... },
// 'foobar/example.mp3': { ... },
// '.': { // this is the fully agumented result when withHelpers=true
// is_file: false,
// is_dir: true,
// filename: '.',
// basename: '1709039915.378868',
// mimeType: 'inode/directory'
// extension: undefined,
// num_bytes: 11540961,
// num_subpaths: 15,
// sha256: '9fc58b3ed887e7139338062ebd49bd6795373759e8acb73d2f7a40f1413789da',
// reldepth: 1,
// relpath: './',
// cwd: '/opt/archivebox/data/archive/1709039915.378868/',
// dirname: '/opt/archivebox/data/archive',
// abspath: '/opt/archivebox/data/archive/1709039915.378868',
// dirent: Stats {
// dev: 16777240,
// mode: 16895,
// uid: 501,
// ...
// mtimeMs: 1717160622956.1357,
// ctimeMs: 1717160622956.1357,
// },
// created: '2024-05-31T13:03:42.956Z',
// modified: '2024-05-31T13:03:42.956Z',
// summary: './data/archive/1709039915.378868 (inode/directory 11541kb 9fc58b3e)',
// helptext: 'Verify these hashes by running:\n' +
// ' cd /opt/archivebox/data/archive/1709039915.378868 \n' +
// " find . -type f -not -path '*/.*' -print0 | sort -z | xargs -0 sha256sum | sha256sum",
// },
// }
pwd = pwd || dir_path
if (!dir_path.startsWith(pwd)) {
dir_path = path.join(pwd, dir_path)
}
// calculate hashes and sizes recursively
const hashes = await getDirSha256(dir_path, {pwd, withRoot, filter, maxdepth, subfiles})
const sizes = await getDirSizes(dir_path, {pwd, withRoot, filter, maxdepth, subfiles})
const num_total_subpaths = Object.keys(hashes).filter(name => name !== '.').length
const details = {}
for (const [filename, sha256] of Object.entries(hashes)) {
if (filename === '.' && !withRoot) continue
const abspath = await fs.promises.realpath(path.join(dir_path, filename))
const dirent = await fs.promises.stat(abspath)
const num_subpaths = Object.keys(hashes).filter(subpath => subpath.startsWith(filename + '/')).length
const is_file = dirent.isFile()
const is_dir = dirent.isDirectory()
// bare-bones info suitable for JSON dumps/exports
const basic_info = {
sha256,
num_bytes: sizes[filename],
created: (new Date(dirent.ctimeMs)).toISOString(),
mimeType: undefined,
extension: undefined,
num_subpaths: undefined,
}
if (is_dir) {
basic_info.mimeType = 'inode/directory'
basic_info.extension = undefined
basic_info.num_subpaths = (filename === '.') ? num_total_subpaths : num_subpaths
}
if (is_file) {
basic_info.mimeType = mime.lookup(abspath) || null
basic_info.extension = path.extname(filename)
basic_info.num_subpaths = undefined
}
// extra helpers suitable for usage in other areas of the codebase
const info_with_helpers = {
...basic_info,
filename,
basename: path.basename(abspath),
dirname: path.dirname(abspath),
cwd: dir_path,
relpath: is_dir ? (filename + '/') : filename,
reldepth: pathDepth(filename),
abspath,
is_file,
is_dir,
dirent,
modified: (new Date(dirent.mtimeMs)).toISOString(),
summary: `${prettyPath(abspath)} (${basic_info.mimeType} ${Math.round(basic_info.num_bytes/1000)}kb ${sha256.substring(0, 8)})`,
helptext: undefined,
}
if (filename === '.') {
info_with_helpers.helptext = `Verify these hashes by running:\n cd ${prettyPath(abspath)} \n find . -type f -not -path '*/.*' -print0 | sort -z | xargs -0 sha256sum | sha256sum`
}
if ((typeof filter) === 'function') {
if (!filter(info_with_helpers)) continue
}
details[filename] = withHelpers ? info_with_helpers : basic_info
}
return details
}
// console.log(await getDirSha256(
// '/opt/archivebox/data/archive/1709039915.378868/',
// {
// withRoot: true,
// maxdepth: -1,
// filter: ({relpath}) => relpath.startsWith('versions'),
// },
// ))
// console.log(await getDirSizes(
// '/opt/archivebox/data/archive/1709039915.378868/',
// {
// withRoot: false,
// maxdepth: 2,
// filter: ({relpath}) => !relpath.startsWith('versions'),
// },
// ))
// console.log(await getDirInfo(
// '/opt/archivebox/data/archive/1709039915.378868/',
// {
// withRoot: true,
// withHelpers: true,
// maxdepth: 1,
// // filter: ({relpath}) => relpath.startsWith('versions'),
// },
// ))
type DetectFilenameOptions = {
url?: string,
response?: HTTPResponse | Response,
page?: Page,
dir?: string,
abspath?: string,
filename?: string,
basename?: string,
extension?: string,
mimeType?: string,
resourceType?: string,
}
async function detectFilename({ url, response, page, dir, abspath, filename, basename, extension, mimeType, resourceType }: DetectFilenameOptions) {
// this function takes a url (and/or response/page), and detects the abspath,dir,filename,basename,extention,mimeType
// from the URL (+ any enforced path components passed in via args)
// example: detectFilename({url: 'https://example.com/favicon.png', extension: 'ico'}) outputs 'favicon.ico'
//
// it has some quirks that are specific to archiving and may not behave as you expect
// e.g. if visiting the url https://example.com/error.zip returns a 500 text/html error page
// this may still save it as a .zip with mimeType=application/x-zip and ignore the response mimeType the url ends in .zip
// however, if the url has no extension, e.g. https://example.com/error it will
// auto-detect the mimeType based on the response and append an extension, saving as error.html
//
// ⚠️ SECURITY WARNING: think carefully about the permissions, shell injection, and RCE implications of any changes made here ⚠️
// this function writes untrusted web content to the filesystem using auto-detected mimetype to co-erce the extension,
// which can be dangerous (e.g. what if one of these downloads is a malicious ransomware .exe, do we really want to give it .exe?
// if we do, how do we make sure it never gets executed? (without damaging the integrity of the copy)
if (!(response || page)) throw 'Either a page or a response must be provided in order to detect mimeType & URL'
if (response && (typeof response.headers !== 'function')) {
const node_fetch_response: Response = response as Response
response = {
url: () => node_fetch_response.url,
headers: () => node_fetch_response.headers,
} as unknown as HTTPResponse
}
response = response as HTTPResponse
url = url || response?.url() || (await page.url())
if (!url) throw 'URL was not provided and could not be detected from {response, page}'
// Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, Manifest, SignedExchange, Ping, CSPViolationReport, Preflight, Other
try {
resourceType = resourceType || response?.request()?.resourceType()
} catch(err) {
// ignore, sometimes response is null/not available
}
const resourceTypeToMimeType = {
'Stylesheet': 'text/css',
'Script': 'application/x-javascript',
'WebSocket': 'application/json',
'Website': 'text/html',
}
mimeType = mimeType || resourceTypeToMimeType[resourceType] // guess extension based on request resourceType
extension = extension || (mimeType ? mime.extension(mimeType) : null)
// handle special url cases (e.g. schemes in URL_SCHEMES_IGNORED)
if (url.startsWith('about:blank')) {
filename = 'about_blank'
mimeType = 'text/html'
}
else if (url.startsWith('data:')) {
filename = `data__${hashCode(url)}`
}
// console.log('detectFilename>', {url, dir, abspath, filename, basename, extension, mimeType, resourceType})
if (abspath) {
if (dir || filename || basename || extension)
throw '{abspath} should not be passed with other options (e.g. dir, filename, basename, extension)'
var {dir, base: filename, ext: extension, name: basename} = path.parse(abspath)
// path.parse('/home/user/dir/file.txt') returns:
// { root: '/',
// dir: '/home/user/dir',
// base: 'file.txt',
// ext: '.txt',
// name: 'file' }
} else {
dir = dir || path.resolve(process.cwd())
filename = filename // https://example.com/a.1.zip?e.pdf=2#g.h=3 => a.1.zip
|| (new URL(url)).pathname.split('/').at(-1) // https://example.com/file124.rss => file124.rss prefers last component of path with no query/hash, falls back to domain name if no path
|| 'index' // https://example.com/abc/def/ => index.html
//|| (new URL(url)).hostname.replaceAll('.', '_') // https://example.com => example_com (but if disabled, this would be index.html)
}
if (!filename) throw 'filename/abspath were not passed and could not be detected from url'
const path_extname = path.extname(filename)
const resp_mimetype = response && (
(response as any).mimeType
|| response.headers()['content-type']?.split(';')[0]
|| resourceTypeToMimeType[resourceType]
|| 'application/octet-stream'
)
mimeType = mimeType // https://example.com/a.1.zip?e.pdf=2#g.h=3 => application/x-zip prefers mimetype based on extension in path, falls back to response mimeType
|| (path_extname && mime.lookup(path_extname)) // https://example.com/file124.rss => application/rss+xml
|| resp_mimetype // https://example.com/get?type=png => image/png
extension = extension
|| (path_extname && path_extname.replace('.', '')) // https://example.com/a.1.zip?e.pdf=2#g.h=3 => zip prefers extension in path, falls back to response mimeType's suggested extension
|| (resp_mimetype && mime.extension(resp_mimetype)) // https://example.com => html
|| '' // https://example.com/websocket.1 =>
if (extension.startsWith('.'))
extension = extension.slice(1)
basename = basename // https://example.com/a.1.zip?e.pdf=2#g.h=3 => a.1 prefers to filename in path (without extension), falls back to domain name
|| (path.parse(filename).name) // https://mp4dl.example.com => mp4dl_example_com
basename = basename.slice(0, 120) // truncate at 120 characters (leaving 8 chars for .ext)
basename = basename.replace(/[^a-zA-Z0-9%+?&=@;_ \.-]/g, '') // strip characters not allowed in filenames
filename = basename + '.' + extension
if (filename.endsWith('.'))
filename = filename.slice(0, -1)
abspath = abspath || path.join(dir, filename)
// console.log('detectFilename<', {url, dir, abspath, filename, basename, extension, mimeType, resourceType})
return {
url,
dir,
abspath,
filename,
basename,
extension,
mimeType,
resourceType,
resp_mimetype,
}
}
interface DowloadOptions extends DetectFilenameOptions {
browser?: Browser
expected_mimetype?: string
timeout?: number
}
async function download({ url, browser, page, response, dir, abspath, filename, basename, extension, expected_mimetype, timeout }: DowloadOptions) {
url = url || (response as HTTPResponse)?.url() || (await page?.url())
ALREADY_ARCHIVED.add(url.slice(0, 4096)) // prevent running whole archive task on tabs we create for just for downloading
browser = browser || (page && (await page.browser()))
timeout = timeout || 120_000
expected_mimetype = expected_mimetype || ''
let newPage = null
let errors = []
let num_bytes = 0
let bytesBuffer = null
// if we need to fetch the url (i.e. it's not already been requested)
if (!response) {
if (!browser) throw 'No {browser} or {page} was provided to download with'
newPage = await browser.newPage()
if (page) await page.bringToFront() // if origin page is provided, make sure it stays in foreground
response = await newPage.goto(url, {timeout: timeout, waitUntil: 'networkidle0'})
if (page) await page.bringToFront() // if origin page is provided, make sure it stays in foreground
}
url = url || (response as HTTPResponse)?.url() || (await newPage?.url()) || (await page?.url());
const response_mimetype = (response as HTTPResponse).headers()['content-type']?.split(';')[0] || 'text/html'
// detect the filename we should write to based on provided url/response/page/filename/extension suggestions
var {
dir,
abspath,
filename,
basename,
extension,
mimeType,
} = await detectFilename({url, page, response, dir, abspath, filename, basename, extension, mimeType})
// if mimeType is passed, make sure response matches expected mimetype, otherwise consider download a failure
if (!response_mimetype.startsWith(expected_mimetype)) {
errors.push(`Expected ${expected_mimetype} but got ${response_mimetype}`)
} else {
// download the file using puppeteer's response.buffer()
try {
// write the response bytes into the output file
bytesBuffer = await (response as HTTPResponse).buffer()
await overwriteFile(abspath, bytesBuffer)
num_bytes = bytesBuffer.length
} catch(err) {
errors.push(err)
}
// security check to make sure downloaded file is not executable (random binaries downloaded off the internet = dangerous)
fs.access(abspath, fs.constants.X_OK, (err) => {
if (!err) console.warn(
'[⚠️] SECURITY WARNING: Downloaded file appears to be executable:', prettyPath(abspath),
'\n (be careful running untrusted programs downloaded from the internet!)'
)
})
}
// if we opened a dedicated page for downloading, close it now
if (newPage) {
newPage.close()
}
if (errors.length) {
// console.warn(`[❌] Downloading ${url} (${mimeType}) to ${abspath} failed:`, JSON.stringify(errors, null, 4))
} else {
console.log(`[💾] Downloaded ${url.substring(0, 40)} (${num_bytes} ${mimeType})...`.padEnd(82), prettyPath(abspath))
}
return {
url, response, errors,
dir, abspath, filename, basename, extension, mimeType,
bytesBuffer, num_bytes,
}
}
/************************** Puppeteer Launching *******************************/
async function startCluster(puppeteer, args=CHROME_ARGS_DEFAULT) {
console.log(`[🎭] Launching ${CHROME_CLUSTER_WORKERS}x Chromium browsers with puppeteer-cluster:`.padEnd(82), prettyPath(CHROME_PROFILE_PATH))
const cluster = await Cluster.launch({
puppeteer,
monitor: true,
maxConcurrency: CHROME_CLUSTER_WORKERS,
sameDomainDelay: 2550,
workerCreationDelay: 250,
timeout: 300_000, // total ms timeout for an entire task (1000ms * 60s * 5m)
concurrency: Cluster.CONCURRENCY_PAGE, // share cookies between all tabs in a given browser
puppeteerOptions: {
args, // all the chrome launch CLI args
ignoreDefaultArgs: true, // trust me, we have enough args already...
// dumpio: true, // full debug log output, super noisy
}
})
console.log('*************************************************************************')
return cluster
}
async function remoteBrowser(puppeteer, {browserURL, browserWSEndpoint}) {
console.log('[🎭] Connecting Puppeteer to existing Chromium browser via:', browserURL || browserWSEndpoint)
let completed_initial_connection = false
const browser = await puppeteer.connect({browserURL, browserWSEndpoint, defaultViewport: null, targetFilter: () => completed_initial_connection})
completed_initial_connection = true
console.log('*************************************************************************')
return browser
}
async function startBrowser(puppeteer, args=CHROME_ARGS_DEFAULT) {
console.log('[🎭] Launching Puppeteer Chromium browser...'.padEnd(82+1), prettyPath(CHROME_PROFILE_PATH))
const browser = await puppeteer.launch({ignoreDefaultArgs: true, args, dumpio: true})
globalThis.browser = browser
console.log('*************************************************************************')
// store all active tabs on global var by url for easier vscode interactive debugging
const storeTabForDebugger = async (target) => {
try {
globalThis.tabs = globalThis.tabs || {}
const url = target.url()
const page = await target.page()
if (!page || page?.isClosed()) {
delete globalThis.tabs[url]
} else {
globalThis.tab = page
globalThis.tabs[url] = page
}
} catch(err) {console.warn(err)}
}
browser.on('targetcreated', storeTabForDebugger)
browser.on('targetchanged', storeTabForDebugger)
browser.on('targetdestroyed', storeTabForDebugger)
// wait for initial extension background.js/service worker targets to load
await wait(3_000)
// prime the extensions cache
const extensions = await getChromeExtensionsFromCache({browser})
globalThis.extensions = extensions // for easier debugging only
// give the user 2min to check any issues with the initial startup pages (bot profile pages),
// solve captchas, re-login, etc. then close them after that to save resources
const startup_pages = (await browser.pages())
const startup_page_close_delay = 120_000
setTimeout(async () => {
for (const page of startup_pages) {
try { await page.close() } catch(err) { /* page may already be closed by now, which is fine */ }
}
}, startup_page_close_delay)
// setup any extensions that need final runtime configuration using their options pages
// await setup2CaptchaExtension({browser, extensions})
// open a placeholder page so browser window stays open when there are no active archiving pages
// (it's annoying to have the entire window open/close/open/close/etc every time an archive task runs)
const empty_page = await browser.newPage()
await wait(250)
await empty_page.goto('chrome://version')
await wait(500)
console.log('*************************************************************************')
return browser
}
async function startAPIServer(port=API_SERVER_PORT, host=API_SERVER_HOST, taskCallback=null) {
// taskCallback should be an async function that takes ({url}) => and does something with it
assert(taskCallback && (typeof taskCallback === 'function'))
const server = createServer(async (req, res) => {
if (req.method === 'POST') {
console.log(`[API][POST] ${req.url}`)
let body = '';
req.on('data', (chunk) => {
body += chunk;
});
req.on('end', () => {
try {
const jsonData = JSON.parse(body);
// Process the JSON data
console.log(jsonData);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ message: 'JSON data received' }));
} catch (error) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Invalid JSON data' }));
}
});
} else if (req.method === 'GET') {
console.log(`[API][GET] ${req.url}`)
const parsedUrl = new URL(`http://${host}:${port}${req.url}`)
const query = new URLSearchParams(parsedUrl.search);
const url = query.get('url');
if (url && url.includes('://')) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
try {
await taskCallback({url})
res.end(`${url}\n${TASK_PATH(url)}`);
} catch(err) {
res.end(`${url}\n${TASK_PATH(url)}\n${err}`);
}
} else {
res.writeHead(500, { 'Content-Type': 'text/plain' });
res.end(`Bad URL: ${url}\n\nExpected: /?url=https://example.com/url/to/archive`);
}
} else {
res.writeHead(405, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Method not allowed' }));
}
})
server.listen(port, host, () => {
console.log(`[🎰] API Server listening for requests on http://${host}:${port}/?url=...`);
})
console.log('*************************************************************************')
return server
}
async function main(urls, cluster=CHROME_CLUSTER) {
process.chdir(DATA_DIR)
const extensions = await getChromeExtensionsFromPersona({CHROME_EXTENSIONS, CHROME_EXTENSIONS_DIR})
const args = getChromeArgs({...CHROME_LAUNCH_OPTIONS, CHROME_EXTENSIONS: extensions})
const preferences = getChromePreferences({CHROME_PREFERENCES_DEFAULT, CHROME_PREFERENCES_EXTRA, CHROME_DOWNLOADS_DIR, CHROME_EXTENSIONS: extensions})
const Puppeteer = applyChromePreferences(PupeteerExtra, CHROME_PREFERENCES_PATH, preferences)
Puppeteer.use(StealthPlugin());
// Puppeteer.use(ReplPlugin());
// handled by uBlock Origin & ReCaptcha browser extensions, probably not needed here anymore:
// Puppeteer.use(RecaptchaPlugin({
// provider: {id: '2captcha', token: API_KEY_2CAPTCHA},
// visualFeedback: true,
// }))
// const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker')
// puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
if (cluster) {
// launch browser with multiple tabs w/ puppeteer
const cluster = await startCluster(Puppeteer, args)
const handleTask = async ({url}) => cluster.queue(url, botArchiveTask)
const server = await startAPIServer(API_SERVER_PORT, API_SERVER_HOST, handleTask)
console.log('[📋] Running tasks in parallel with puppeteer cluster...')
for (const url of urls) {
if (fs.existsSync(path.join(TASK_PATH(url), 'aiqa.json'))) {
try {
JSON.parse((await fs.promises.readFile(path.join(TASK_PATH(url), 'aiqa.json'))).toString())
console.log(' skipping (already present):', TASK_PATH(url), url)
continue
} catch(err) {
// pass
}
}
cluster.queue(url, botArchiveTask)
await wait(3_000)
}
await cluster.idle();
await cluster.close();
} else {
// launch single new browser w/ puppeter / connect to remote CDP browser w/ puppeteer
const browser = await startBrowser(Puppeteer, args)
// const browser = await remoteBrowser(Puppeteer, {browserURL, browserWSEndpoint})
// run speedtest in the background
speedtest({browser})
const handleTask = async ({url}) => await botArchiveTask({page: (await browser.newPage()), data: url})
const server = await startAPIServer(API_SERVER_PORT, API_SERVER_HOST, handleTask)
// wait for any pre-run setup tasks or server requests
await wait(5_000)
let num_succeeded = 0
let num_failed = 0
console.log(`[📋] Running ${urls.length} tasks sequentially with puppeteer browser...`)
for (const url of urls) {
const run_count = (num_succeeded + num_failed) || 1
// check if task should be run or skipped based on existing snapshot data present in directory
const metrics_path = path.join(TASK_PATH(url), 'metrics.json')
const screenshot_path = path.join(TASK_PATH(url), 'screenrecording.gif')
const aiqa_path = path.join(TASK_PATH(url), 'aiqa.json')
const versions_path = path.join(TASK_PATH(url), 'versions')
if (fs.existsSync(metrics_path) && fs.existsSync(screenshot_path) && fs.existsSync(aiqa_path) && fs.existsSync(versions_path)) {
try {
const ai_qa_result = JSON.parse(await fs.promises.readFile(aiqa_path, 'utf-8'))
console.log(prettyPath(TASK_PATH(url)), `${ai_qa_result.pct_visible}%`, ai_qa_result.website_brand_name, url.substring(0, 80))
assert(ai_qa_result.website_brand_name)
continue
} catch(err) {
// pass
}
}
let delay = 0
// create a new browser page and run the archiving task
const page = (await browser.newPage())
try {
console.log(ANSI.black + `◤==============================================================================[${String(run_count).padStart(3)}]/[${urls.length}]◥` + ANSI.reset)
await botArchiveTask({page, data: url})
delay = 1_000
num_succeeded += 1
} catch(err) {
console.error('[❌] Archiving task failed!', url)
console.error(err)
num_failed += 1
delay = 15_000 // extra delay if there are errors
}
console.log(ANSI.black + `◣==============================================================================[☑ ${num_succeeded}][🆇 ${num_failed}]◢` + ANSI.reset)
// check for abnormally high failure rates and exit early if needed
const failure_pct = Math.round((num_failed/run_count) * 100)
if (failure_pct > 50) {
if (run_count > 5) {
console.warn(`[⚠️] ${failure_pct}% Task failure rate is very high! Will self-cancel after 10 URLs if >50% continue to fail...`)
}
if (run_count > 10) {
throw `Too many tasks failed in a row! Quitting early after ${run_count}/${urls.length} tasks.`
}
}
// increase the delay between tasks based on the ratio of how many are failing:succeeding
delay = Math.pow(4, (num_failed/(num_succeeded + 3))) * delay
// e.g. 0:1 failure ratio == 1 * delay == 1 ~ 15s
// 1:1 failure ratio == 5 * delay == 5 ~ 1m ... 5^(failed:succeeded) exponential increase
// 2:1 failure ratio == 25 * delay == 25s ~ 6m
// 3:1 failure ratio == 125 * delay == 2m ~ 31m
// etc...
// up to 1hr+
delay = Math.min(delay, 3_600_000) // 1hr maximum delay between tasks
delay = Math.max(delay, 1_000) // 1s minimum delay between tasks
if (delay > 2_500) {
console.log('... waiting', Math.round(delay/1000), 'seconds (self rate-limit)...')
}
await wait(delay) // base ratelimit
console.log()
}
if (PASSIVE_ARCHIVING) {
// replace these as-needed:
const browserURL = 'http://localhost:9222/'
const browserWSEndpoint = 'ws://localhost:9222/devtools/browser'
const driver_browser = browser || await remoteBrowser(Puppeteer, {browserURL, browserWSEndpoint})
const archiver_browser = {} //await startBrowser(Puppeteer, args)
const extensions = await getChromeExtensionsFromCache({browser: driver_browser})
// close both browsers if either one is closed
let browser_is_open = true
driver_browser.on('disconnected', async () => {browser_is_open = false}) // await archiver_browser.close()
// archiver_browser.on('disconnected', async () => {browser_is_open = false; await driver_browser.close()})
// handle any tab navigation to a new URL in the driver browser
const handleUserNavigation = async (target) => {
const url = target.url()
const page = await target.page()
// const client = await target.createCDPSession()
if (target.type() == 'page' && page && url) {
console.log(ANSI.black + '==============================================================================' + ANSI.reset)
console.warn('[➕] DRIVER BROWSER NAVIGATED:', ANSI.blue, url, ANSI.reset)
try {
await passiveArchiveTask({browser: driver_browser, page, url})
await wait(3_000)
} catch(err) {
console.error('[❌] Archiving task failed!', url)
console.error(err)
await wait(10_000) // base ratelimit
}
console.log(ANSI.black + '==============================================================================' + ANSI.reset)
// await client.send('Page.enable')
// await client.send('Page.setWebLifecycleState', {state: 'active'})
}
// await client.send('Runtime.runIfWaitingForDebugger')
}
// setup handler to archive new page whenever one is opened
driver_browser.on('targetcreated', handleUserNavigation)
driver_browser.on('targetchanged', handleUserNavigation)
console.log('------------------------------------------------------')
console.log('[👀] Waiting for browser tabs to be opened by human...')
while (browser_is_open) {
await wait(2_000)
}
} else {
while (true) {
await wait(2_000)
}
}
await browser.close()
}
console.log('[✅] Finished all tasks and stopped browsers.')
process.exit(0);
}
/******************************************************************************/
if (import.meta.main) {
main(URLS).catch(console.error);
}
/******************************************************************************/
// if we want to handle CLI args in the future, minimist is great:
// var argv = require('minimist')(process.argv.slice(2));
// console.log(argv); // --url=https://example.com --binpath=/browsers/chromium-1047/bin/chromium --datadir=/Chromium
// const {url, binpath, datadir} = argv;
// OLD CODE, may be useful in the future if we need audio in screenrecordings:
// async function setupScreenrecordingWithAudio(page, wss) {
// console.log('[🎬] Setting up screen-recording plugin...');
// const stream_port = (await wss).options.port;
// // streamPage = await (page.browser()).newPage()
// await page.goto(`chrome-extension://jjndjgheafjngoipoacpjgeicjeomjli/options.html#${stream_port}`)
//
// // puppeteer-stream recording start
// streamFile = fs.createWriteStream(SCREENRECORDING_PATH(page))
// stream = await getStream(page, {
// audio: true,
// video: true,
// bitsPerSecond: 8000000, // 1080p video
// });
// stream.pipe(streamFile);
// return {stream, streamFile}
//
// // puppeteer-stream recording stop & cleanup
// if (stream && streamFile) {
// await stream?.destroy();
// streamFile?.close();
// // await streamPage.close();
// }
// }
Reference in New Issue Copy Permalink