#!/usr/bin/env node import { spawnSync } from "node:child_process"; import fs from "node:fs/promises"; import readline from "node:readline/promises"; import process from "node:process"; import path from "node:path"; import { chromium } from "../apps/web/node_modules/playwright/index.mjs"; const ROOT = path.resolve(new URL("..", import.meta.url).pathname); const PYTHON = process.env.RF4_PYTHON || path.join(ROOT, ".venv/bin/python"); const DEFAULT_STATE = path.join(ROOT, ".cache/community-fetch-state.json"); const DEFAULT_PROFILE = path.join(ROOT, ".cache/rf4db-chromium-profile"); function usage() { console.log(`Usage: node scripts/fetch-waterbodies-chromium.mjs catalog [options] node scripts/fetch-waterbodies-chromium.mjs detail --url URL [options] Options: --output PATH JSON snapshot path (default: .cache/waterbodies/) --html-output PATH retain browser DOM HTML (default: temporary .cache file) --state-file PATH shared cooldown state file --profile PATH persistent Chromium profile directory --headless run Chromium headless; cannot handle manual challenges --wait-seconds N headed wait after load (default: 30) `); } function parseArgs(argv) { if (!argv.length || argv.includes("--help")) { usage(); process.exit(0); } const mode = argv.shift(); if (!new Set(["catalog", "detail"]).has(mode)) throw new Error(`unknown mode: ${mode}`); const args = { mode, url: mode === "catalog" ? "https://rf4db.com/ru/maps" : null, output: null, htmlOutput: null, stateFile: DEFAULT_STATE, profile: DEFAULT_PROFILE, headless: false, waitSeconds: 30, }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--url") args.url = argv[++index]; else if (arg === "--output") args.output = argv[++index]; else if (arg === "--html-output") args.htmlOutput = argv[++index]; else if (arg === "--state-file") args.stateFile = argv[++index]; else if (arg === "--profile") args.profile = argv[++index]; else if (arg === "--headless") args.headless = true; else if (arg === "--wait-seconds") args.waitSeconds = Number(argv[++index]); else throw new Error(`unknown argument: ${arg}`); } if (args.mode === "detail" && !args.url) throw new Error("detail requires --url"); if (!Number.isInteger(args.waitSeconds) || args.waitSeconds < 0 || args.waitSeconds > 600) { throw new Error("--wait-seconds must be an integer from 0 to 600"); } const parsed = new URL(args.url); if (parsed.protocol !== "https:" || !["rf4db.com", "download.rf4db.com"].includes(parsed.hostname)) { throw new Error("Chromium launcher accepts only HTTPS rf4db.com URLs"); } return args; } function reserveCooldown(args) { const source = args.mode === "catalog" ? "rf4db-waterbodies" : "rf4db-waterbody"; const command = ["-m", "rf4_research.community_cli", source, "--url", args.url, "--state-file", args.stateFile, "--reserve-only"]; const result = spawnSync(PYTHON, command, { cwd: ROOT, encoding: "utf8" }); if (result.status !== 0) { if (result.stderr) process.stderr.write(result.stderr); throw new Error(`cooldown reservation failed with exit code ${result.status}`); } process.stdout.write(`${result.stdout.trim()}\n`); } function outputPath(args) { if (args.output) return path.resolve(args.output); const stamp = new Date().toISOString().replaceAll(/[-:]/g, "").replace(".000", ""); const stem = args.mode === "catalog" ? "rf4db-catalog" : `rf4db-${path.basename(new URL(args.url).pathname)}`; return path.join(ROOT, ".cache/waterbodies", `${stem}-${stamp}.json`); } async function main() { const args = parseArgs(process.argv.slice(2)); reserveCooldown(args); const output = outputPath(args); const htmlOutput = path.resolve(args.htmlOutput || `${output}.html`); await fs.mkdir(path.dirname(htmlOutput), { recursive: true }); await fs.mkdir(path.dirname(output), { recursive: true }); const context = await chromium.launchPersistentContext(path.resolve(args.profile), { headless: args.headless }); try { const page = context.pages()[0] || await context.newPage(); await page.goto(args.url, { waitUntil: "domcontentloaded", timeout: 60_000 }); if (!args.headless && args.waitSeconds > 0) { console.log(`Chromium opened ${args.url}. Complete only an ordinary site challenge if shown, then press Enter (or wait ${args.waitSeconds}s).`); const input = readline.createInterface({ input: process.stdin, output: process.stdout }); await Promise.race([ input.question("Ready to capture DOM? "), new Promise(resolve => setTimeout(resolve, args.waitSeconds * 1000)), ]); input.close(); } else if (args.waitSeconds > 0) { await page.waitForTimeout(args.waitSeconds * 1000); } const html = await page.locator("html").evaluate(element => element.outerHTML); await fs.writeFile(htmlOutput, `\n${html}\n`, "utf8"); } finally { await context.close(); } const parser = path.join(ROOT, "scripts/fetch-waterbodies.py"); const result = spawnSync(PYTHON, [parser, args.mode, "--html", htmlOutput, "--output", output, ...(args.mode === "detail" ? ["--url", args.url] : [])], { cwd: ROOT, encoding: "utf8" }); if (result.stdout) process.stdout.write(result.stdout); if (result.stderr) process.stderr.write(result.stderr); if (result.status !== 0) process.exitCode = result.status || 1; } main().catch(error => { console.error(`Chromium waterbody fetch failed: ${error.message}`); process.exitCode = 1; });