feat: add gear provenance models and browser fetcher

2026-09-20 15:50:43 +07:00
parent b0edae98c6
commit 4e9895fbf4
19 changed files with 986 additions and 20 deletions
@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+
+import { spawnSync } from "node:child_process";
+import fs from "node:fs/promises";
+import readline from "node:readline/promises";
+import process from "node:process";
+import path from "node:path";
+import { chromium } from "../apps/web/node_modules/playwright/index.mjs";
+
+const ROOT = path.resolve(new URL("..", import.meta.url).pathname);
+const PYTHON = process.env.RF4_PYTHON || path.join(ROOT, ".venv/bin/python");
+const DEFAULT_STATE = path.join(ROOT, ".cache/community-fetch-state.json");
+const DEFAULT_PROFILE = path.join(ROOT, ".cache/rf4db-chromium-profile");
+
+function usage() {
+  console.log(`Usage:
+  node scripts/fetch-waterbodies-chromium.mjs catalog [options]
+  node scripts/fetch-waterbodies-chromium.mjs detail --url URL [options]
+
+Options:
+  --output PATH       JSON snapshot path (default: .cache/waterbodies/)
+  --html-output PATH  retain browser DOM HTML (default: temporary .cache file)
+  --state-file PATH   shared cooldown state file
+  --profile PATH      persistent Chromium profile directory
+  --headless          run Chromium headless; cannot handle manual challenges
+  --wait-seconds N    headed wait after load (default: 30)
+`);
+}
+
+function parseArgs(argv) {
+  if (!argv.length || argv.includes("--help")) {
+    usage();
+    process.exit(0);
+  }
+  const mode = argv.shift();
+  if (!new Set(["catalog", "detail"]).has(mode)) throw new Error(`unknown mode: ${mode}`);
+  const args = {
+    mode, url: mode === "catalog" ? "https://rf4db.com/ru/maps" : null,
+    output: null, htmlOutput: null, stateFile: DEFAULT_STATE, profile: DEFAULT_PROFILE,
+    headless: false, waitSeconds: 30,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--url") args.url = argv[++index];
+    else if (arg === "--output") args.output = argv[++index];
+    else if (arg === "--html-output") args.htmlOutput = argv[++index];
+    else if (arg === "--state-file") args.stateFile = argv[++index];
+    else if (arg === "--profile") args.profile = argv[++index];
+    else if (arg === "--headless") args.headless = true;
+    else if (arg === "--wait-seconds") args.waitSeconds = Number(argv[++index]);
+    else throw new Error(`unknown argument: ${arg}`);
+  }
+  if (args.mode === "detail" && !args.url) throw new Error("detail requires --url");
+  if (!Number.isInteger(args.waitSeconds) || args.waitSeconds < 0 || args.waitSeconds > 600) {
+    throw new Error("--wait-seconds must be an integer from 0 to 600");
+  }
+  const parsed = new URL(args.url);
+  if (parsed.protocol !== "https:" || !["rf4db.com", "download.rf4db.com"].includes(parsed.hostname)) {
+    throw new Error("Chromium launcher accepts only HTTPS rf4db.com URLs");
+  }
+  return args;
+}
+
+function reserveCooldown(args) {
+  const source = args.mode === "catalog" ? "rf4db-waterbodies" : "rf4db-waterbody";
+  const command = ["-m", "rf4_research.community_cli", source, "--url", args.url, "--state-file", args.stateFile, "--reserve-only"];
+  const result = spawnSync(PYTHON, command, { cwd: ROOT, encoding: "utf8" });
+  if (result.status !== 0) {
+    if (result.stderr) process.stderr.write(result.stderr);
+    throw new Error(`cooldown reservation failed with exit code ${result.status}`);
+  }
+  process.stdout.write(`${result.stdout.trim()}\n`);
+}
+
+function outputPath(args) {
+  if (args.output) return path.resolve(args.output);
+  const stamp = new Date().toISOString().replaceAll(/[-:]/g, "").replace(".000", "");
+  const stem = args.mode === "catalog" ? "rf4db-catalog" : `rf4db-${path.basename(new URL(args.url).pathname)}`;
+  return path.join(ROOT, ".cache/waterbodies", `${stem}-${stamp}.json`);
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  reserveCooldown(args);
+  const output = outputPath(args);
+  const htmlOutput = path.resolve(args.htmlOutput || `${output}.html`);
+  await fs.mkdir(path.dirname(htmlOutput), { recursive: true });
+  await fs.mkdir(path.dirname(output), { recursive: true });
+
+  const context = await chromium.launchPersistentContext(path.resolve(args.profile), { headless: args.headless });
+  try {
+    const page = context.pages()[0] || await context.newPage();
+    await page.goto(args.url, { waitUntil: "domcontentloaded", timeout: 60_000 });
+    if (!args.headless && args.waitSeconds > 0) {
+      console.log(`Chromium opened ${args.url}. Complete only an ordinary site challenge if shown, then press Enter (or wait ${args.waitSeconds}s).`);
+      const input = readline.createInterface({ input: process.stdin, output: process.stdout });
+      await Promise.race([
+        input.question("Ready to capture DOM? "),
+        new Promise(resolve => setTimeout(resolve, args.waitSeconds * 1000)),
+      ]);
+      input.close();
+    } else if (args.waitSeconds > 0) {
+      await page.waitForTimeout(args.waitSeconds * 1000);
+    }
+    const html = await page.locator("html").evaluate(element => element.outerHTML);
+    await fs.writeFile(htmlOutput, `<!doctype html>\n${html}\n`, "utf8");
+  } finally {
+    await context.close();
+  }
+
+  const parser = path.join(ROOT, "scripts/fetch-waterbodies.py");
+  const result = spawnSync(PYTHON, [parser, args.mode, "--html", htmlOutput, "--output", output, ...(args.mode === "detail" ? ["--url", args.url] : [])], { cwd: ROOT, encoding: "utf8" });
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+  if (result.status !== 0) process.exitCode = result.status || 1;
+}
+
+main().catch(error => {
+  console.error(`Chromium waterbody fetch failed: ${error.message}`);
+  process.exitCode = 1;
+});