feat: add gear provenance models and browser fetcher
This commit is contained in:
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { chromium } from "../apps/web/node_modules/playwright/index.mjs";
|
||||
|
||||
const routes = [
|
||||
"/", "/waterbodies", "/records", "/report", "/status", "/media", "/rules",
|
||||
"/waterbodies/р-вьюнок", "/fish/pike", "/admin", "/admin/moderation",
|
||||
"/admin/external-sources", "/admin/media",
|
||||
];
|
||||
const viewports = [
|
||||
[320, 800], [390, 844], [768, 900], [1280, 900],
|
||||
];
|
||||
const themes = ["system", "light", "dark"];
|
||||
|
||||
function usage() {
|
||||
console.log("Usage: node scripts/capture-visual-matrix.mjs [--base-url URL] [--output DIR]");
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { baseUrl: process.env.WEB_URL || "http://127.0.0.1:4321", output: "/tmp/rf4-visual-matrix" };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
if (argv[index] === "--help" || argv[index] === "-h") {
|
||||
usage();
|
||||
process.exit(0);
|
||||
}
|
||||
if (argv[index] === "--base-url") args.baseUrl = argv[++index];
|
||||
else if (argv[index] === "--output") args.output = argv[++index];
|
||||
else throw new Error(`unknown argument: ${argv[index]}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function safeName(value) {
|
||||
return value.replace(/^\//, "home").replaceAll("/", "-").replace(/[^\p{L}\p{N}._-]+/gu, "-");
|
||||
}
|
||||
|
||||
const { baseUrl, output } = parseArgs(process.argv.slice(2));
|
||||
const outputDir = path.resolve(output);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const page = await browser.newPage();
|
||||
const manifest = [];
|
||||
|
||||
try {
|
||||
for (const theme of themes) {
|
||||
await page.context().clearCookies();
|
||||
if (theme !== "system") {
|
||||
await page.context().addCookies([{ name: "rf4-theme", value: theme, url: baseUrl }]);
|
||||
}
|
||||
await page.emulateMedia({ colorScheme: theme === "system" ? "light" : theme });
|
||||
for (const [width, height] of viewports) {
|
||||
await page.setViewportSize({ width, height });
|
||||
for (const route of routes) {
|
||||
await page.goto(new URL(route, baseUrl).toString(), { waitUntil: "networkidle" });
|
||||
const state = await page.evaluate(() => ({
|
||||
clientWidth: document.documentElement.clientWidth,
|
||||
scrollWidth: document.documentElement.scrollWidth,
|
||||
theme: document.documentElement.dataset.theme || "system",
|
||||
}));
|
||||
const filename = `${theme}-${width}x${height}-${safeName(route)}.png`;
|
||||
await page.screenshot({ path: path.join(outputDir, filename), fullPage: true });
|
||||
manifest.push({ theme, width, height, route, filename, ...state });
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, "manifest.json"),
|
||||
`${JSON.stringify({ baseUrl, count: manifest.length, items: manifest }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
const overflow = manifest.filter((item) => item.scrollWidth > item.clientWidth);
|
||||
console.log(`captured ${manifest.length} screenshots in ${outputDir}`);
|
||||
console.log(`document overflow: ${overflow.length}`);
|
||||
if (overflow.length) process.exitCode = 1;
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import readline from "node:readline/promises";
|
||||
import process from "node:process";
|
||||
import path from "node:path";
|
||||
import { chromium } from "../apps/web/node_modules/playwright/index.mjs";
|
||||
|
||||
const ROOT = path.resolve(new URL("..", import.meta.url).pathname);
|
||||
const PYTHON = process.env.RF4_PYTHON || path.join(ROOT, ".venv/bin/python");
|
||||
const DEFAULT_STATE = path.join(ROOT, ".cache/community-fetch-state.json");
|
||||
const DEFAULT_PROFILE = path.join(ROOT, ".cache/rf4db-chromium-profile");
|
||||
|
||||
function usage() {
|
||||
console.log(`Usage:
|
||||
node scripts/fetch-waterbodies-chromium.mjs catalog [options]
|
||||
node scripts/fetch-waterbodies-chromium.mjs detail --url URL [options]
|
||||
|
||||
Options:
|
||||
--output PATH JSON snapshot path (default: .cache/waterbodies/)
|
||||
--html-output PATH retain browser DOM HTML (default: temporary .cache file)
|
||||
--state-file PATH shared cooldown state file
|
||||
--profile PATH persistent Chromium profile directory
|
||||
--headless run Chromium headless; cannot handle manual challenges
|
||||
--wait-seconds N headed wait after load (default: 30)
|
||||
`);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
if (!argv.length || argv.includes("--help")) {
|
||||
usage();
|
||||
process.exit(0);
|
||||
}
|
||||
const mode = argv.shift();
|
||||
if (!new Set(["catalog", "detail"]).has(mode)) throw new Error(`unknown mode: ${mode}`);
|
||||
const args = {
|
||||
mode, url: mode === "catalog" ? "https://rf4db.com/ru/maps" : null,
|
||||
output: null, htmlOutput: null, stateFile: DEFAULT_STATE, profile: DEFAULT_PROFILE,
|
||||
headless: false, waitSeconds: 30,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--url") args.url = argv[++index];
|
||||
else if (arg === "--output") args.output = argv[++index];
|
||||
else if (arg === "--html-output") args.htmlOutput = argv[++index];
|
||||
else if (arg === "--state-file") args.stateFile = argv[++index];
|
||||
else if (arg === "--profile") args.profile = argv[++index];
|
||||
else if (arg === "--headless") args.headless = true;
|
||||
else if (arg === "--wait-seconds") args.waitSeconds = Number(argv[++index]);
|
||||
else throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
if (args.mode === "detail" && !args.url) throw new Error("detail requires --url");
|
||||
if (!Number.isInteger(args.waitSeconds) || args.waitSeconds < 0 || args.waitSeconds > 600) {
|
||||
throw new Error("--wait-seconds must be an integer from 0 to 600");
|
||||
}
|
||||
const parsed = new URL(args.url);
|
||||
if (parsed.protocol !== "https:" || !["rf4db.com", "download.rf4db.com"].includes(parsed.hostname)) {
|
||||
throw new Error("Chromium launcher accepts only HTTPS rf4db.com URLs");
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function reserveCooldown(args) {
|
||||
const source = args.mode === "catalog" ? "rf4db-waterbodies" : "rf4db-waterbody";
|
||||
const command = ["-m", "rf4_research.community_cli", source, "--url", args.url, "--state-file", args.stateFile, "--reserve-only"];
|
||||
const result = spawnSync(PYTHON, command, { cwd: ROOT, encoding: "utf8" });
|
||||
if (result.status !== 0) {
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
throw new Error(`cooldown reservation failed with exit code ${result.status}`);
|
||||
}
|
||||
process.stdout.write(`${result.stdout.trim()}\n`);
|
||||
}
|
||||
|
||||
function outputPath(args) {
|
||||
if (args.output) return path.resolve(args.output);
|
||||
const stamp = new Date().toISOString().replaceAll(/[-:]/g, "").replace(".000", "");
|
||||
const stem = args.mode === "catalog" ? "rf4db-catalog" : `rf4db-${path.basename(new URL(args.url).pathname)}`;
|
||||
return path.join(ROOT, ".cache/waterbodies", `${stem}-${stamp}.json`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
reserveCooldown(args);
|
||||
const output = outputPath(args);
|
||||
const htmlOutput = path.resolve(args.htmlOutput || `${output}.html`);
|
||||
await fs.mkdir(path.dirname(htmlOutput), { recursive: true });
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
|
||||
const context = await chromium.launchPersistentContext(path.resolve(args.profile), { headless: args.headless });
|
||||
try {
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
await page.goto(args.url, { waitUntil: "domcontentloaded", timeout: 60_000 });
|
||||
if (!args.headless && args.waitSeconds > 0) {
|
||||
console.log(`Chromium opened ${args.url}. Complete only an ordinary site challenge if shown, then press Enter (or wait ${args.waitSeconds}s).`);
|
||||
const input = readline.createInterface({ input: process.stdin, output: process.stdout });
|
||||
await Promise.race([
|
||||
input.question("Ready to capture DOM? "),
|
||||
new Promise(resolve => setTimeout(resolve, args.waitSeconds * 1000)),
|
||||
]);
|
||||
input.close();
|
||||
} else if (args.waitSeconds > 0) {
|
||||
await page.waitForTimeout(args.waitSeconds * 1000);
|
||||
}
|
||||
const html = await page.locator("html").evaluate(element => element.outerHTML);
|
||||
await fs.writeFile(htmlOutput, `<!doctype html>\n${html}\n`, "utf8");
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const parser = path.join(ROOT, "scripts/fetch-waterbodies.py");
|
||||
const result = spawnSync(PYTHON, [parser, args.mode, "--html", htmlOutput, "--output", output, ...(args.mode === "detail" ? ["--url", args.url] : [])], { cwd: ROOT, encoding: "utf8" });
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
if (result.status !== 0) process.exitCode = result.status || 1;
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(`Chromium waterbody fetch failed: ${error.message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Manually fetch RF4DB waterbody snapshots through the guarded research CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from rf4_research.community_cli import _validate_url_before_io
|
||||
from rf4_research.community_sources import parse_rf4db_waterbodies, parse_rf4db_waterbody_detail
|
||||
|
||||
|
||||
DEFAULT_STATE_FILE = ROOT / ".cache" / "community-fetch-state.json"
|
||||
DEFAULT_OUTPUT_DIR = ROOT / ".cache" / "waterbodies"
|
||||
CATALOG_SOURCE = "rf4db-waterbodies"
|
||||
DETAIL_SOURCE = "rf4db-waterbody"
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
|
||||
def _safe_detail_name(url: str) -> str:
|
||||
name = Path(urlsplit(url).path.rstrip("/")).name or "detail"
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "-", name)
|
||||
|
||||
|
||||
def _default_output(mode: str, url: str | None) -> Path:
|
||||
stem = "rf4db-catalog" if mode == "catalog" else f"rf4db-{_safe_detail_name(url or '')}"
|
||||
return DEFAULT_OUTPUT_DIR / f"{stem}-{_timestamp()}.json"
|
||||
|
||||
|
||||
def _write_json_atomically(path: Path, payload: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = tempfile.NamedTemporaryFile(
|
||||
mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.",
|
||||
suffix=".tmp", delete=False,
|
||||
)
|
||||
temporary = Path(handle.name)
|
||||
try:
|
||||
with handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
except Exception:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def fetch_snapshot(
|
||||
mode: str, *, url: str | None, html: Path | None = None, output: Path,
|
||||
state_file: Path, limit: int,
|
||||
) -> int:
|
||||
if html is not None:
|
||||
source_url = url or "https://rf4db.com/ru/maps"
|
||||
try:
|
||||
_validate_url_before_io(source_url)
|
||||
document = html.read_text(encoding="utf-8")
|
||||
parsed = (
|
||||
parse_rf4db_waterbodies(document)
|
||||
if mode == "catalog"
|
||||
else parse_rf4db_waterbody_detail(document, source_url=source_url)
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"local HTML parse failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
payload = [asdict(item) for item in parsed] if mode == "catalog" else asdict(parsed)
|
||||
_write_json_atomically(output, payload)
|
||||
count = len(payload) if isinstance(payload, list) else 1
|
||||
print(f"parsed and saved {count} waterbody snapshot(s) to {output}")
|
||||
return 0
|
||||
|
||||
source = CATALOG_SOURCE if mode == "catalog" else DETAIL_SOURCE
|
||||
command = [
|
||||
sys.executable, "-m", "rf4_research.community_cli", source,
|
||||
"--state-file", str(state_file), "--limit", str(limit),
|
||||
]
|
||||
if url:
|
||||
command.extend(["--url", url])
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
|
||||
if result.returncode != 0:
|
||||
if result.stderr:
|
||||
print(result.stderr, file=sys.stderr, end="")
|
||||
return result.returncode
|
||||
try:
|
||||
payload = json.loads(result.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"fetch succeeded but returned invalid JSON: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if mode == "catalog" and not isinstance(payload, list):
|
||||
print("fetch succeeded but catalog payload is not a JSON array", file=sys.stderr)
|
||||
return 1
|
||||
if mode == "detail" and not isinstance(payload, dict):
|
||||
print("fetch succeeded but detail payload is not a JSON object", file=sys.stderr)
|
||||
return 1
|
||||
_write_json_atomically(output, payload)
|
||||
count = len(payload) if isinstance(payload, list) else 1
|
||||
print(f"saved {count} waterbody snapshot(s) to {output}")
|
||||
print(f"next import: python -m app.cli import-waterbody-{'catalog' if mode == 'catalog' else 'detail'} --input {output}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Manually fetch RF4DB waterbody data with the shared cooldown guard"
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="mode", required=True)
|
||||
|
||||
catalog = subparsers.add_parser("catalog", help="fetch the public waterbody catalog")
|
||||
catalog.add_argument("--output", type=Path, help="snapshot path; default: .cache/waterbodies/")
|
||||
catalog.add_argument("--html", type=Path, help="parse an HTML file saved from an authorized browser session; no network")
|
||||
catalog.add_argument("--limit", type=int, default=500, choices=range(1, 501), metavar="1..500")
|
||||
|
||||
detail = subparsers.add_parser("detail", help="fetch one waterbody detail page")
|
||||
detail.add_argument("--url", required=True, help="authorized RF4DB detail URL")
|
||||
detail.add_argument("--output", type=Path, help="snapshot path; default: .cache/waterbodies/")
|
||||
detail.add_argument("--html", type=Path, help="parse an HTML file saved from an authorized browser session; no network")
|
||||
detail.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
|
||||
|
||||
for command in (catalog, detail):
|
||||
command.add_argument(
|
||||
"--state-file", type=Path, default=DEFAULT_STATE_FILE,
|
||||
help=f"cooldown state file (default: {DEFAULT_STATE_FILE})",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
output = args.output or _default_output(args.mode, getattr(args, "url", None))
|
||||
return fetch_snapshot(
|
||||
args.mode,
|
||||
url=getattr(args, "url", None),
|
||||
html=args.html,
|
||||
output=output,
|
||||
state_file=args.state_file,
|
||||
limit=args.limit,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user