feat: add gear provenance models and browser fetcher

scripts/fetch-waterbodies.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Manually fetch RF4DB waterbody snapshots through the guarded research CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timezone
+from dataclasses import asdict
+from pathlib import Path
+from urllib.parse import urlsplit
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from rf4_research.community_cli import _validate_url_before_io
+from rf4_research.community_sources import parse_rf4db_waterbodies, parse_rf4db_waterbody_detail
+
+
+DEFAULT_STATE_FILE = ROOT / ".cache" / "community-fetch-state.json"
+DEFAULT_OUTPUT_DIR = ROOT / ".cache" / "waterbodies"
+CATALOG_SOURCE = "rf4db-waterbodies"
+DETAIL_SOURCE = "rf4db-waterbody"
+
+
+def _timestamp() -> str:
+    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+
+
+def _safe_detail_name(url: str) -> str:
+    name = Path(urlsplit(url).path.rstrip("/")).name or "detail"
+    return re.sub(r"[^A-Za-z0-9_.-]+", "-", name)
+
+
+def _default_output(mode: str, url: str | None) -> Path:
+    stem = "rf4db-catalog" if mode == "catalog" else f"rf4db-{_safe_detail_name(url or '')}"
+    return DEFAULT_OUTPUT_DIR / f"{stem}-{_timestamp()}.json"
+
+
+def _write_json_atomically(path: Path, payload: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    handle = tempfile.NamedTemporaryFile(
+        mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.",
+        suffix=".tmp", delete=False,
+    )
+    temporary = Path(handle.name)
+    try:
+        with handle:
+            json.dump(payload, handle, ensure_ascii=False, indent=2)
+            handle.write("\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+    except Exception:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def fetch_snapshot(
+    mode: str, *, url: str | None, html: Path | None = None, output: Path,
+    state_file: Path, limit: int,
+) -> int:
+    if html is not None:
+        source_url = url or "https://rf4db.com/ru/maps"
+        try:
+            _validate_url_before_io(source_url)
+            document = html.read_text(encoding="utf-8")
+            parsed = (
+                parse_rf4db_waterbodies(document)
+                if mode == "catalog"
+                else parse_rf4db_waterbody_detail(document, source_url=source_url)
+            )
+        except Exception as exc:
+            print(f"local HTML parse failed: {exc}", file=sys.stderr)
+            return 1
+        payload = [asdict(item) for item in parsed] if mode == "catalog" else asdict(parsed)
+        _write_json_atomically(output, payload)
+        count = len(payload) if isinstance(payload, list) else 1
+        print(f"parsed and saved {count} waterbody snapshot(s) to {output}")
+        return 0
+
+    source = CATALOG_SOURCE if mode == "catalog" else DETAIL_SOURCE
+    command = [
+        sys.executable, "-m", "rf4_research.community_cli", source,
+        "--state-file", str(state_file), "--limit", str(limit),
+    ]
+    if url:
+        command.extend(["--url", url])
+    result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=False)
+    if result.returncode != 0:
+        if result.stderr:
+            print(result.stderr, file=sys.stderr, end="")
+        return result.returncode
+    try:
+        payload = json.loads(result.stdout)
+    except json.JSONDecodeError as exc:
+        print(f"fetch succeeded but returned invalid JSON: {exc}", file=sys.stderr)
+        return 1
+    if mode == "catalog" and not isinstance(payload, list):
+        print("fetch succeeded but catalog payload is not a JSON array", file=sys.stderr)
+        return 1
+    if mode == "detail" and not isinstance(payload, dict):
+        print("fetch succeeded but detail payload is not a JSON object", file=sys.stderr)
+        return 1
+    _write_json_atomically(output, payload)
+    count = len(payload) if isinstance(payload, list) else 1
+    print(f"saved {count} waterbody snapshot(s) to {output}")
+    print(f"next import: python -m app.cli import-waterbody-{'catalog' if mode == 'catalog' else 'detail'} --input {output}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Manually fetch RF4DB waterbody data with the shared cooldown guard"
+    )
+    subparsers = parser.add_subparsers(dest="mode", required=True)
+
+    catalog = subparsers.add_parser("catalog", help="fetch the public waterbody catalog")
+    catalog.add_argument("--output", type=Path, help="snapshot path; default: .cache/waterbodies/")
+    catalog.add_argument("--html", type=Path, help="parse an HTML file saved from an authorized browser session; no network")
+    catalog.add_argument("--limit", type=int, default=500, choices=range(1, 501), metavar="1..500")
+
+    detail = subparsers.add_parser("detail", help="fetch one waterbody detail page")
+    detail.add_argument("--url", required=True, help="authorized RF4DB detail URL")
+    detail.add_argument("--output", type=Path, help="snapshot path; default: .cache/waterbodies/")
+    detail.add_argument("--html", type=Path, help="parse an HTML file saved from an authorized browser session; no network")
+    detail.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
+
+    for command in (catalog, detail):
+        command.add_argument(
+            "--state-file", type=Path, default=DEFAULT_STATE_FILE,
+            help=f"cooldown state file (default: {DEFAULT_STATE_FILE})",
+        )
+    args = parser.parse_args(argv)
+    output = args.output or _default_output(args.mode, getattr(args, "url", None))
+    return fetch_snapshot(
+        args.mode,
+        url=getattr(args, "url", None),
+        html=args.html,
+        output=output,
+        state_file=args.state_file,
+        limit=args.limit,
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())