from __future__ import annotations import argparse import fcntl import json import os import sys import time from dataclasses import asdict from pathlib import Path from urllib.parse import urlsplit from urllib.request import Request, urlopen from .community_sources import ( parse_rf4db_catches, parse_rf4map_point, parse_rf4posts_spot, parse_rf4stat_fishing, parse_rf4stat_posts, ) SOURCES = { "rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches), "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing), "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts), } DETAIL_SOURCES = { "rf4map-point": parse_rf4map_point, "rf4posts-spot": parse_rf4posts_spot, } USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)" MIN_FETCH_INTERVAL_SECONDS = 30 * 60 DEFAULT_STATE_FILE = Path(".cache/community-fetch-state.json") ALLOWED_HOSTS = frozenset({ "download.rf4db.com", "rf4db.com", "rf4-stat.ru", "rf4map.ru", "rf4-posts.com", }) MAX_RESPONSE_BYTES = 5 * 1024 * 1024 # 5 MB def _read_state(state_file: Path) -> dict: """Read state file with shared lock; return empty dict if missing/corrupt.""" try: with open(state_file, "r") as f: fcntl.flock(f, fcntl.LOCK_SH) try: return json.loads(f.read()) finally: fcntl.flock(f, fcntl.LOCK_UN) except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError): return {} def _write_state(state_file: Path, state: dict) -> None: """Write state file atomically with exclusive lock.""" state_file.parent.mkdir(parents=True, exist_ok=True) with open(state_file, "w") as f: fcntl.flock(f, fcntl.LOCK_EX) try: f.write(json.dumps(state, sort_keys=True)) finally: fcntl.flock(f, fcntl.LOCK_UN) def fetch_site_key(url: str) -> str: """Return a stable cooldown key shared by all endpoints of one site.""" hostname = (urlsplit(url).hostname or "").lower() if hostname.startswith("www."): hostname = hostname[4:] if not hostname: raise ValueError("source URL must include a hostname") return hostname def enforce_fetch_interval( source: str, *, state_file: Path, now: float | None = None, ) -> None: now = time.time() if now is None else now state = _read_state(state_file) last_fetch = state.get(source) if isinstance(last_fetch, (int, float)) and now - last_fetch < MIN_FETCH_INTERVAL_SECONDS: wait = int(MIN_FETCH_INTERVAL_SECONDS - (now - last_fetch)) raise RuntimeError(f"source cooldown is active; retry in {wait} seconds") def mark_fetch(source: str, *, state_file: Path, now: float | None = None) -> None: now = time.time() if now is None else now state = _read_state(state_file) state[source] = now _write_state(state_file, state) def _validate_url_host(url: str) -> str: """Validate URL hostname is in allowlist before making network call.""" hostname = (urlsplit(url).hostname or "").lower() if hostname.startswith("www."): hostname = hostname[4:] if not hostname: raise ValueError("URL must include a valid hostname") if hostname not in ALLOWED_HOSTS: raise ValueError(f"URL hostname {hostname} not in allowlist") return hostname def fetch_html(url: str, *, timeout: float = 30) -> str: # Validate host BEFORE network I/O to prevent SSRF to internal endpoints _validate_url_host(url) request = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"}) with urlopen(request, timeout=timeout) as response: final_url = response.url # Also validate redirect target final_hostname = (urlsplit(final_url).hostname or "").lower() if final_hostname.startswith("www."): final_hostname = final_hostname[4:] if final_hostname not in ALLOWED_HOSTS: raise ValueError(f"Redirect hostname {final_hostname} not in allowlist") if response.headers.get_content_type() != "text/html": raise ValueError(f"expected text/html, got {response.headers.get_content_type()}") data = response.read(MAX_RESPONSE_BYTES + 1) if len(data) > MAX_RESPONSE_BYTES: raise ValueError("response exceeded 5MB limit") return data.decode(response.headers.get_content_charset() or "utf-8") def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page") parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES)) parser.add_argument("--url", help="Override the configured public page URL") parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500") parser.add_argument( "--state-file", type=Path, default=Path(os.environ.get("RF4_COMMUNITY_FETCH_STATE", DEFAULT_STATE_FILE)), help="Persistent per-source cooldown state", ) args = parser.parse_args(argv) if args.source in DETAIL_SOURCES and not args.url: parser.error(f"--url is required for {args.source}") default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source))) url = args.url or default_url try: site_key = fetch_site_key(url) enforce_fetch_interval(site_key, state_file=args.state_file) # Reserve before network I/O: failed attempts count toward the limit too. mark_fetch(site_key, state_file=args.state_file) html = fetch_html(url) records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit] except Exception as exc: print(f"community source failed: {exc}", file=sys.stderr) return 1 print(json.dumps([asdict(item) for item in records], ensure_ascii=False, default=str)) return 0 if __name__ == "__main__": raise SystemExit(main())