from __future__ import annotations import argparse import json import os import sys import time from dataclasses import asdict from pathlib import Path from urllib.request import Request, urlopen from .community_sources import ( parse_rf4db_catches, parse_rf4map_point, parse_rf4posts_spot, parse_rf4stat_fishing, parse_rf4stat_posts, ) SOURCES = { "rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches), "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing), "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts), } DETAIL_SOURCES = { "rf4map-point": parse_rf4map_point, "rf4posts-spot": parse_rf4posts_spot, } USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)" MIN_FETCH_INTERVAL_SECONDS = 30 * 60 DEFAULT_STATE_FILE = Path(".cache/community-fetch-state.json") def enforce_fetch_interval( source: str, *, state_file: Path, now: float | None = None, ) -> None: now = time.time() if now is None else now try: state = json.loads(state_file.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError, OSError): state = {} last_fetch = state.get(source) if isinstance(last_fetch, (int, float)) and now - last_fetch < MIN_FETCH_INTERVAL_SECONDS: wait = int(MIN_FETCH_INTERVAL_SECONDS - (now - last_fetch)) raise RuntimeError(f"source cooldown is active; retry in {wait} seconds") def mark_fetch(source: str, *, state_file: Path, now: float | None = None) -> None: now = time.time() if now is None else now try: state = json.loads(state_file.read_text(encoding="utf-8")) except (FileNotFoundError, json.JSONDecodeError, OSError): state = {} state[source] = now state_file.parent.mkdir(parents=True, exist_ok=True) temporary = state_file.with_suffix(".tmp") temporary.write_text(json.dumps(state, sort_keys=True), encoding="utf-8") temporary.replace(state_file) def fetch_html(url: str, *, timeout: float = 30) -> str: request = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"}) with urlopen(request, timeout=timeout) as response: if response.headers.get_content_type() != "text/html": raise ValueError(f"expected text/html, got {response.headers.get_content_type()}") return response.read().decode(response.headers.get_content_charset() or "utf-8") def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page") parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES)) parser.add_argument("--url", help="Override the configured public page URL") parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500") parser.add_argument( "--state-file", type=Path, default=Path(os.environ.get("RF4_COMMUNITY_FETCH_STATE", DEFAULT_STATE_FILE)), help="Persistent per-source cooldown state", ) args = parser.parse_args(argv) if args.source in DETAIL_SOURCES and not args.url: parser.error(f"--url is required for {args.source}") default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source))) url = args.url or default_url try: enforce_fetch_interval(args.source, state_file=args.state_file) html = fetch_html(url) mark_fetch(args.source, state_file=args.state_file) records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit] except Exception as exc: print(f"community source failed: {exc}", file=sys.stderr) return 1 print(json.dumps([asdict(item) for item in records], ensure_ascii=False, default=str)) return 0 if __name__ == "__main__": raise SystemExit(main())