feat: add RF4DB and RF4-STAT research parsers

rf4_research/community_cli.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict
+from urllib.request import Request, urlopen
+
+from .community_sources import parse_rf4db_catches, parse_rf4stat_fishing, parse_rf4stat_posts
+
+
+SOURCES = {
+    "rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches),
+    "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
+    "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
+}
+USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)"
+
+
+def fetch_html(url: str, *, timeout: float = 30) -> str:
+    request = Request(url, headers={"User-Agent": USER_AGENT, "Accept": "text/html"})
+    with urlopen(request, timeout=timeout) as response:
+        if response.headers.get_content_type() != "text/html":
+            raise ValueError(f"expected text/html, got {response.headers.get_content_type()}")
+        return response.read().decode(response.headers.get_content_charset() or "utf-8")
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
+    parser.add_argument("source", choices=SOURCES)
+    parser.add_argument("--url", help="Override the configured public page URL")
+    parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
+    args = parser.parse_args(argv)
+    default_url, parse = SOURCES[args.source]
+    url = args.url or default_url
+    try:
+        records = parse(fetch_html(url))[:args.limit]
+    except Exception as exc:
+        print(f"community source failed: {exc}", file=sys.stderr)
+        return 1
+    print(json.dumps([asdict(item) for item in records], ensure_ascii=False, default=str))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())