feat: add RF4MAP and RF4 Posts research parsers

rf4_research/community_cli.py

@@ -6,7 +6,13 @@ import sys
 from dataclasses import asdict
 from urllib.request import Request, urlopen
 
-from .community_sources import parse_rf4db_catches, parse_rf4stat_fishing, parse_rf4stat_posts
+from .community_sources import (
+    parse_rf4db_catches,
+    parse_rf4map_point,
+    parse_rf4posts_spot,
+    parse_rf4stat_fishing,
+    parse_rf4stat_posts,
+)
 
 
 SOURCES = {
@@ -14,6 +20,10 @@ SOURCES = {
     "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
     "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
 }
+DETAIL_SOURCES = {
+    "rf4map-point": parse_rf4map_point,
+    "rf4posts-spot": parse_rf4posts_spot,
+}
 USER_AGENT = "RF4-Spotter/0.1 (authorized data integration)"
 
 
@@ -27,14 +37,17 @@ def fetch_html(url: str, *, timeout: float = 30) -> str:
 
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
-    parser.add_argument("source", choices=SOURCES)
+    parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES))
     parser.add_argument("--url", help="Override the configured public page URL")
     parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
     args = parser.parse_args(argv)
-    default_url, parse = SOURCES[args.source]
+    if args.source in DETAIL_SOURCES and not args.url:
+        parser.error(f"--url is required for {args.source}")
+    default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source)))
     url = args.url or default_url
     try:
-        records = parse(fetch_html(url))[:args.limit]
+        html = fetch_html(url)
+        records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit]
     except Exception as exc:
         print(f"community source failed: {exc}", file=sys.stderr)
         return 1

rf4_research/community_sources.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import re
 from dataclasses import dataclass
 from datetime import datetime, time, timezone
@@ -83,6 +84,49 @@ def _weight(raw: str) -> int | None:
     return round(float(compact) * 1000) if unit in {"кг", "kg"} else int(compact.replace(".", ""))
 
 
+def _next_payloads(html: str) -> list[str]:
+    """Decode the string payloads emitted by Next.js' server components."""
+    soup = BeautifulSoup(html, "html.parser")
+    result: list[str] = []
+    pattern = re.compile(r'^self\.__next_f\.push\(\[1,("(?:\\.|[^"\\])*")\]\)$', re.DOTALL)
+    for script in soup.find_all("script"):
+        raw = script.string or ""
+        match = pattern.fullmatch(raw.strip())
+        if not match:
+            continue
+        try:
+            result.append(json.loads(match.group(1)))
+        except json.JSONDecodeError:
+            continue
+    return result
+
+
+def _json_after_marker(
+    payloads: list[str], marker: str, *, required_key: str | None = None,
+) -> dict[str, object] | None:
+    decoder = json.JSONDecoder()
+    for payload in payloads:
+        offset = 0
+        while (start := payload.find(marker, offset)) >= 0:
+            offset = start + len(marker)
+            try:
+                value, _ = decoder.raw_decode(payload[offset:])
+            except json.JSONDecodeError:
+                continue
+            if isinstance(value, dict) and (required_key is None or required_key in value):
+                return value
+    return None
+
+
+def _datetime(raw: object) -> datetime | None:
+    if not isinstance(raw, str):
+        return None
+    try:
+        return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
 def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> list[ExternalCatch]:
     soup = BeautifulSoup(html, "html.parser")
     result: list[ExternalCatch] = []
@@ -243,3 +287,87 @@ def parse_rf4stat_posts(
     if not result:
         raise CommunityParseError("RF4-STAT posts not found")
     return result
+
+
+def parse_rf4map_point(html: str, *, source_url: str) -> list[ExternalCatch]:
+    soup = BeautifulSoup(html, "html.parser")
+    lake_link = soup.select_one('a[href^="/lakes/"]')
+    waterbody = _text(lake_link)
+    waterbody_id = _key(lake_link.get("href")) if lake_link else None
+    points = _json_after_marker(_next_payloads(html), '"points":')
+    items = points.get("items") if points else None
+    if not waterbody or not waterbody_id or not isinstance(items, list):
+        raise CommunityParseError("RF4MAP point payload not found")
+
+    result: list[ExternalCatch] = []
+    for item in items:
+        if not isinstance(item, dict) or not isinstance(item.get("id"), int):
+            continue
+        fish = item.get("fish")
+        bait = item.get("bait")
+        evidence = item.get("imageUrls")
+        if not isinstance(fish, dict) or not isinstance(fish.get("name"), str):
+            continue
+        if not isinstance(evidence, list):
+            evidence = []
+        result.append(ExternalCatch(
+            source_system="rf4map", source_external_id=str(item["id"]), source_url=source_url,
+            fish=str(fish["name"]), fish_external_id=str(fish["id"]) if isinstance(fish.get("id"), int) else None,
+            waterbody=waterbody, waterbody_external_id=waterbody_id,
+            x=item.get("positionX") if isinstance(item.get("positionX"), int) else None,
+            y=item.get("positionY") if isinstance(item.get("positionY"), int) else None,
+            weight_g=None,
+            bait=str(bait["name"]) if isinstance(bait, dict) and isinstance(bait.get("name"), str) else None,
+            rig_type=None, game_time=None, published_at=_datetime(item.get("createdAt")),
+            player_name=item.get("authorName") if isinstance(item.get("authorName"), str) else None,
+            weather=None, water_temperature_c=None,
+            clip=str(item["clip"]) if isinstance(item.get("clip"), (int, float)) else None,
+            fishing_style=None,
+            evidence_urls=tuple(url for url in evidence or [] if isinstance(url, str)),
+        ))
+    if not result:
+        raise CommunityParseError("RF4MAP point observations not found")
+    return result
+
+
+def parse_rf4posts_spot(html: str, *, source_url: str) -> list[ExternalCatch]:
+    soup = BeautifulSoup(html, "html.parser")
+    spot = _json_after_marker(_next_payloads(html), '"spot":', required_key="id")
+    species = spot.get("fishSpecies") if spot else None
+    spot_id = spot.get("id") if spot else None
+    if not isinstance(spot_id, str) or not isinstance(species, list) or not species:
+        raise CommunityParseError("RF4 Posts spot payload not found")
+
+    waterbody = _text(soup.select_one("h1"))
+    fish_list = soup.select_one('[role="list"][aria-label]')
+    fish_names = [_text(node) for node in fish_list.select('[role="listitem"]')] if fish_list else []
+    if not waterbody or len(fish_names) != len(species):
+        raise CommunityParseError("RF4 Posts localized spot labels not found")
+    x, y = _coordinates(str(spot.get("coordinates", "")))
+    screenshots = spot.get("screenshots")
+    if not isinstance(screenshots, list):
+        screenshots = []
+    evidence = tuple(
+        str(item["url"]) for item in screenshots or []
+        if isinstance(item, dict) and isinstance(item.get("url"), str)
+    )
+    result: list[ExternalCatch] = []
+    for fish_id, fish_name in zip(species, fish_names, strict=True):
+        if not isinstance(fish_id, str):
+            continue
+        result.append(ExternalCatch(
+            source_system="rf4posts-spot", source_external_id=f"{spot_id}:{fish_id}", source_url=source_url,
+            fish=fish_name, fish_external_id=fish_id,
+            waterbody=waterbody,
+            waterbody_external_id=str(spot["waterBody"]) if isinstance(spot.get("waterBody"), str) else None,
+            x=x, y=y, weight_g=None, bait=None,
+            rig_type=str(spot["bottomRigType"]) if isinstance(spot.get("bottomRigType"), str) else None,
+            game_time=None, published_at=_datetime(spot.get("createdAt")), player_name=None,
+            weather=None, water_temperature_c=None,
+            clip=str(spot["clip"]) if isinstance(spot.get("clip"), (int, float)) else None,
+            fishing_style=str(spot["tackleType"]) if isinstance(spot.get("tackleType"), str) else None,
+            evidence_urls=evidence,
+        ))
+    if not result:
+        raise CommunityParseError("RF4 Posts fish species not found")
+    return result