sync

rf4_research/community_cli.py

@@ -16,6 +16,7 @@ from urllib.request import Request, urlopen, HTTPRedirectHandler, build_opener
 from .community_sources import (
     parse_rf4db_catches,
     parse_rf4db_waterbodies,
+    parse_rf4db_waterbody_detail,
     parse_rf4map_point,
     parse_rf4posts_spot,
     parse_rf4stat_fishing,
@@ -30,6 +31,7 @@ SOURCES = {
     "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
 }
 DETAIL_SOURCES = {
+    "rf4db-waterbody": parse_rf4db_waterbody_detail,
     "rf4map-point": parse_rf4map_point,
     "rf4posts-spot": parse_rf4posts_spot,
 }

rf4_research/community_sources.py

@@ -35,6 +35,7 @@ class ExternalCatch:
     clip: str | None
     fishing_style: str | None
     evidence_urls: tuple[str, ...]
+    coordinate_raw: str | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -67,10 +68,29 @@ class RF4DBWaterbody:
     image_url: str | None
 
 
+@dataclass(frozen=True, slots=True)
+class RF4DBWaterbodyDetail:
+    source_system: str
+    source_external_id: str
+    source_url: str
+    name: str
+    description: str | None
+    aliases: tuple[str, ...]
+    fish_species: tuple[str, ...]
+    fish_external_ids: tuple[str | None, ...]
+    image_urls: tuple[str, ...]
+    point_urls: tuple[str, ...]
+
+
 def _text(node: Tag | None) -> str:
     return " ".join(node.get_text(" ", strip=True).split()) if node else ""
 
 
+def _fish_name(node: Tag) -> str:
+    """Read a fish label while dropping the optional trophy weight suffix."""
+    return re.sub(r"\s*\d+(?:[.,]\d+)?\s*(?:кг|kg|г|g)\s*$", "", _text(node), flags=re.I).strip()
+
+
 def _key(href: str | None) -> str | None:
     if not href:
         return None
@@ -189,6 +209,63 @@ def parse_rf4db_waterbodies(
     return result
 
 
+def parse_rf4db_waterbody_detail(
+    html: str, *, source_url: str,
+) -> RF4DBWaterbodyDetail:
+    """Parse one RF4DB waterbody page without assigning media or coordinates.
+
+    The detail page is accepted only when its localized heading and fish list
+    are present. Images and point links remain source candidates; review and
+    canonical crosswalks happen in later pipeline stages.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    root = soup.select_one("article.waterbody-detail, main[data-waterbody-detail]") or soup
+    external_id = _key(source_url)
+    name = _text(root.select_one("h1"))
+    fish_nodes = root.select(".waterbody-fish a[href], [data-fish-list] a[href], a[href*='/fishes/']")
+    if not fish_nodes:
+        fish_nodes = root.select(".fish-list a[href], ul.fish a[href]")
+    fish_species: list[str] = []
+    fish_external_ids: list[str | None] = []
+    seen_fish: set[str] = set()
+    for node in fish_nodes:
+        fish_name = _fish_name(node)
+        fish_id = _key(node.get("href"))
+        identity = fish_id or fish_name.casefold()
+        if not fish_name or identity in seen_fish:
+            continue
+        fish_species.append(fish_name)
+        fish_external_ids.append(fish_id)
+        seen_fish.add(identity)
+    if not external_id or not name or not fish_species:
+        raise CommunityParseError("RF4DB waterbody detail not found or incomplete")
+
+    description_node = root.select_one("[data-description], .waterbody-description, .description")
+    description = _text(description_node) or None
+    aliases = tuple(dict.fromkeys(
+        _text(node) for node in root.select("[data-alias], .waterbody-aliases li, .aliases li")
+        if _text(node) and _text(node) != name
+    ))
+    image_urls = tuple(dict.fromkeys(
+        urljoin(source_url, str(node.get("src") or node.get("data-src")))
+        for node in root.select("img[src], img[data-src]")
+        if (node.get("src") or node.get("data-src"))
+        and "/fish/" not in str(node.get("src") or node.get("data-src"))
+    ))
+    point_urls = tuple(dict.fromkeys(
+        urljoin(source_url, str(node.get("href")))
+        for node in root.select('a[href*="/spots/"], a[href*="/points/"]')
+        if node.get("href")
+    ))
+    return RF4DBWaterbodyDetail(
+        source_system="rf4db", source_external_id=external_id,
+        source_url=source_url, name=name, description=description,
+        aliases=aliases, fish_species=tuple(fish_species),
+        fish_external_ids=tuple(fish_external_ids), image_urls=image_urls,
+        point_urls=point_urls,
+    )
+
+
 def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> list[ExternalCatch]:
     soup = BeautifulSoup(html, "html.parser")
     result: list[ExternalCatch] = []
@@ -201,7 +278,8 @@ def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> li
         external_id = _key(detail.get("href"))
         if not external_id:
             continue
-        x, y = _coordinates(_text(card.select_one(".catch-card__place b")))
+        coordinate_raw = _text(card.select_one(".catch-card__place b")) or None
+        x, y = _coordinates(coordinate_raw or "")
         bait_link = card.select_one('.catch-card__place a[href*="/wiki/baits/"]')
         badges = card.select(".catch-badge")
         weather = next((_text(b.select_one("b")) for b in badges if _text(b).startswith("Погода")), None)
@@ -218,7 +296,7 @@ def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> li
             game_time=_game_time(_text(card.select_one(".catch-card__place"))),
             published_at=None, player_name=None, weather=weather or None,
             water_temperature_c=float(temperature_match.group().replace(",", ".")) if temperature_match else None,
-            clip=None, fishing_style=None, evidence_urls=(),
+            clip=None, fishing_style=None, evidence_urls=(), coordinate_raw=coordinate_raw,
         ))
     if not result:
         raise CommunityParseError("RF4DB catch cards not found")
@@ -288,6 +366,7 @@ def parse_rf4stat_fishing(
         if not external_id:
             continue
         position = row.select_one(".list-position .position:not(.position-locked)")
+        coordinate_raw = _text(row.select_one(".list-position .position")) or None
         x, y = _coordinates(_text(position))
         style = row.select_one(".post-style-icon[title]")
         style_text = str(style.get("title", "")) if style else ""
@@ -305,6 +384,7 @@ def parse_rf4stat_fishing(
             clip=_text(row.select_one(".clip")) or None,
             fishing_style=style_text.removeprefix("Вид ловли:").strip() or None,
             evidence_urls=(urljoin(base_url, str(row.select_one("a.share.hide-print").get("href"))),) if row.select_one("a.share.hide-print") else (),
+            coordinate_raw=coordinate_raw,
         ))
     if not result:
         raise CommunityParseError("RF4-STAT fishing rows not found")
@@ -323,6 +403,7 @@ def parse_rf4stat_posts(
         published_raw = str(post.get("data-published-at", ""))
         published_at = datetime.fromtimestamp(int(published_raw), tz=timezone.utc) if published_raw.isdigit() else None
         position = post.select_one(".spot-col .position:not(.position-locked)")
+        coordinate_raw = _text(post.select_one(".spot-col .position")) or None
         x, y = _coordinates(_text(position))
         style = post.select_one(".post-style-icon[title]")
         style_text = str(style.get("title", "")) if style else ""
@@ -344,7 +425,7 @@ def parse_rf4stat_posts(
                 weather=None, water_temperature_c=None,
                 clip=_text(post.select_one(".clip")) or None,
                 fishing_style=style_text.removeprefix("Вид ловли:").strip() or None,
-                evidence_urls=evidence,
+                evidence_urls=evidence, coordinate_raw=coordinate_raw,
             ))
     if not result:
         raise CommunityParseError("RF4-STAT posts not found")
@@ -386,6 +467,7 @@ def parse_rf4map_point(html: str, *, source_url: str) -> list[ExternalCatch]:
             clip=str(item["clip"]) if isinstance(item.get("clip"), (int, float)) else None,
             fishing_style=None,
             evidence_urls=tuple(url for url in evidence or [] if isinstance(url, str)),
+            coordinate_raw=f"{item['positionX']}:{item['positionY']}" if isinstance(item.get("positionX"), int) and isinstance(item.get("positionY"), int) else None,
         ))
     if not result:
         raise CommunityParseError("RF4MAP point observations not found")
@@ -428,7 +510,7 @@ def parse_rf4posts_spot(html: str, *, source_url: str) -> list[ExternalCatch]:
             weather=None, water_temperature_c=None,
             clip=str(spot["clip"]) if isinstance(spot.get("clip"), (int, float)) else None,
             fishing_style=str(spot["tackleType"]) if isinstance(spot.get("tackleType"), str) else None,
-            evidence_urls=evidence,
+            evidence_urls=evidence, coordinate_raw=str(spot["coordinates"]) if isinstance(spot.get("coordinates"), str) else None,
         ))
     if not result:
         raise CommunityParseError("RF4 Posts fish species not found")