feat: parse RF4DB waterbody catalog

rf4_research/community_cli.py

@@ -15,6 +15,7 @@ from urllib.request import Request, urlopen, HTTPRedirectHandler, build_opener
 
 from .community_sources import (
     parse_rf4db_catches,
+    parse_rf4db_waterbodies,
     parse_rf4map_point,
     parse_rf4posts_spot,
     parse_rf4stat_fishing,
@@ -24,6 +25,7 @@ from .community_sources import (
 
 SOURCES = {
     "rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches),
+    "rf4db-waterbodies": ("https://rf4db.com/ru/maps", parse_rf4db_waterbodies),
     "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
     "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
 }
@@ -316,7 +318,11 @@ def main(argv: list[str] | None = None) -> int:
         # Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too.
         check_and_reserve(site_key, state_file=args.state_file)
         html = fetch_html(url)
-        records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit]
+        records = (
+            parse(html, source_url=url)
+            if args.source in DETAIL_SOURCES or args.source == "rf4db-waterbodies"
+            else parse(html)
+        )[:args.limit]
     except Exception as exc:
         print(f"community source failed: {exc}", file=sys.stderr)
         return 1

rf4_research/community_sources.py

@@ -55,6 +55,18 @@ class RF4DBCatchDetail:
     equipment: tuple[EquipmentItem, ...]
 
 
+@dataclass(frozen=True, slots=True)
+class RF4DBWaterbody:
+    source_system: str
+    source_external_id: str
+    source_url: str
+    name: str
+    unlock_level: int | None
+    unlock_label: str
+    fish_species_count: int
+    image_url: str | None
+
+
 def _text(node: Tag | None) -> str:
     return " ".join(node.get_text(" ", strip=True).split()) if node else ""
 
@@ -127,6 +139,56 @@ def _datetime(raw: object) -> datetime | None:
         return None
 
 
+def parse_rf4db_waterbodies(
+    html: str, *, source_url: str = "https://rf4db.com/ru/maps", expected_count: int = 19,
+) -> list[RF4DBWaterbody]:
+    """Parse the public RF4DB waterbody index without assigning image roles."""
+    soup = BeautifulSoup(html, "html.parser")
+    result: list[RF4DBWaterbody] = []
+    seen: set[str] = set()
+    for link in soup.select('a[href*="/ru/maps/level_"]'):
+        href = link.get("href")
+        if not isinstance(href, str):
+            continue
+        external_id = _key(href)
+        if not external_id or external_id in seen:
+            continue
+        name = _text(link)
+        if not name:
+            continue
+        card = link.find_parent(["article", "li"])
+        if card is None:
+            card = link.find_parent("div")
+        card_text = _text(card)
+        level_match = re.search(r"(?:Уровень|Level)\s*(?:Lv\.?\s*)?(Старт|Start|\d+)", card_text, re.I)
+        fish_match = re.search(r"(?:Рыбы|Fish(?:es)?)\s*(\d+)", card_text, re.I)
+        if not level_match or not fish_match:
+            continue
+        unlock_label = level_match.group(1)
+        unlock_level = int(unlock_label) if unlock_label.isdigit() else None
+        image = card.select_one("img[src], img[data-src]") if card else None
+        image_raw = image.get("src") or image.get("data-src") if image else None
+        image_url = urljoin(source_url, str(image_raw)) if image_raw else None
+        result.append(RF4DBWaterbody(
+            source_system="rf4db",
+            source_external_id=external_id,
+            source_url=urljoin(source_url, href),
+            name=name,
+            unlock_level=unlock_level,
+            unlock_label=unlock_label,
+            fish_species_count=int(fish_match.group(1)),
+            image_url=image_url,
+        ))
+        seen.add(external_id)
+    if not result:
+        raise CommunityParseError("RF4DB waterbody cards not found")
+    if len(result) != expected_count:
+        raise CommunityParseError(
+            f"RF4DB waterbody catalog is incomplete: expected {expected_count}, got {len(result)}"
+        )
+    return result
+
+
 def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> list[ExternalCatch]:
     soup = BeautifulSoup(html, "html.parser")
     result: list[ExternalCatch] = []

tests/fixtures/rf4db_waterbodies_sample.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="ru">
+  <body>
+    <article class="map-card">
+      <img src="https://oss.rf4db.com/game/maps/level_001_mosquito.webp" alt="оз. Комариное">
+      <h3><a href="/ru/maps/level_001_mosquito">оз. Комариное</a></h3>
+      <dl><dt>Уровень</dt><dd>Lv.1</dd><dt>Рыбы</dt><dd>20</dd></dl>
+    </article>
+    <article class="map-card">
+      <img data-src="/game/maps/level_000_cottage.webp" alt="Дачный пруд">
+      <h3><a href="/ru/maps/level_000_cottage">Дачный пруд</a></h3>
+      <dl><dt>Уровень</dt><dd>Старт</dd><dt>Рыбы</dt><dd>7</dd></dl>
+    </article>
+  </body>
+</html>

tests/test_community_sources.py

@@ -7,6 +7,7 @@ from rf4_research.community_sources import (
     CommunityParseError,
     parse_rf4db_catches,
     parse_rf4db_detail,
+    parse_rf4db_waterbodies,
     parse_rf4map_point,
     parse_rf4posts_spot,
     parse_rf4stat_fishing,
@@ -51,6 +52,28 @@ def test_rf4db_detail_rejects_unrelated_html() -> None:
         parse_rf4db_detail("<html></html>", source_url="https://rf4db.com/ru/catches/id")
 
 
+def test_parses_rf4db_waterbody_catalog_without_inventing_image_roles() -> None:
+    rows = parse_rf4db_waterbodies(fixture("rf4db_waterbodies_sample.html"), expected_count=2)
+
+    assert len(rows) == 2
+    assert (rows[0].source_external_id, rows[0].name) == ("level_001_mosquito", "оз. Комариное")
+    assert (rows[0].unlock_level, rows[0].unlock_label, rows[0].fish_species_count) == (1, "1", 20)
+    assert rows[0].source_url == "https://rf4db.com/ru/maps/level_001_mosquito"
+    assert rows[0].image_url == "https://oss.rf4db.com/game/maps/level_001_mosquito.webp"
+    assert (rows[1].unlock_level, rows[1].unlock_label) == (None, "Старт")
+    assert rows[1].image_url == "https://rf4db.com/game/maps/level_000_cottage.webp"
+
+
+def test_rf4db_waterbody_catalog_rejects_unrelated_html() -> None:
+    with pytest.raises(CommunityParseError, match="waterbody cards not found"):
+        parse_rf4db_waterbodies("<html></html>")
+
+
+def test_rf4db_waterbody_catalog_rejects_partial_results() -> None:
+    with pytest.raises(CommunityParseError, match="expected 19, got 2"):
+        parse_rf4db_waterbodies(fixture("rf4db_waterbodies_sample.html"))
+
+
 def test_parses_rf4stat_fishing_rows() -> None:
     row = parse_rf4stat_fishing(
         fixture("rf4stat_fishing_sample.html"),