From 4c75db1f74f02678d106eaac58824e3468f6f4c1 Mon Sep 17 00:00:00 2001 From: IK Date: Tue, 15 Sep 2026 17:19:15 +0700 Subject: [PATCH] feat: parse RF4DB waterbody catalog --- rf4_research/community_cli.py | 8 ++- rf4_research/community_sources.py | 62 ++++++++++++++++++++ tests/fixtures/rf4db_waterbodies_sample.html | 15 +++++ tests/test_community_sources.py | 23 ++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/rf4db_waterbodies_sample.html diff --git a/rf4_research/community_cli.py b/rf4_research/community_cli.py index e86c270..2c770db 100644 --- a/rf4_research/community_cli.py +++ b/rf4_research/community_cli.py @@ -15,6 +15,7 @@ from urllib.request import Request, urlopen, HTTPRedirectHandler, build_opener from .community_sources import ( parse_rf4db_catches, + parse_rf4db_waterbodies, parse_rf4map_point, parse_rf4posts_spot, parse_rf4stat_fishing, @@ -24,6 +25,7 @@ from .community_sources import ( SOURCES = { "rf4db": ("https://download.rf4db.com/ru/catches", parse_rf4db_catches), + "rf4db-waterbodies": ("https://rf4db.com/ru/maps", parse_rf4db_waterbodies), "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing), "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts), } @@ -316,7 +318,11 @@ def main(argv: list[str] | None = None) -> int: # Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too. check_and_reserve(site_key, state_file=args.state_file) html = fetch_html(url) - records = (parse(html, source_url=url) if args.source in DETAIL_SOURCES else parse(html))[:args.limit] + records = ( + parse(html, source_url=url) + if args.source in DETAIL_SOURCES or args.source == "rf4db-waterbodies" + else parse(html) + )[:args.limit] except Exception as exc: print(f"community source failed: {exc}", file=sys.stderr) return 1 diff --git a/rf4_research/community_sources.py b/rf4_research/community_sources.py index e79bcfa..ca11f4d 100644 --- a/rf4_research/community_sources.py +++ b/rf4_research/community_sources.py @@ -55,6 +55,18 @@ class RF4DBCatchDetail: equipment: tuple[EquipmentItem, ...] +@dataclass(frozen=True, slots=True) +class RF4DBWaterbody: + source_system: str + source_external_id: str + source_url: str + name: str + unlock_level: int | None + unlock_label: str + fish_species_count: int + image_url: str | None + + def _text(node: Tag | None) -> str: return " ".join(node.get_text(" ", strip=True).split()) if node else "" @@ -127,6 +139,56 @@ def _datetime(raw: object) -> datetime | None: return None +def parse_rf4db_waterbodies( + html: str, *, source_url: str = "https://rf4db.com/ru/maps", expected_count: int = 19, +) -> list[RF4DBWaterbody]: + """Parse the public RF4DB waterbody index without assigning image roles.""" + soup = BeautifulSoup(html, "html.parser") + result: list[RF4DBWaterbody] = [] + seen: set[str] = set() + for link in soup.select('a[href*="/ru/maps/level_"]'): + href = link.get("href") + if not isinstance(href, str): + continue + external_id = _key(href) + if not external_id or external_id in seen: + continue + name = _text(link) + if not name: + continue + card = link.find_parent(["article", "li"]) + if card is None: + card = link.find_parent("div") + card_text = _text(card) + level_match = re.search(r"(?:Уровень|Level)\s*(?:Lv\.?\s*)?(Старт|Start|\d+)", card_text, re.I) + fish_match = re.search(r"(?:Рыбы|Fish(?:es)?)\s*(\d+)", card_text, re.I) + if not level_match or not fish_match: + continue + unlock_label = level_match.group(1) + unlock_level = int(unlock_label) if unlock_label.isdigit() else None + image = card.select_one("img[src], img[data-src]") if card else None + image_raw = image.get("src") or image.get("data-src") if image else None + image_url = urljoin(source_url, str(image_raw)) if image_raw else None + result.append(RF4DBWaterbody( + source_system="rf4db", + source_external_id=external_id, + source_url=urljoin(source_url, href), + name=name, + unlock_level=unlock_level, + unlock_label=unlock_label, + fish_species_count=int(fish_match.group(1)), + image_url=image_url, + )) + seen.add(external_id) + if not result: + raise CommunityParseError("RF4DB waterbody cards not found") + if len(result) != expected_count: + raise CommunityParseError( + f"RF4DB waterbody catalog is incomplete: expected {expected_count}, got {len(result)}" + ) + return result + + def parse_rf4db_catches(html: str, *, base_url: str = "https://rf4db.com") -> list[ExternalCatch]: soup = BeautifulSoup(html, "html.parser") result: list[ExternalCatch] = [] diff --git a/tests/fixtures/rf4db_waterbodies_sample.html b/tests/fixtures/rf4db_waterbodies_sample.html new file mode 100644 index 0000000..1cb066f --- /dev/null +++ b/tests/fixtures/rf4db_waterbodies_sample.html @@ -0,0 +1,15 @@ + + + +
+ оз. Комариное +

оз. Комариное

+
Уровень
Lv.1
Рыбы
20
+
+
+ Дачный пруд +

Дачный пруд

+
Уровень
Старт
Рыбы
7
+
+ + diff --git a/tests/test_community_sources.py b/tests/test_community_sources.py index 5506c1f..c8f860f 100644 --- a/tests/test_community_sources.py +++ b/tests/test_community_sources.py @@ -7,6 +7,7 @@ from rf4_research.community_sources import ( CommunityParseError, parse_rf4db_catches, parse_rf4db_detail, + parse_rf4db_waterbodies, parse_rf4map_point, parse_rf4posts_spot, parse_rf4stat_fishing, @@ -51,6 +52,28 @@ def test_rf4db_detail_rejects_unrelated_html() -> None: parse_rf4db_detail("", source_url="https://rf4db.com/ru/catches/id") +def test_parses_rf4db_waterbody_catalog_without_inventing_image_roles() -> None: + rows = parse_rf4db_waterbodies(fixture("rf4db_waterbodies_sample.html"), expected_count=2) + + assert len(rows) == 2 + assert (rows[0].source_external_id, rows[0].name) == ("level_001_mosquito", "оз. Комариное") + assert (rows[0].unlock_level, rows[0].unlock_label, rows[0].fish_species_count) == (1, "1", 20) + assert rows[0].source_url == "https://rf4db.com/ru/maps/level_001_mosquito" + assert rows[0].image_url == "https://oss.rf4db.com/game/maps/level_001_mosquito.webp" + assert (rows[1].unlock_level, rows[1].unlock_label) == (None, "Старт") + assert rows[1].image_url == "https://rf4db.com/game/maps/level_000_cottage.webp" + + +def test_rf4db_waterbody_catalog_rejects_unrelated_html() -> None: + with pytest.raises(CommunityParseError, match="waterbody cards not found"): + parse_rf4db_waterbodies("") + + +def test_rf4db_waterbody_catalog_rejects_partial_results() -> None: + with pytest.raises(CommunityParseError, match="expected 19, got 2"): + parse_rf4db_waterbodies(fixture("rf4db_waterbodies_sample.html")) + + def test_parses_rf4stat_fishing_rows() -> None: row = parse_rf4stat_fishing( fixture("rf4stat_fishing_sample.html"),