feat: add offline waterbody source crosswalk

2026-09-20 19:16:46 +07:00
parent d541443a1a
commit f02cb247a3
11 changed files with 311 additions and 14 deletions
@@ -0,0 +1,5 @@
+<html><body>
+  <a href="/lakes/12">+ 35 р. Нижняя Тунгуска 38 видов рыбы</a>
+  <a href="/lakes/12">дубликат</a>
+  <a href="/lakes/16">оз. Комариное</a>
+</body></html>
@@ -0,0 +1,12 @@
+<html><body>
+  <ul>
+    <li>
+      <a href="/locations/location/1">Mosquito Lake</a>
+      <span>Catches: 254</span><span>42 baits</span><span>20 kinds of fish</span><span>9 spots</span>
+    </li>
+    <li>
+      <a href="/locations/location/19">Elk Lake</a>
+      <span>Catches: 520</span><span>121 baits</span><span>19 kinds of fish</span><span>11 spots</span>
+    </li>
+  </ul>
+</body></html>
@@ -5,7 +5,7 @@ import pytest

 from rf4_research import community_cli
 from rf4_research.community_cli import enforce_fetch_interval, fetch_site_key, mark_fetch
-from rf4_research.community_sources import RF4DBWaterbodyDetail
+from rf4_research.community_sources import RF4DBWaterbodyDetail, RF4MapWaterbodyCandidate, RF4StatWaterbodyCandidate


 def test_fetch_cooldown_is_persistent_per_source(tmp_path: Path) -> None:
@@ -36,6 +36,7 @@ def test_fetch_site_key_normalizes_common_subdomains() -> None:
    assert fetch_site_key("https://cdn.rf4db.com/assets/img.jpg") == "rf4db.com"
    assert fetch_site_key("https://oss.rf4db.com/game/fish/pike.webp") == "rf4db.com"
    assert fetch_site_key("https://www.rf4db.com/") == "rf4db.com"
+    assert fetch_site_key("https://en.rf4-stat.ru/locations/") == "rf4-stat.ru"
    # Base domain stays the same
    assert fetch_site_key("https://rf4db.com/") == "rf4db.com"
    assert fetch_site_key("https://gw.rf4map.ru/public/images/fish.webp") == "rf4map.ru"
@@ -72,6 +73,68 @@ def test_reserve_only_does_not_make_http_request(
    assert payload == {"reserved": True, "source": "rf4db-waterbodies", "site_key": "rf4db.com"}


+def test_rf4map_waterbody_directory_is_a_manual_source(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
+) -> None:
+    state_file = tmp_path / "fetch-state.json"
+    monkeypatch.setattr(community_cli, "fetch_html", lambda _url: "<html></html>")
+    monkeypatch.setitem(
+        community_cli.SOURCES,
+        "rf4map-waterbodies",
+        ("https://rf4map.ru/lakes", lambda _html: [RF4MapWaterbodyCandidate(
+            source_system="rf4map-waterbodies", source_external_id="fixture",
+            source_url="https://rf4map.ru/lakes/fixture", name="fixture", fish_species_count=None,
+        )]),
+    )
+
+    assert community_cli.main([
+        "rf4map-waterbodies", "--state-file", str(state_file),
+    ]) == 0
+    assert json.loads(capsys.readouterr().out)[0]["source_external_id"] == "fixture"
+
+
+def test_rf4stat_location_directory_is_a_manual_source(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
+) -> None:
+    state_file = tmp_path / "fetch-state.json"
+    monkeypatch.setattr(community_cli, "fetch_html", lambda _url: "<html></html>")
+    monkeypatch.setitem(
+        community_cli.SOURCES,
+        "rf4stat-locations",
+        ("https://en.rf4-stat.ru/locations/", lambda _html: [RF4StatWaterbodyCandidate(
+            source_system="rf4stat-locations", source_external_id="fixture",
+            source_url="https://en.rf4-stat.ru/locations/location/fixture", name="fixture",
+            catches_count=None, fish_species_count=None, bait_count=None, spot_count=None,
+        )]),
+    )
+
+    assert community_cli.main(["rf4stat-locations", "--state-file", str(state_file)]) == 0
+    assert json.loads(capsys.readouterr().out)[0]["source_external_id"] == "fixture"
+
+
+def test_saved_html_source_does_not_reserve_or_make_http_request(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
+) -> None:
+    state_file = tmp_path / "fetch-state.json"
+    html_file = tmp_path / "source.html"
+    html_file.write_text("<html></html>", encoding="utf-8")
+    monkeypatch.setattr(community_cli, "fetch_html", lambda _url: (_ for _ in ()).throw(AssertionError("offline mode must not fetch")))
+    monkeypatch.setitem(
+        community_cli.SOURCES,
+        "rf4map-waterbodies",
+        ("https://rf4map.ru/lakes", lambda _html: [RF4MapWaterbodyCandidate(
+            source_system="rf4map-waterbodies", source_external_id="fixture",
+            source_url="https://rf4map.ru/lakes/fixture", name="fixture", fish_species_count=None,
+        )]),
+    )
+
+    assert community_cli.main([
+        "rf4map-waterbodies", "--html", str(html_file), "--state-file", str(state_file),
+    ]) == 0
+    assert not state_file.exists()
+    assert json.loads(capsys.readouterr().out)[0]["source_external_id"] == "fixture"
+
+
 def test_detail_fetch_serializes_single_record(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
 ) -> None:
@@ -10,13 +10,15 @@ from rf4_research.community_sources import (
    parse_rf4db_waterbodies,
    parse_rf4db_waterbody_detail,
    parse_rf4map_point,
+    parse_rf4map_waterbodies,
    parse_rf4posts_spot,
    parse_rf4stat_fishing,
    parse_rf4stat_posts,
+    parse_rf4stat_waterbodies,
 )
 from rf4_research.waterbody_crosswalk import (
    CanonicalWaterbody, WaterbodyIdentity, missing_external_ids,
-    suggest_waterbody_crosswalk,
+    identities_from_candidates, suggest_waterbody_crosswalk,
 )


@@ -69,6 +71,48 @@ def test_parses_rf4db_waterbody_catalog_without_inventing_image_roles() -> None:
    assert rows[1].image_url == "https://rf4db.com/game/maps/level_000_cottage.webp"


+def test_parses_rf4map_waterbody_directory_as_secondary_candidates() -> None:
+    rows = parse_rf4map_waterbodies(fixture("rf4map_waterbodies_sample.html"))
+
+    assert [(row.source_external_id, row.name, row.fish_species_count) for row in rows] == [
+        ("12", "р. Нижняя Тунгуска", 38),
+        ("16", "оз. Комариное", None),
+    ]
+    assert rows[0].source_system == "rf4map-waterbodies"
+    assert rows[0].source_url == "https://rf4map.ru/lakes/12"
+
+
+def test_rf4map_waterbody_directory_rejects_unrelated_html() -> None:
+    with pytest.raises(CommunityParseError, match="RF4MAP waterbody directory not found"):
+        parse_rf4map_waterbodies("<html></html>")
+
+
+def test_parses_rf4stat_waterbody_directory_metrics() -> None:
+    rows = parse_rf4stat_waterbodies(fixture("rf4stat_waterbodies_sample.html"))
+
+    assert [(row.source_external_id, row.name, row.catches_count, row.fish_species_count, row.bait_count, row.spot_count) for row in rows] == [
+        ("1", "Mosquito Lake", 254, 20, 42, 9),
+        ("19", "Elk Lake", 520, 19, 121, 11),
+    ]
+    assert rows[0].source_url == "https://en.rf4-stat.ru/locations/location/1"
+
+
+def test_rf4stat_waterbody_directory_rejects_unrelated_html() -> None:
+    with pytest.raises(CommunityParseError, match="RF4-STAT waterbody directory not found"):
+        parse_rf4stat_waterbodies("<html></html>")
+
+
+def test_secondary_candidates_convert_to_crosswalk_identities_only() -> None:
+    candidates = parse_rf4map_waterbodies(fixture("rf4map_waterbodies_sample.html"))
+
+    identities = identities_from_candidates(candidates)
+
+    assert identities == [
+        WaterbodyIdentity("rf4map-waterbodies", "12", "р. Нижняя Тунгуска"),
+        WaterbodyIdentity("rf4map-waterbodies", "16", "оз. Комариное"),
+    ]
+
+
 def test_rf4db_waterbody_catalog_rejects_unrelated_html() -> None:
    with pytest.raises(CommunityParseError, match="waterbody cards not found"):
        parse_rf4db_waterbodies("<html></html>")