feat: add gear provenance models and browser fetcher

2026-09-20 15:50:43 +07:00
parent b0edae98c6
commit 4e9895fbf4
19 changed files with 986 additions and 20 deletions
@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="ru">
+  <body>
+    <article class="gear-card" data-gear-card data-gear-id="spiker-2" data-category="lure" data-subcategory="spinner" data-brand="RF4" data-family="spoon" data-unlock-level="12">
+      <a href="/ru/wiki/lures/spiker-2"><h3 data-name>Spiker #2</h3></a>
+      <img src="https://oss.rf4db.com/game/gear/spiker-2.webp" alt="Spiker #2">
+    </article>
+    <article class="gear-card" data-gear-card data-gear-id="method-popup" data-category="rig" data-subcategory="feeder" data-unlock-level="0">
+      <a href="/ru/wiki/rigs/method-popup"><h3 data-name>Method Popup</h3></a>
+    </article>
+  </body>
+</html>
@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="ru">
+  <body>
+    <main data-gear-detail data-category="lure" data-subcategory="spinner" data-brand="RF4" data-family="spoon" data-unlock-level="12">
+      <h1>Spiker #2</h1>
+      <dl>
+        <dd data-gear-attribute="weight" data-type="number">0</dd>
+        <dd data-gear-attribute="floating" data-state="not_applicable">Не применяется</dd>
+        <dd data-gear-attribute="target_fish" data-state="missing">Нет данных</dd>
+      </dl>
+      <a data-gear-variant>Spiker #2 01-015</a>
+      <a data-compatible-with>Light spinning rod</a>
+      <a data-rig-type>Spinning</a>
+      <img src="/game/gear/spiker-2.webp" alt="Spiker #2">
+    </main>
+  </body>
+</html>
@@ -56,6 +56,22 @@ def test_failed_fetch_still_reserves_site_cooldown(tmp_path: Path, monkeypatch:
    assert "rf4db.com" in json.loads(state_file.read_text(encoding="utf-8"))


+def test_reserve_only_does_not_make_http_request(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
+) -> None:
+    state_file = tmp_path / "fetch-state.json"
+
+    def fail(_url: str) -> str:
+        raise AssertionError("reserve-only must stop before HTTP")
+
+    monkeypatch.setattr(community_cli, "fetch_html", fail)
+    assert community_cli.main([
+        "rf4db-waterbodies", "--state-file", str(state_file), "--reserve-only",
+    ]) == 0
+    payload = json.loads(capsys.readouterr().out)
+    assert payload == {"reserved": True, "source": "rf4db-waterbodies", "site_key": "rf4db.com"}
+
+
 def test_detail_fetch_serializes_single_record(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
 ) -> None:
@@ -0,0 +1,76 @@
+import importlib.util
+import json
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[1] / "scripts" / "fetch-waterbodies.py"
+SPEC = importlib.util.spec_from_file_location("fetch_waterbodies", SCRIPT)
+assert SPEC and SPEC.loader
+fetch_waterbodies = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(fetch_waterbodies)
+
+
+def test_fetch_snapshot_saves_catalog_atomically(tmp_path: Path, monkeypatch) -> None:
+    output = tmp_path / "nested" / "catalog.json"
+    calls = []
+
+    class Result:
+        returncode = 0
+        stdout = '[{"source_external_id": "level_019_american_pond"}]'
+        stderr = ""
+
+    def run(command, **kwargs):
+        calls.append((command, kwargs))
+        return Result()
+
+    monkeypatch.setattr(fetch_waterbodies.subprocess, "run", run)
+    assert fetch_waterbodies.fetch_snapshot(
+        "catalog", url=None, output=output, state_file=tmp_path / "state.json", limit=19,
+    ) == 0
+    assert json.loads(output.read_text(encoding="utf-8"))[0]["source_external_id"] == "level_019_american_pond"
+    assert calls[0][0][0] == fetch_waterbodies.sys.executable
+    assert "--state-file" in calls[0][0]
+    assert "--limit" in calls[0][0]
+
+
+def test_fetch_snapshot_parses_saved_catalog_without_network(tmp_path: Path, monkeypatch) -> None:
+    html = Path(__file__).parent / "fixtures" / "rf4db_waterbodies_sample.html"
+    output = tmp_path / "catalog.json"
+    parser = fetch_waterbodies.parse_rf4db_waterbodies
+    monkeypatch.setattr(fetch_waterbodies, "parse_rf4db_waterbodies", lambda document: parser(document, expected_count=2))
+    assert fetch_waterbodies.fetch_snapshot(
+        "catalog", url=None, html=html, output=output,
+        state_file=tmp_path / "state.json", limit=19,
+    ) == 0
+    payload = json.loads(output.read_text(encoding="utf-8"))
+    assert len(payload) == 2
+    assert payload[0]["source_system"] == "rf4db"
+
+
+def test_fetch_snapshot_reports_invalid_saved_detail_html(tmp_path: Path, capsys) -> None:
+    html = tmp_path / "challenge.html"
+    html.write_text("<html><title>Just a moment...</title></html>", encoding="utf-8")
+    output = tmp_path / "detail.json"
+    assert fetch_waterbodies.fetch_snapshot(
+        "detail", url="https://download.rf4db.com/ru/maps/level_019_american_pond",
+        html=html, output=output, state_file=tmp_path / "state.json", limit=100,
+    ) == 1
+    assert not output.exists()
+    assert "local HTML parse failed" in capsys.readouterr().err
+
+
+def test_fetch_snapshot_does_not_write_failed_fetch(tmp_path: Path, monkeypatch, capsys) -> None:
+    output = tmp_path / "detail.json"
+
+    class Result:
+        returncode = 1
+        stdout = ""
+        stderr = "community source failed: source cooldown is active\n"
+
+    monkeypatch.setattr(fetch_waterbodies.subprocess, "run", lambda *args, **kwargs: Result())
+    assert fetch_waterbodies.fetch_snapshot(
+        "detail", url="https://download.rf4db.com/ru/maps/level_019_american_pond",
+        output=output, state_file=tmp_path / "state.json", limit=100,
+    ) == 1
+    assert not output.exists()
+    assert "cooldown" in capsys.readouterr().err
@@ -0,0 +1,49 @@
+from dataclasses import asdict
+from pathlib import Path
+
+import pytest
+
+from rf4_research.community_sources import (
+    CommunityParseError,
+    parse_rf4db_gear_catalog,
+    parse_rf4db_gear_detail,
+)
+
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def test_gear_catalog_preserves_zero_level_and_unknown_total() -> None:
+    html = (FIXTURES / "rf4db_gear_catalog_sample.html").read_text(encoding="utf-8")
+    rows = parse_rf4db_gear_catalog(html)
+    assert len(rows) == 2
+    assert rows[0].category == "lure"
+    assert rows[0].source_external_id == "spiker-2"
+    assert rows[1].unlock_level == 0
+    assert rows[1].image_url is None
+
+
+def test_gear_catalog_rejects_duplicate_ids() -> None:
+    html = '<article class="gear-card" data-gear-id="same" data-category="bait"><h3 data-name>A</h3></article><article class="gear-card" data-gear-id="same" data-category="bait"><h3 data-name>B</h3></article>'
+    with pytest.raises(CommunityParseError, match="duplicate"):
+        parse_rf4db_gear_catalog(html)
+
+
+def test_gear_detail_distinguishes_value_not_applicable_and_missing() -> None:
+    html = (FIXTURES / "rf4db_gear_detail_sample.html").read_text(encoding="utf-8")
+    detail = parse_rf4db_gear_detail(html, source_url="https://rf4db.com/ru/wiki/lures/spiker-2")
+    attributes = {item.key: asdict(item) for item in detail.attributes}
+    assert attributes["weight"] == {"key": "weight", "state": "value", "value": 0, "source_text": "0"}
+    assert attributes["floating"]["state"] == "not_applicable"
+    assert attributes["floating"]["value"] is None
+    assert attributes["target_fish"]["state"] == "missing"
+    assert detail.variants == ("Spiker #2 01-015",)
+    assert detail.rig_types == ("Spinning",)
+
+
+def test_gear_detail_rejects_unknown_category() -> None:
+    with pytest.raises(CommunityParseError, match="category"):
+        parse_rf4db_gear_detail(
+            '<main data-gear-detail data-category="unknown-category"><h1>Test</h1></main>',
+            source_url="https://rf4db.com/ru/wiki/gear/test",
+        )