feat: add offline waterbody source crosswalk

rf4_research/community_cli.py

@@ -18,9 +18,11 @@ from .community_sources import (
     parse_rf4db_waterbodies,
     parse_rf4db_waterbody_detail,
     parse_rf4map_point,
+    parse_rf4map_waterbodies,
     parse_rf4posts_spot,
     parse_rf4stat_fishing,
     parse_rf4stat_posts,
+    parse_rf4stat_waterbodies,
 )
 
 
@@ -29,6 +31,8 @@ SOURCES = {
     "rf4db-waterbodies": ("https://rf4db.com/ru/maps", parse_rf4db_waterbodies),
     "rf4stat-fishing": ("https://rf4-stat.ru/fishing/", parse_rf4stat_fishing),
     "rf4stat-posts": ("https://rf4-stat.ru/posts/", parse_rf4stat_posts),
+    "rf4map-waterbodies": ("https://rf4map.ru/lakes", parse_rf4map_waterbodies),
+    "rf4stat-locations": ("https://en.rf4-stat.ru/locations/", parse_rf4stat_waterbodies),
 }
 DETAIL_SOURCES = {
     "rf4db-waterbody": parse_rf4db_waterbody_detail,
@@ -40,7 +44,7 @@ MIN_FETCH_INTERVAL_SECONDS = 30 * 60
 DEFAULT_STATE_FILE = Path(".cache/community-fetch-state.json")
 ALLOWED_HOSTS = frozenset({
     "download.rf4db.com", "rf4db.com", "oss.rf4db.com",
-    "rf4-stat.ru",
+    "rf4-stat.ru", "en.rf4-stat.ru",
     "rf4map.ru", "gw.rf4map.ru", "hb.ru-msk.vkcloud-storage.ru",
     "rf4-posts.com",
     "rf4game.de", "rf4game.ru",
@@ -111,6 +115,8 @@ def fetch_site_key(url: str) -> str:
         hostname = hostname[4:]
     elif hostname.startswith("cdn."):
         hostname = hostname[4:]
+    elif hostname.endswith(".rf4-stat.ru"):
+        hostname = "rf4-stat.ru"
     if hostname == "gw.rf4map.ru":
         hostname = "rf4map.ru"
     if not hostname:
@@ -304,6 +310,7 @@ def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description="Fetch one authorized RF4 community source page")
     parser.add_argument("source", choices=(*SOURCES, *DETAIL_SOURCES))
     parser.add_argument("--url", help="Override the configured public page URL")
+    parser.add_argument("--html", type=Path, help="Parse a previously saved HTML file without network or cooldown")
     parser.add_argument("--limit", type=int, default=100, choices=range(1, 501), metavar="1..500")
     parser.add_argument(
         "--state-file", type=Path,
@@ -315,18 +322,23 @@ def main(argv: list[str] | None = None) -> int:
         help="Reserve the shared site cooldown and stop before making an HTTP request",
     )
     args = parser.parse_args(argv)
+    if args.html and args.reserve_only:
+        parser.error("--html cannot be combined with --reserve-only")
     if args.source in DETAIL_SOURCES and not args.url:
         parser.error(f"--url is required for {args.source}")
     default_url, parse = SOURCES.get(args.source, (None, DETAIL_SOURCES.get(args.source)))
     url = args.url or default_url
     try:
-        site_key = fetch_site_key(url)
-        # Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too.
-        check_and_reserve(site_key, state_file=args.state_file)
-        if args.reserve_only:
-            print(json.dumps({"reserved": True, "source": args.source, "site_key": site_key}, ensure_ascii=False))
-            return 0
-        html = fetch_html(url)
+        if args.html:
+            html = args.html.read_text(encoding="utf-8")
+        else:
+            site_key = fetch_site_key(url)
+            # Single atomic check-and-reserve before network I/O: failed attempts count toward the limit too.
+            check_and_reserve(site_key, state_file=args.state_file)
+            if args.reserve_only:
+                print(json.dumps({"reserved": True, "source": args.source, "site_key": site_key}, ensure_ascii=False))
+                return 0
+            html = fetch_html(url)
         parsed = (
             parse(html, source_url=url)
             if args.source in DETAIL_SOURCES or args.source == "rf4db-waterbodies"

rf4_research/community_sources.py

@@ -127,6 +127,31 @@ class RF4DBWaterbodyDetail:
     point_urls: tuple[str, ...]
 
 
+@dataclass(frozen=True, slots=True)
+class RF4MapWaterbodyCandidate:
+    """Secondary waterbody directory facts; never a canonical catalog row."""
+
+    source_system: str
+    source_external_id: str
+    source_url: str
+    name: str
+    fish_species_count: int | None
+
+
+@dataclass(frozen=True, slots=True)
+class RF4StatWaterbodyCandidate:
+    """Secondary RF4-STAT location metrics for crosswalk and comparison."""
+
+    source_system: str
+    source_external_id: str
+    source_url: str
+    name: str
+    catches_count: int | None
+    fish_species_count: int | None
+    bait_count: int | None
+    spot_count: int | None
+
+
 def _text(node: Tag | None) -> str:
     return " ".join(node.get_text(" ", strip=True).split()) if node else ""
 
@@ -254,6 +279,88 @@ def parse_rf4db_waterbodies(
     return result
 
 
+def parse_rf4map_waterbodies(
+    html: str, *, base_url: str = "https://rf4map.ru/lakes",
+) -> list[RF4MapWaterbodyCandidate]:
+    """Parse the public RF4MAP directory as secondary crosswalk candidates."""
+    soup = BeautifulSoup(html, "html.parser")
+    result: list[RF4MapWaterbodyCandidate] = []
+    seen: set[str] = set()
+    for link in soup.select('a[href^="/lakes/"]'):
+        href = link.get("href")
+        external_id = _key(href)
+        if not isinstance(href, str) or not external_id or external_id in seen:
+            continue
+        label = _text(link)
+        fish_match = re.search(r"(\d+)\s+вид(?:а|ов)?\s+рыб", label, flags=re.I)
+        name = re.sub(r"\s*\d+\s+вид(?:а|ов)?\s+рыб(?:ы)?\s*$", "", label, flags=re.I).strip()
+        name = re.sub(r"^\+\s*\d+\s+", "", name).strip()
+        if not name:
+            continue
+        seen.add(external_id)
+        result.append(RF4MapWaterbodyCandidate(
+            source_system="rf4map-waterbodies",
+            source_external_id=external_id,
+            source_url=urljoin(base_url, href),
+            name=name,
+            fish_species_count=int(fish_match.group(1)) if fish_match else None,
+        ))
+    if not result:
+        raise CommunityParseError("RF4MAP waterbody directory not found")
+    return result
+
+
+def parse_rf4stat_waterbodies(
+    html: str, *, base_url: str = "https://en.rf4-stat.ru/locations/",
+) -> list[RF4StatWaterbodyCandidate]:
+    """Parse public RF4-STAT location cards without treating metrics as canonical facts."""
+    soup = BeautifulSoup(html, "html.parser")
+    result: list[RF4StatWaterbodyCandidate] = []
+    seen: set[str] = set()
+    for link in soup.select('a[href*="/locations/location/"]'):
+        href = link.get("href")
+        external_id = _key(href)
+        if not isinstance(href, str) or not external_id or external_id in seen:
+            continue
+        container = link.find_parent(["article", "li"]) or link.parent
+        text = _text(container)
+        name = _text(link)
+        metric_texts = [
+            value for node in container.select("span, div, small, strong")
+            if (value := _text(node)) and value != name
+        ]
+        metric_texts.append(text)
+        if not name:
+            continue
+
+        def metric(label: str) -> int | None:
+            patterns = (
+                rf"(?:{label})\s*:\s*([0-9]+(?: [0-9]{{3}})*)",
+                rf"([0-9]+(?: [0-9]{{3}})*)\s+(?:{label})",
+            )
+            for candidate in metric_texts:
+                for pattern in patterns:
+                    match = re.search(pattern, candidate, flags=re.I)
+                    if match:
+                        return int(match.group(1).replace(" ", ""))
+            return None
+
+        seen.add(external_id)
+        result.append(RF4StatWaterbodyCandidate(
+            source_system="rf4stat-locations",
+            source_external_id=external_id,
+            source_url=urljoin(base_url, href),
+            name=name,
+            catches_count=metric(r"catches|улов"),
+            fish_species_count=metric(r"kinds of fish|вид(?:а|ов) рыб"),
+            bait_count=metric(r"baits|нажив"),
+            spot_count=metric(r"spots|точ(?:ек|ки)"),
+        ))
+    if not result:
+        raise CommunityParseError("RF4-STAT waterbody directory not found")
+    return result
+
+
 def _gear_level(value: str | None) -> int | None:
     text = (value or "").strip()
     if not text:

rf4_research/waterbody_crosswalk.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from typing import Iterable, Protocol
 
 from .media_assets import normalize_entity_label
 
@@ -21,6 +22,14 @@ class WaterbodyIdentity:
     name: str
 
 
+class WaterbodyCandidate(Protocol):
+    """Minimum shape shared by parsed secondary-source candidates."""
+
+    source_system: str
+    source_external_id: str
+    name: str
+
+
 @dataclass(frozen=True, slots=True)
 class CrosswalkSuggestion:
     source_system: str
@@ -30,6 +39,24 @@ class CrosswalkSuggestion:
     canonical_keys: tuple[str, ...]
 
 
+def identities_from_candidates(
+    candidates: Iterable[WaterbodyCandidate],
+) -> list[WaterbodyIdentity]:
+    """Convert parser rows without selecting a canonical key.
+
+    Metrics, URLs, and other source facts stay outside the crosswalk
+    decision and must not influence automatic canonical matching.
+    """
+    return [
+        WaterbodyIdentity(
+            source_system=row.source_system,
+            external_id=row.source_external_id,
+            name=row.name,
+        )
+        for row in candidates
+    ]
+
+
 def suggest_waterbody_crosswalk(
     canonical: list[CanonicalWaterbody], identities: list[WaterbodyIdentity],
 ) -> list[CrosswalkSuggestion]: