sync

apps/api/app/community_importer.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import json
+import hashlib
+import re
 from datetime import datetime, timezone
 from typing import Any, Iterable
 from urllib.parse import urlparse
@@ -26,12 +28,153 @@ SOURCE_HOSTS = {
     "rf4map": {"rf4map.ru"},
     "rf4posts-spot": {"rf4-posts.com"},
 }
+COORDINATE_PRECISIONS = frozenset({"exact", "approximate", "area", "missing"})
 
 
 class CommunityImportError(ValueError):
     pass
 
 
+def upsert_waterbody_catalog(
+    session: Session, rows: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
+) -> tuple[int, int]:
+    """Apply a validated canonical waterbody snapshot without destructive sync.
+
+    Rows are matched by the RF4DB source identity first and by an exact existing
+    name second. Missing rows are deliberately left untouched: an incomplete
+    response must never withdraw a previously known waterbody.
+    """
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    created = updated = 0
+    for raw in rows:
+        payload = _json_payload(raw)
+        if payload.get("source_system") != "rf4db":
+            raise CommunityImportError("waterbody catalog requires source_system=rf4db")
+        external_id = _required(payload, "source_external_id", 200)
+        name = _required(payload, "name", 200)
+        source_url = _required(payload, "source_url", 2000)
+        parsed_url = urlparse(source_url)
+        if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
+            raise CommunityImportError("waterbody source_url does not match rf4db")
+        unlock_level = _integer(payload.get("unlock_level"), minimum=0, maximum=1_000)
+        unlock_label = _required(payload, "unlock_label", 50)
+        fish_species_count = _integer(payload.get("fish_species_count"), minimum=0, maximum=10_000)
+        if fish_species_count is None:
+            raise CommunityImportError("invalid fish_species_count")
+
+        item = session.scalar(select(Waterbody).where(
+            Waterbody.source_system == "rf4db",
+            Waterbody.source_external_id == external_id,
+        ))
+        if item is None:
+            item = session.scalar(select(Waterbody).where(Waterbody.name_ru == name))
+        if item is None:
+            item = Waterbody(
+                slug=_catalog_slug(session, name, external_id),
+                name_ru=name,
+                unlock_level=unlock_level,
+            )
+            session.add(item)
+            created += 1
+        else:
+            updated += 1
+        item.name_ru = name
+        item.unlock_level = unlock_level
+        item.fish_species_count = fish_species_count
+        item.source_system = "rf4db"
+        item.source_external_id = external_id
+        item.source_url = source_url
+        item.source_checked_at = fetched_at
+    session.commit()
+    return created, updated
+
+
+def update_waterbody_detail(
+    session: Session, detail: dict[str, Any], *, fetched_at: datetime | None = None,
+) -> bool:
+    """Persist one complete RF4DB detail snapshot without assigning media roles."""
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    payload = _validate_waterbody_detail(detail)
+    _apply_waterbody_detail(session, payload, fetched_at=fetched_at)
+    session.commit()
+    return True
+
+
+def update_waterbody_details(
+    session: Session, details: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
+) -> tuple[int, int]:
+    """Validate and apply a detail batch in one transaction."""
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    payloads = [_validate_waterbody_detail(detail) for detail in details]
+    external_ids = [str(payload["source_external_id"]) for payload in payloads]
+    if len(external_ids) != len(set(external_ids)):
+        raise CommunityImportError("waterbody detail batch contains duplicate source identities")
+    updated = 0
+    for payload in payloads:
+        _apply_waterbody_detail(session, payload, fetched_at=fetched_at)
+        updated += 1
+    session.commit()
+    return 0, updated
+
+
+def _validate_waterbody_detail(detail: dict[str, Any]) -> dict[str, Any]:
+    payload = _json_payload(detail)
+    if payload.get("source_system") != "rf4db":
+        raise CommunityImportError("waterbody detail requires source_system=rf4db")
+    external_id = _required(payload, "source_external_id", 200)
+    source_url = _required(payload, "source_url", 2000)
+    parsed_url = urlparse(source_url)
+    if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
+        raise CommunityImportError("waterbody detail source_url does not match rf4db")
+    _required(payload, "name", 200)
+    _optional(payload, "description", 20_000)
+    _string_list(payload, "aliases", 100, 200)
+    _string_list(payload, "fish_species", 10_000, 200)
+    _string_list(payload, "image_urls", 100, 2_000)
+    _string_list(payload, "point_urls", 10_000, 2_000)
+    return payload
+
+
+def _apply_waterbody_detail(session: Session, payload: dict[str, Any], *, fetched_at: datetime) -> None:
+    external_id = str(payload["source_external_id"])
+    source_url = str(payload["source_url"])
+    item = session.scalar(select(Waterbody).where(
+        Waterbody.source_system == "rf4db", Waterbody.source_external_id == external_id,
+    ))
+    if item is None:
+        raise CommunityImportError("waterbody detail has no imported catalog identity")
+    item.description = _optional(payload, "description", 20_000)
+    item.source_aliases = _string_list(payload, "aliases", 100, 200)
+    item.source_fish_species = _string_list(payload, "fish_species", 10_000, 200)
+    item.source_image_urls = _string_list(payload, "image_urls", 100, 2_000)
+    item.source_point_urls = _string_list(payload, "point_urls", 10_000, 2_000)
+    item.source_url = source_url
+    item.source_checked_at = fetched_at
+
+
+def _catalog_slug(session: Session, name: str, external_id: str) -> str:
+    base = re.sub(r"[^a-z0-9а-яё]+", "-", name.casefold(), flags=re.IGNORECASE).strip("-")
+    base = base or "waterbody"
+    candidate = base[:100]
+    if session.scalar(select(Waterbody.id).where(Waterbody.slug == candidate)) is None:
+        return candidate
+    suffix = hashlib.sha256(external_id.encode()).hexdigest()[:10]
+    return f"{base[:89]}-{suffix}"
+
+
+def _string_list(payload: dict[str, Any], key: str, max_items: int, max_length: int) -> list[str]:
+    value = payload.get(key)
+    if not isinstance(value, list) or len(value) > max_items:
+        raise CommunityImportError(f"invalid {key}")
+    result = []
+    for item in value:
+        text = str(item).strip()
+        if not text or len(text) > max_length:
+            raise CommunityImportError(f"invalid {key}")
+        result.append(text)
+    return list(dict.fromkeys(result))
+
+
 def stage_observations(
     session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
 ) -> tuple[int, int]:
@@ -65,6 +208,8 @@ def stage_observations(
             "waterbody_external_id": _optional(payload, "waterbody_external_id", 200),
             "x": _integer(payload.get("x"), maximum=10_000),
             "y": _integer(payload.get("y"), maximum=10_000),
+            "coordinate_raw": _coordinate_raw(payload),
+            "coordinate_precision": _coordinate_precision(payload),
             "weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000),
             "published_at": _datetime(payload.get("published_at")),
             "last_seen_at": fetched_at, "payload": payload,
@@ -83,6 +228,7 @@ def stage_observations(
             changed = any(getattr(observation, key) != values[key] for key in (
                 "source_url", "fish_name", "fish_external_id", "waterbody_name",
                 "waterbody_external_id", "x", "y", "weight_g",
+                "coordinate_raw", "coordinate_precision",
             )) or observation.payload != payload
             if observation.status != "rejected" and changed and observation.catch_report is not None:
                 observation.catch_report.moderation_status = ModerationStatus.pending
@@ -186,6 +332,27 @@ def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None:
     return value or None
 
 
+def _coordinate_raw(payload: dict[str, Any]) -> str | None:
+    value = str(payload.get("coordinate_raw") or "").strip()
+    if len(value) > 200:
+        raise CommunityImportError("invalid coordinate_raw")
+    if value:
+        return value
+    x, y = payload.get("x"), payload.get("y")
+    return f"{x}:{y}" if isinstance(x, int) and isinstance(y, int) else None
+
+
+def _coordinate_precision(payload: dict[str, Any]) -> str:
+    value = str(payload.get("coordinate_precision") or "").strip().casefold()
+    if not value:
+        return "exact" if isinstance(payload.get("x"), int) and isinstance(payload.get("y"), int) else "missing"
+    if value not in COORDINATE_PRECISIONS:
+        raise CommunityImportError("invalid coordinate_precision")
+    if value == "exact" and (not isinstance(payload.get("x"), int) or not isinstance(payload.get("y"), int)):
+        raise CommunityImportError("exact coordinates require x and y")
+    return value
+
+
 def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None:
     if value is None:
         return None