sync

apps/api/app/activity.py

@@ -61,6 +61,9 @@ def activity_rows(
             confidence = min(confidence, 50)
         elif len(players) == 2:
             confidence = min(confidence, 65)
+        coordinate_precisions = {_coordinate_precision(item) for item in items}
+        coordinate_precision = max(coordinate_precisions, key=_precision_rank)
+        coordinate_sources = sorted({_source_system(item) for item in items})
         latest = max(_aware(r.reported_at) for r in items)
         baits = Counter(r.bait.name for r in items if r.bait)
         freshness_text = _freshness_text(now - latest)
@@ -74,7 +77,9 @@ def activity_rows(
             max_weight_g=max(r.weight_g for r in items), last_confirmed_at=latest,
             activity_score=activity, confidence_score=confidence,
             explanation=_explanation(len(items), len(players), freshness_text, activity, confidence),
-            sources=sorted({_source_system(item) for item in items}),
+            sources=coordinate_sources,
+            coordinate_precision=coordinate_precision,
+            coordinate_sources=coordinate_sources,
         ))
     return sorted(result, key=lambda row: (row.activity_score, row.last_confirmed_at), reverse=True)
 
@@ -90,6 +95,16 @@ def _source_system(report: CatchReport) -> str:
     return "manual-import"
 
 
+def _coordinate_precision(report: CatchReport) -> str:
+    provenance = (report.raw_payload or {}).get("provenance", {})
+    value = provenance.get("coordinate_precision") if isinstance(provenance, dict) else None
+    return value if value in {"exact", "approximate", "area", "missing"} else "exact"
+
+
+def _precision_rank(value: str) -> int:
+    return {"exact": 0, "approximate": 1, "area": 2, "missing": 3}[value]
+
+
 def _aware(value: datetime) -> datetime:
     return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
 

apps/api/app/catalog_audit.py

@@ -15,3 +15,28 @@ def audit_catalog(db: Session) -> dict[str, int]:
         "incomplete_published_staging": db.scalar(select(func.count()).select_from(ExternalObservation).where(ExternalObservation.status == "published", or_(ExternalObservation.fish_id.is_(None), ExternalObservation.waterbody_id.is_(None), ExternalObservation.x.is_(None), ExternalObservation.y.is_(None), ExternalObservation.weight_g.is_(None), ExternalObservation.catch_report_id.is_(None)))) or 0,
     }
     return {"fishes": count(Fish), "waterbodies": count(Waterbody), "reports": count(CatchReport), "staging": count(ExternalObservation), **failures, "failures": sum(failures.values())}
+
+
+def audit_waterbody_catalog(db: Session, expected_ids: set[str]) -> dict:
+    """Check a verified RF4DB snapshot without withdrawing legacy rows."""
+    rows = list(db.scalars(select(Waterbody).where(Waterbody.source_system == "rf4db")))
+    observed_ids = [str(row.source_external_id) for row in rows if row.source_external_id]
+    observed = set(observed_ids)
+    duplicate_ids = sorted({item for item in observed_ids if observed_ids.count(item) > 1})
+    missing = sorted(expected_ids - observed)
+    unexpected = sorted(observed - expected_ids)
+    provenance_issues = sorted(
+        str(row.source_external_id)
+        for row in rows
+        if not row.source_external_id or not row.source_url or not row.source_checked_at
+    )
+    failures = len(missing) + len(duplicate_ids) + len(provenance_issues)
+    return {
+        "expected": len(expected_ids),
+        "observed": len(observed),
+        "missing_source_external_ids": missing,
+        "unexpected_source_external_ids": unexpected,
+        "duplicate_source_external_ids": duplicate_ids,
+        "provenance_issues": provenance_issues,
+        "failures": failures,
+    }

apps/api/app/cli.py

@@ -8,10 +8,10 @@ from dataclasses import asdict
 from .config import settings
 from .database import SessionLocal
 from .importer import import_records
-from .community_importer import stage_observations
+from .community_importer import stage_observations, update_waterbody_detail, update_waterbody_details, upsert_waterbody_catalog
 from .retention import RetentionPolicy, apply_retention
 from .storage import delete_screenshot
-from .catalog_audit import audit_catalog
+from .catalog_audit import audit_catalog, audit_waterbody_catalog
 from .community_scheduler import run_source, configured_sources
 
 # Static registry for argparse choices — no DB required for --help
@@ -34,11 +34,20 @@ def main() -> int:
     community = sub.add_parser("stage-community-json")
     community.add_argument("--input", default="-", help="JSON array path or - for stdin")
     community.add_argument("--limit", type=int, default=500)
+    waterbodies = sub.add_parser("import-waterbody-catalog")
+    waterbodies.add_argument("--input", required=True, help="JSON snapshot path or - for stdin")
+    waterbodies.add_argument("--limit", type=int, default=100)
+    detail = sub.add_parser("import-waterbody-detail")
+    detail.add_argument("--input", required=True, help="JSON detail snapshot path")
+    details = sub.add_parser("import-waterbody-details")
+    details.add_argument("--input", required=True, help="JSON array of detail snapshots")
     fetch_community = sub.add_parser("fetch-community")
     fetch_community.add_argument("source", choices=STATIC_SOURCE_CHOICES)
     cleanup = sub.add_parser("cleanup-retention")
     cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run")
     sub.add_parser("audit-catalog")
+    waterbody_audit = sub.add_parser("audit-waterbody-catalog")
+    waterbody_audit.add_argument("--input", required=True, help="JSON snapshot path")
     args = parser.parse_args()
     with SessionLocal() as session:
         if args.command == "import-records":
@@ -57,6 +66,43 @@ def main() -> int:
                 parser.error("input must be a JSON array")
             created, updated = stage_observations(session, payload[:args.limit])
             print(f"staged: created={created} updated={updated}")
+        elif args.command == "import-waterbody-catalog":
+            if not 1 <= args.limit <= 500:
+                parser.error("--limit must be between 1 and 500")
+            stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
+            try:
+                snapshot = json.load(stream)
+            finally:
+                if stream is not sys.stdin:
+                    stream.close()
+            if isinstance(snapshot, dict):
+                payload = snapshot.get("items")
+                source_system = snapshot.get("source_system")
+                if isinstance(payload, list) and isinstance(source_system, str):
+                    payload = [
+                        {"source_system": source_system, **item}
+                        for item in payload if isinstance(item, dict)
+                    ]
+            else:
+                payload = snapshot
+            if not isinstance(payload, list):
+                parser.error("input must be a JSON array or an object with an items array")
+            created, updated = upsert_waterbody_catalog(session, payload[:args.limit])
+            print(f"waterbodies: created={created} updated={updated}")
+        elif args.command == "import-waterbody-detail":
+            with open(args.input, encoding="utf-8") as stream:
+                payload = json.load(stream)
+            if not isinstance(payload, dict):
+                parser.error("input must be a JSON object")
+            update_waterbody_detail(session, payload)
+            print(f"waterbody detail: updated={payload.get('source_external_id', 'unknown')}")
+        elif args.command == "import-waterbody-details":
+            with open(args.input, encoding="utf-8") as stream:
+                payload = json.load(stream)
+            if not isinstance(payload, list):
+                parser.error("input must be a JSON array")
+            created, updated = update_waterbody_details(session, payload)
+            print(f"waterbody details: created={created} updated={updated}")
         elif args.command == "fetch-community":
             # A09: Verify source is enabled at runtime (not just in static choices)
             enabled = configured_sources()
@@ -76,6 +122,24 @@ def main() -> int:
             )
             counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot)
             print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False))
+        elif args.command == "audit-waterbody-catalog":
+            stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
+            try:
+                snapshot = json.load(stream)
+            finally:
+                if stream is not sys.stdin:
+                    stream.close()
+            items = snapshot.get("items") if isinstance(snapshot, dict) else snapshot
+            if not isinstance(items, list):
+                parser.error("input must be a JSON array or an object with an items array")
+            expected_ids = {
+                str(item["source_external_id"])
+                for item in items
+                if isinstance(item, dict) and item.get("source_external_id")
+            }
+            result = audit_waterbody_catalog(session, expected_ids)
+            print(json.dumps(result, ensure_ascii=False))
+            return 1 if result["failures"] else 0
         else:
             result = audit_catalog(session)
             print(json.dumps(result, ensure_ascii=False))

apps/api/app/community_importer.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import json
+import hashlib
+import re
 from datetime import datetime, timezone
 from typing import Any, Iterable
 from urllib.parse import urlparse
@@ -26,12 +28,153 @@ SOURCE_HOSTS = {
     "rf4map": {"rf4map.ru"},
     "rf4posts-spot": {"rf4-posts.com"},
 }
+COORDINATE_PRECISIONS = frozenset({"exact", "approximate", "area", "missing"})
 
 
 class CommunityImportError(ValueError):
     pass
 
 
+def upsert_waterbody_catalog(
+    session: Session, rows: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
+) -> tuple[int, int]:
+    """Apply a validated canonical waterbody snapshot without destructive sync.
+
+    Rows are matched by the RF4DB source identity first and by an exact existing
+    name second. Missing rows are deliberately left untouched: an incomplete
+    response must never withdraw a previously known waterbody.
+    """
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    created = updated = 0
+    for raw in rows:
+        payload = _json_payload(raw)
+        if payload.get("source_system") != "rf4db":
+            raise CommunityImportError("waterbody catalog requires source_system=rf4db")
+        external_id = _required(payload, "source_external_id", 200)
+        name = _required(payload, "name", 200)
+        source_url = _required(payload, "source_url", 2000)
+        parsed_url = urlparse(source_url)
+        if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
+            raise CommunityImportError("waterbody source_url does not match rf4db")
+        unlock_level = _integer(payload.get("unlock_level"), minimum=0, maximum=1_000)
+        unlock_label = _required(payload, "unlock_label", 50)
+        fish_species_count = _integer(payload.get("fish_species_count"), minimum=0, maximum=10_000)
+        if fish_species_count is None:
+            raise CommunityImportError("invalid fish_species_count")
+
+        item = session.scalar(select(Waterbody).where(
+            Waterbody.source_system == "rf4db",
+            Waterbody.source_external_id == external_id,
+        ))
+        if item is None:
+            item = session.scalar(select(Waterbody).where(Waterbody.name_ru == name))
+        if item is None:
+            item = Waterbody(
+                slug=_catalog_slug(session, name, external_id),
+                name_ru=name,
+                unlock_level=unlock_level,
+            )
+            session.add(item)
+            created += 1
+        else:
+            updated += 1
+        item.name_ru = name
+        item.unlock_level = unlock_level
+        item.fish_species_count = fish_species_count
+        item.source_system = "rf4db"
+        item.source_external_id = external_id
+        item.source_url = source_url
+        item.source_checked_at = fetched_at
+    session.commit()
+    return created, updated
+
+
+def update_waterbody_detail(
+    session: Session, detail: dict[str, Any], *, fetched_at: datetime | None = None,
+) -> bool:
+    """Persist one complete RF4DB detail snapshot without assigning media roles."""
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    payload = _validate_waterbody_detail(detail)
+    _apply_waterbody_detail(session, payload, fetched_at=fetched_at)
+    session.commit()
+    return True
+
+
+def update_waterbody_details(
+    session: Session, details: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
+) -> tuple[int, int]:
+    """Validate and apply a detail batch in one transaction."""
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    payloads = [_validate_waterbody_detail(detail) for detail in details]
+    external_ids = [str(payload["source_external_id"]) for payload in payloads]
+    if len(external_ids) != len(set(external_ids)):
+        raise CommunityImportError("waterbody detail batch contains duplicate source identities")
+    updated = 0
+    for payload in payloads:
+        _apply_waterbody_detail(session, payload, fetched_at=fetched_at)
+        updated += 1
+    session.commit()
+    return 0, updated
+
+
+def _validate_waterbody_detail(detail: dict[str, Any]) -> dict[str, Any]:
+    payload = _json_payload(detail)
+    if payload.get("source_system") != "rf4db":
+        raise CommunityImportError("waterbody detail requires source_system=rf4db")
+    external_id = _required(payload, "source_external_id", 200)
+    source_url = _required(payload, "source_url", 2000)
+    parsed_url = urlparse(source_url)
+    if parsed_url.scheme != "https" or parsed_url.hostname not in {"rf4db.com", "www.rf4db.com"}:
+        raise CommunityImportError("waterbody detail source_url does not match rf4db")
+    _required(payload, "name", 200)
+    _optional(payload, "description", 20_000)
+    _string_list(payload, "aliases", 100, 200)
+    _string_list(payload, "fish_species", 10_000, 200)
+    _string_list(payload, "image_urls", 100, 2_000)
+    _string_list(payload, "point_urls", 10_000, 2_000)
+    return payload
+
+
+def _apply_waterbody_detail(session: Session, payload: dict[str, Any], *, fetched_at: datetime) -> None:
+    external_id = str(payload["source_external_id"])
+    source_url = str(payload["source_url"])
+    item = session.scalar(select(Waterbody).where(
+        Waterbody.source_system == "rf4db", Waterbody.source_external_id == external_id,
+    ))
+    if item is None:
+        raise CommunityImportError("waterbody detail has no imported catalog identity")
+    item.description = _optional(payload, "description", 20_000)
+    item.source_aliases = _string_list(payload, "aliases", 100, 200)
+    item.source_fish_species = _string_list(payload, "fish_species", 10_000, 200)
+    item.source_image_urls = _string_list(payload, "image_urls", 100, 2_000)
+    item.source_point_urls = _string_list(payload, "point_urls", 10_000, 2_000)
+    item.source_url = source_url
+    item.source_checked_at = fetched_at
+
+
+def _catalog_slug(session: Session, name: str, external_id: str) -> str:
+    base = re.sub(r"[^a-z0-9а-яё]+", "-", name.casefold(), flags=re.IGNORECASE).strip("-")
+    base = base or "waterbody"
+    candidate = base[:100]
+    if session.scalar(select(Waterbody.id).where(Waterbody.slug == candidate)) is None:
+        return candidate
+    suffix = hashlib.sha256(external_id.encode()).hexdigest()[:10]
+    return f"{base[:89]}-{suffix}"
+
+
+def _string_list(payload: dict[str, Any], key: str, max_items: int, max_length: int) -> list[str]:
+    value = payload.get(key)
+    if not isinstance(value, list) or len(value) > max_items:
+        raise CommunityImportError(f"invalid {key}")
+    result = []
+    for item in value:
+        text = str(item).strip()
+        if not text or len(text) > max_length:
+            raise CommunityImportError(f"invalid {key}")
+        result.append(text)
+    return list(dict.fromkeys(result))
+
+
 def stage_observations(
     session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
 ) -> tuple[int, int]:
@@ -65,6 +208,8 @@ def stage_observations(
             "waterbody_external_id": _optional(payload, "waterbody_external_id", 200),
             "x": _integer(payload.get("x"), maximum=10_000),
             "y": _integer(payload.get("y"), maximum=10_000),
+            "coordinate_raw": _coordinate_raw(payload),
+            "coordinate_precision": _coordinate_precision(payload),
             "weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000),
             "published_at": _datetime(payload.get("published_at")),
             "last_seen_at": fetched_at, "payload": payload,
@@ -83,6 +228,7 @@ def stage_observations(
             changed = any(getattr(observation, key) != values[key] for key in (
                 "source_url", "fish_name", "fish_external_id", "waterbody_name",
                 "waterbody_external_id", "x", "y", "weight_g",
+                "coordinate_raw", "coordinate_precision",
             )) or observation.payload != payload
             if observation.status != "rejected" and changed and observation.catch_report is not None:
                 observation.catch_report.moderation_status = ModerationStatus.pending
@@ -186,6 +332,27 @@ def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None:
     return value or None
 
 
+def _coordinate_raw(payload: dict[str, Any]) -> str | None:
+    value = str(payload.get("coordinate_raw") or "").strip()
+    if len(value) > 200:
+        raise CommunityImportError("invalid coordinate_raw")
+    if value:
+        return value
+    x, y = payload.get("x"), payload.get("y")
+    return f"{x}:{y}" if isinstance(x, int) and isinstance(y, int) else None
+
+
+def _coordinate_precision(payload: dict[str, Any]) -> str:
+    value = str(payload.get("coordinate_precision") or "").strip().casefold()
+    if not value:
+        return "exact" if isinstance(payload.get("x"), int) and isinstance(payload.get("y"), int) else "missing"
+    if value not in COORDINATE_PRECISIONS:
+        raise CommunityImportError("invalid coordinate_precision")
+    if value == "exact" and (not isinstance(payload.get("x"), int) or not isinstance(payload.get("y"), int)):
+        raise CommunityImportError("exact coordinates require x and y")
+    return value
+
+
 def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None:
     if value is None:
         return None

apps/api/app/community_review.py

@@ -104,6 +104,8 @@ def publish_observation(session: Session, observation: ExternalObservation) -> C
                 "external_observation_id": str(observation.id),
                 "source_system": observation.source_system,
                 "source_external_id": observation.source_external_id,
+                "coordinate_raw": observation.coordinate_raw,
+                "coordinate_precision": observation.coordinate_precision,
             },
             "original": observation.payload,
         },

apps/api/app/models.py

@@ -49,6 +49,16 @@ class Waterbody(Base):
     slug: Mapped[str] = mapped_column(String(100), unique=True)
     name_ru: Mapped[str] = mapped_column(String(200), unique=True)
     unlock_level: Mapped[int | None]
+    fish_species_count: Mapped[int | None]
+    source_system: Mapped[str | None] = mapped_column(String(50))
+    source_external_id: Mapped[str | None] = mapped_column(String(200))
+    source_url: Mapped[str | None] = mapped_column(Text)
+    description: Mapped[str | None] = mapped_column(Text)
+    source_aliases: Mapped[list[str] | None] = mapped_column(JSON)
+    source_fish_species: Mapped[list[str] | None] = mapped_column(JSON)
+    source_image_urls: Mapped[list[str] | None] = mapped_column(JSON)
+    source_point_urls: Mapped[list[str] | None] = mapped_column(JSON)
+    source_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
 
 
 class Bait(Base):
@@ -187,6 +197,8 @@ class ExternalObservation(Base):
     waterbody_external_id: Mapped[str | None] = mapped_column(String(200))
     x: Mapped[int | None]
     y: Mapped[int | None]
+    coordinate_raw: Mapped[str | None] = mapped_column(String(200))
+    coordinate_precision: Mapped[str] = mapped_column(String(20), default="missing")
     weight_g: Mapped[int | None]
     published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
     first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))

apps/api/app/routers/activity.py

@@ -68,7 +68,15 @@ def spot_detail(spot_id: UUID, db: Db) -> SpotOut:
     def count_since(delta: timedelta) -> int:
         return sum(aware(report.reported_at) >= now - delta for report in reports)
 
-    return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)])
+    provenance = [
+        (report.raw_payload or {}).get("provenance", {})
+        for report in reports
+        if isinstance((report.raw_payload or {}).get("provenance", {}), dict)
+    ]
+    precisions = [item.get("coordinate_precision") for item in provenance]
+    precision = max((value for value in precisions if value in {"exact", "approximate", "area", "missing"}), key={"exact": 0, "approximate": 1, "area": 2, "missing": 3}.get, default="exact")
+    sources = sorted({str(item.get("source_system")) for item in provenance if item.get("source_system")}) or ["players"]
+    return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)], coordinate_precision=precision, coordinate_sources=sources)
 
 
 def _report_source(report: CatchReport) -> str:

apps/api/app/schemas.py

@@ -20,6 +20,16 @@ class WaterbodyOut(BaseModel):
     slug: str
     name_ru: str
     unlock_level: int | None
+    fish_species_count: int | None
+    source_system: str | None
+    source_external_id: str | None
+    source_url: str | None
+    description: str | None
+    source_aliases: list[str] | None
+    source_fish_species: list[str] | None
+    source_image_urls: list[str] | None
+    source_point_urls: list[str] | None
+    source_checked_at: datetime | None
 
 
 class BaitOut(BaseModel):
@@ -48,6 +58,8 @@ class ActivityOut(BaseModel):
     confidence_score: int
     explanation: str
     sources: list[str]
+    coordinate_precision: str
+    coordinate_sources: list[str]
 
 
 class PaginatedActivityOut(BaseModel):
@@ -82,6 +94,8 @@ class SpotOut(BaseModel):
     catches_3d: int
     catches_7d: int
     top_baits: list[str]
+    coordinate_precision: str
+    coordinate_sources: list[str]
 
 
 class OfficialRecordOut(BaseModel):