feat: stage external source observations

2026-09-03 18:41:32 +07:00
parent 16d64606f8
commit 53d700126a
8 changed files with 352 additions and 9 deletions
@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any, Iterable
+from urllib.parse import urlparse
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .models import DataSource, ExternalObservation
+
+
+SOURCE_DEFAULTS = {
+    "rf4db": ("RF4DB", "https://rf4db.com", 70),
+    "rf4stat-fishing": ("RF4-STAT fishing", "https://rf4-stat.ru/fishing/", 65),
+    "rf4stat-post": ("RF4-STAT posts", "https://rf4-stat.ru/posts/", 60),
+}
+SOURCE_HOSTS = {
+    "rf4db": {"rf4db.com", "download.rf4db.com"},
+    "rf4stat-fishing": {"rf4-stat.ru"},
+    "rf4stat-post": {"rf4-stat.ru"},
+}
+
+
+class CommunityImportError(ValueError):
+    pass
+
+
+def stage_observations(
+    session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
+) -> tuple[int, int]:
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    created = updated = 0
+    for raw in records:
+        payload = _json_payload(raw)
+        source_system = _required(payload, "source_system", 50)
+        if source_system not in SOURCE_DEFAULTS:
+            raise CommunityImportError(f"unsupported source_system: {source_system}")
+        external_id = _required(payload, "source_external_id", 200)
+        source = session.get(DataSource, source_system)
+        if source is None:
+            name, base_url, confidence = SOURCE_DEFAULTS[source_system]
+            source = DataSource(key=source_system, name=name, base_url=base_url, default_confidence=confidence, enabled=False)
+            session.add(source)
+            session.flush()
+        observation = session.scalar(select(ExternalObservation).where(
+            ExternalObservation.source_system == source_system,
+            ExternalObservation.source_external_id == external_id,
+        ))
+        source_url = _required(payload, "source_url", 2000)
+        if urlparse(source_url).scheme != "https" or urlparse(source_url).hostname not in SOURCE_HOSTS[source_system]:
+            raise CommunityImportError("source_url does not match source_system")
+        values = {
+            "source_url": source_url,
+            "fish_name": _required(payload, "fish", 200),
+            "fish_external_id": _optional(payload, "fish_external_id", 200),
+            "waterbody_name": _required(payload, "waterbody", 200),
+            "waterbody_external_id": _optional(payload, "waterbody_external_id", 200),
+            "x": _integer(payload.get("x"), maximum=10_000),
+            "y": _integer(payload.get("y"), maximum=10_000),
+            "weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000),
+            "published_at": _datetime(payload.get("published_at")),
+            "last_seen_at": fetched_at, "payload": payload,
+        }
+        if observation is None:
+            observation = ExternalObservation(
+                source_system=source_system, source_external_id=external_id,
+                first_seen_at=fetched_at, status="staged", **values,
+            )
+            session.add(observation)
+            created += 1
+        else:
+            for key, value in values.items():
+                setattr(observation, key, value)
+            updated += 1
+    session.commit()
+    return created, updated
+
+
+def _json_payload(raw: dict[str, Any]) -> dict[str, Any]:
+    if not isinstance(raw, dict):
+        raise CommunityImportError("each observation must be an object")
+    return json.loads(json.dumps(raw, default=str))
+
+
+def _required(payload: dict[str, Any], key: str, limit: int) -> str:
+    value = str(payload.get(key) or "").strip()
+    if not value or len(value) > limit:
+        raise CommunityImportError(f"invalid {key}")
+    return value
+
+
+def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None:
+    value = str(payload.get(key) or "").strip()
+    if len(value) > limit:
+        raise CommunityImportError(f"invalid {key}")
+    return value or None
+
+
+def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None:
+    if value is None:
+        return None
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as exc:
+        raise CommunityImportError("invalid integer field") from exc
+    if not minimum <= parsed <= maximum:
+        raise CommunityImportError("integer field outside allowed range")
+    return parsed
+
+
+def _datetime(value: Any) -> datetime | None:
+    if value in {None, ""}:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value))
+    except ValueError as exc:
+        raise CommunityImportError("invalid published_at") from exc
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)