feat: stage external source observations

2026-09-03 18:41:32 +07:00
parent 16d64606f8
commit 53d700126a
8 changed files with 352 additions and 9 deletions
@@ -1,9 +1,12 @@
 from __future__ import annotations

 import argparse
+import json
+import sys

 from .database import SessionLocal
 from .importer import import_records
+from .community_importer import stage_observations


 def main() -> int:
@@ -13,10 +16,27 @@ def main() -> int:
    command.add_argument("--url", default="https://rf4game.de/records/region/RU/")
    command.add_argument("--region", default="RU")
    command.add_argument("--category", default="records")
+    community = sub.add_parser("stage-community-json")
+    community.add_argument("--input", default="-", help="JSON array path or - for stdin")
+    community.add_argument("--limit", type=int, default=500)
    args = parser.parse_args()
    with SessionLocal() as session:
-        run = import_records(session, url=args.url, region=args.region, category=args.category)
-    print(f"import {run.status.value}: seen={run.rows_seen} created={run.rows_created} updated={run.rows_updated}")
+        if args.command == "import-records":
+            run = import_records(session, url=args.url, region=args.region, category=args.category)
+            print(f"import {run.status.value}: seen={run.rows_seen} created={run.rows_created} updated={run.rows_updated}")
+        else:
+            if not 1 <= args.limit <= 5000:
+                parser.error("--limit must be between 1 and 5000")
+            stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
+            try:
+                payload = json.load(stream)
+            finally:
+                if stream is not sys.stdin:
+                    stream.close()
+            if not isinstance(payload, list):
+                parser.error("input must be a JSON array")
+            created, updated = stage_observations(session, payload[:args.limit])
+            print(f"staged: created={created} updated={updated}")
    return 0


@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any, Iterable
+from urllib.parse import urlparse
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .models import DataSource, ExternalObservation
+
+
+SOURCE_DEFAULTS = {
+    "rf4db": ("RF4DB", "https://rf4db.com", 70),
+    "rf4stat-fishing": ("RF4-STAT fishing", "https://rf4-stat.ru/fishing/", 65),
+    "rf4stat-post": ("RF4-STAT posts", "https://rf4-stat.ru/posts/", 60),
+}
+SOURCE_HOSTS = {
+    "rf4db": {"rf4db.com", "download.rf4db.com"},
+    "rf4stat-fishing": {"rf4-stat.ru"},
+    "rf4stat-post": {"rf4-stat.ru"},
+}
+
+
+class CommunityImportError(ValueError):
+    pass
+
+
+def stage_observations(
+    session: Session, records: Iterable[dict[str, Any]], *, fetched_at: datetime | None = None,
+) -> tuple[int, int]:
+    fetched_at = fetched_at or datetime.now(timezone.utc)
+    created = updated = 0
+    for raw in records:
+        payload = _json_payload(raw)
+        source_system = _required(payload, "source_system", 50)
+        if source_system not in SOURCE_DEFAULTS:
+            raise CommunityImportError(f"unsupported source_system: {source_system}")
+        external_id = _required(payload, "source_external_id", 200)
+        source = session.get(DataSource, source_system)
+        if source is None:
+            name, base_url, confidence = SOURCE_DEFAULTS[source_system]
+            source = DataSource(key=source_system, name=name, base_url=base_url, default_confidence=confidence, enabled=False)
+            session.add(source)
+            session.flush()
+        observation = session.scalar(select(ExternalObservation).where(
+            ExternalObservation.source_system == source_system,
+            ExternalObservation.source_external_id == external_id,
+        ))
+        source_url = _required(payload, "source_url", 2000)
+        if urlparse(source_url).scheme != "https" or urlparse(source_url).hostname not in SOURCE_HOSTS[source_system]:
+            raise CommunityImportError("source_url does not match source_system")
+        values = {
+            "source_url": source_url,
+            "fish_name": _required(payload, "fish", 200),
+            "fish_external_id": _optional(payload, "fish_external_id", 200),
+            "waterbody_name": _required(payload, "waterbody", 200),
+            "waterbody_external_id": _optional(payload, "waterbody_external_id", 200),
+            "x": _integer(payload.get("x"), maximum=10_000),
+            "y": _integer(payload.get("y"), maximum=10_000),
+            "weight_g": _integer(payload.get("weight_g"), minimum=1, maximum=3_000_000),
+            "published_at": _datetime(payload.get("published_at")),
+            "last_seen_at": fetched_at, "payload": payload,
+        }
+        if observation is None:
+            observation = ExternalObservation(
+                source_system=source_system, source_external_id=external_id,
+                first_seen_at=fetched_at, status="staged", **values,
+            )
+            session.add(observation)
+            created += 1
+        else:
+            for key, value in values.items():
+                setattr(observation, key, value)
+            updated += 1
+    session.commit()
+    return created, updated
+
+
+def _json_payload(raw: dict[str, Any]) -> dict[str, Any]:
+    if not isinstance(raw, dict):
+        raise CommunityImportError("each observation must be an object")
+    return json.loads(json.dumps(raw, default=str))
+
+
+def _required(payload: dict[str, Any], key: str, limit: int) -> str:
+    value = str(payload.get(key) or "").strip()
+    if not value or len(value) > limit:
+        raise CommunityImportError(f"invalid {key}")
+    return value
+
+
+def _optional(payload: dict[str, Any], key: str, limit: int) -> str | None:
+    value = str(payload.get(key) or "").strip()
+    if len(value) > limit:
+        raise CommunityImportError(f"invalid {key}")
+    return value or None
+
+
+def _integer(value: Any, *, minimum: int = -10_000, maximum: int) -> int | None:
+    if value is None:
+        return None
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as exc:
+        raise CommunityImportError("invalid integer field") from exc
+    if not minimum <= parsed <= maximum:
+        raise CommunityImportError("integer field outside allowed range")
+    return parsed
+
+
+def _datetime(value: Any) -> datetime | None:
+    if value in {None, ""}:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value))
+    except ValueError as exc:
+        raise CommunityImportError("invalid published_at") from exc
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
@@ -136,3 +136,34 @@ class SubmissionAttempt(Base):
    id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
    client_hash: Mapped[str] = mapped_column(String(64), index=True)
    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+
+
+class DataSource(Base):
+    __tablename__ = "data_source"
+    key: Mapped[str] = mapped_column(String(50), primary_key=True)
+    name: Mapped[str] = mapped_column(String(100))
+    base_url: Mapped[str] = mapped_column(Text)
+    default_confidence: Mapped[int]
+    enabled: Mapped[bool] = mapped_column(default=False)
+
+
+class ExternalObservation(Base):
+    __tablename__ = "external_observation"
+    __table_args__ = (UniqueConstraint("source_system", "source_external_id"),)
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+    source_system: Mapped[str] = mapped_column(ForeignKey("data_source.key"), index=True)
+    source_external_id: Mapped[str] = mapped_column(String(200))
+    source_url: Mapped[str] = mapped_column(Text)
+    fish_name: Mapped[str] = mapped_column(String(200))
+    fish_external_id: Mapped[str | None] = mapped_column(String(200))
+    waterbody_name: Mapped[str] = mapped_column(String(200))
+    waterbody_external_id: Mapped[str | None] = mapped_column(String(200))
+    x: Mapped[int | None]
+    y: Mapped[int | None]
+    weight_g: Mapped[int | None]
+    published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+    status: Mapped[str] = mapped_column(String(30), default="staged", index=True)
+    payload: Mapped[dict] = mapped_column(JSON)
+    source: Mapped[DataSource] = relationship()