Add administrative official import API

apps/api/app/config.py

@@ -10,6 +10,9 @@ class Settings(BaseSettings):
     s3_secret_key: str = "rf4-local-secret"
     s3_bucket: str = "catch-screenshots"
     screenshot_max_bytes: int = 8 * 1024 * 1024
+    official_records_url: str = "https://rf4game.de/records/region/RU/"
+    official_records_region: str = "RU"
+    official_records_category: str = "records"
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 

apps/api/app/main.py

@@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone
 from typing import Annotated, Literal
 from uuid import UUID
 
+import httpx
 from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from sqlalchemy import select
@@ -13,7 +14,7 @@ from sqlalchemy.orm import Session, joinedload
 from .activity import activity_rows
 from .database import get_session
 from .config import settings
-from .importer import normalize
+from .importer import ImportSourceError, import_records, normalize
 from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
 from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
 from .storage import ScreenshotError, signed_screenshot_url, upload_screenshot
@@ -103,11 +104,43 @@ def records(
     return [OfficialRecordOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, waterbody=r.waterbody.name_ru, bait=r.bait.name if r.bait else None, player_name=r.player_name, record_date=r.caught_at, category=(r.raw_payload or {}).get("category"), region=(r.raw_payload or {}).get("region"), source_url=r.source_url) for r in items]
 
 
+def _admin(authorization: Annotated[str | None, Header()] = None) -> str:
+    if not authorization or authorization != f"Bearer {settings.admin_token}":
+        raise HTTPException(status_code=401, detail="invalid admin token", headers={"WWW-Authenticate": "Bearer"})
+    return "admin"
+
+
 @app.get("/api/v1/imports", response_model=list[ImportRunOut])
 def imports(db: Db, limit: int = Query(20, ge=1, le=100)) -> list[OfficialRecordImport]:
     return list(db.scalars(select(OfficialRecordImport).order_by(OfficialRecordImport.started_at.desc()).limit(limit)))
 
 
+@app.get("/api/v1/admin/imports", response_model=list[ImportRunOut])
+def admin_imports(
+    db: Db,
+    _: Annotated[str, Depends(_admin)],
+    limit: int = Query(20, ge=1, le=100),
+    offset: int = Query(0, ge=0),
+) -> list[OfficialRecordImport]:
+    query = select(OfficialRecordImport).order_by(
+        OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc()
+    ).offset(offset).limit(limit)
+    return list(db.scalars(query))
+
+
+@app.post("/api/v1/admin/imports/official-records", response_model=ImportRunOut, status_code=201)
+def admin_start_official_import(db: Db, _: Annotated[str, Depends(_admin)]) -> OfficialRecordImport:
+    try:
+        return import_records(
+            db,
+            url=settings.official_records_url,
+            region=settings.official_records_region,
+            category=settings.official_records_category,
+        )
+    except (ImportSourceError, httpx.HTTPError) as exc:
+        raise HTTPException(status_code=502, detail=f"official records import failed: {exc}") from exc
+
+
 @app.post("/api/v1/catch-reports", response_model=CatchReportCreated, status_code=201)
 def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated:
     if payload.website:
@@ -150,12 +183,6 @@ def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) ->
     return Response(status_code=204)
 
 
-def _admin(authorization: Annotated[str | None, Header()] = None) -> str:
-    if not authorization or authorization != f"Bearer {settings.admin_token}":
-        raise HTTPException(status_code=401, detail="invalid admin token", headers={"WWW-Authenticate": "Bearer"})
-    return "admin"
-
-
 @app.get("/api/v1/admin/catch-reports", response_model=list[AdminCatchReportOut])
 def admin_reports(db: Db, _: Annotated[str, Depends(_admin)], status: ModerationStatus = ModerationStatus.pending, limit: int = Query(50, ge=1, le=100)) -> list[AdminCatchReportOut]:
     reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), joinedload(CatchReport.spot), joinedload(CatchReport.bait)).where(CatchReport.source_type == SourceType.user, CatchReport.moderation_status == status).order_by(CatchReport.reported_at).limit(limit)))

apps/api/tests/test_api.py

@@ -9,7 +9,7 @@ from sqlalchemy.pool import StaticPool
 
 from app.database import Base, get_session
 from app.main import app
-from app.models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody
+from app.models import Bait, BaitKind, CatchReport, Fish, ImportStatus, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
 
 
 engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
@@ -85,6 +85,34 @@ def test_user_report_requires_moderation_before_activity() -> None:
 
 def test_admin_requires_token() -> None:
     assert client.get("/api/v1/admin/catch-reports").status_code == 401
+    assert client.get("/api/v1/admin/imports").status_code == 401
+    assert client.post("/api/v1/admin/imports/official-records").status_code == 401
+
+
+def test_admin_can_start_and_list_official_import(monkeypatch) -> None:
+    def fake_import(db: Session, **_: str) -> OfficialRecordImport:
+        run = OfficialRecordImport(
+            started_at=datetime.now(timezone.utc),
+            finished_at=datetime.now(timezone.utc),
+            status=ImportStatus.success,
+            source_url="fixture://admin",
+            rows_seen=2,
+            rows_created=2,
+            rows_updated=0,
+        )
+        db.add(run)
+        db.commit()
+        db.refresh(run)
+        return run
+
+    monkeypatch.setattr("app.main.import_records", fake_import)
+    headers = {"Authorization": "Bearer change-me-in-production"}
+    started = client.post("/api/v1/admin/imports/official-records", headers=headers)
+    assert started.status_code == 201
+    assert started.json()["source_url"] == "fixture://admin"
+    listed = client.get("/api/v1/admin/imports?limit=1&offset=0", headers=headers)
+    assert listed.status_code == 200
+    assert listed.json()[0]["id"] == started.json()["id"]
 
 
 def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:

apps/api/tests/test_importer.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from pathlib import Path
 
+import pytest
 from sqlalchemy import create_engine, func, select
 from sqlalchemy.orm import Session
 
 from app.database import Base
-from app.importer import import_records, parse_html
-from app.models import CatchReport, OfficialRecordImport, SourceType
+from app.importer import ImportSourceError, import_records, parse_html
+from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType
 
 
 FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "records_ru_sample.html"
@@ -28,3 +29,29 @@ def test_parser_and_import_are_idempotent() -> None:
         assert (second.rows_created, second.rows_updated) == (0, 2)
         assert db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.source_type == SourceType.official_record)) == 2
         assert db.scalar(select(func.count()).select_from(OfficialRecordImport)) == 2
+
+
+def test_failed_import_preserves_previous_records_and_is_logged() -> None:
+    html = FIXTURE.read_text(encoding="utf-8")
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as db:
+        import_records(db, url="fixture://records", region="RU", category="records", html=html)
+        before = db.scalar(select(func.count()).select_from(CatchReport))
+
+        with pytest.raises(ImportSourceError, match="records table not found"):
+            import_records(db, url="fixture://broken", region="RU", category="records", html="<html></html>")
+
+        assert db.scalar(select(func.count()).select_from(CatchReport)) == before
+        failed = db.scalar(select(OfficialRecordImport).where(OfficialRecordImport.status == ImportStatus.failed))
+        assert failed is not None
+        assert failed.source_url == "fixture://broken"
+        assert "records table not found" in (failed.error_summary or "")
+
+
+def test_import_rejects_changed_column_contract() -> None:
+    html = FIXTURE.read_text(encoding="utf-8").replace(
+        'class="col data"', 'class="col changed"', 1
+    )
+    with pytest.raises(ImportSourceError, match="record columns changed"):
+        parse_html(html, region="RU", category="records")