Add administrative official import API

.env.example

@@ -7,3 +7,6 @@ S3_PUBLIC_ENDPOINT_URL=http://localhost:9000
 S3_ACCESS_KEY=rf4-local
 S3_SECRET_KEY=rf4-local-secret
 S3_BUCKET=catch-screenshots
+OFFICIAL_RECORDS_URL=https://rf4game.de/records/region/RU/
+OFFICIAL_RECORDS_REGION=RU
+OFFICIAL_RECORDS_CATEGORY=records

apps/api/app/config.py

@@ -10,6 +10,9 @@ class Settings(BaseSettings):
     s3_secret_key: str = "rf4-local-secret"
     s3_bucket: str = "catch-screenshots"
     screenshot_max_bytes: int = 8 * 1024 * 1024
+    official_records_url: str = "https://rf4game.de/records/region/RU/"
+    official_records_region: str = "RU"
+    official_records_category: str = "records"
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 

apps/api/app/main.py

@@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone
 from typing import Annotated, Literal
 from uuid import UUID
 
+import httpx
 from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from sqlalchemy import select
@@ -13,7 +14,7 @@ from sqlalchemy.orm import Session, joinedload
 from .activity import activity_rows
 from .database import get_session
 from .config import settings
-from .importer import normalize
+from .importer import ImportSourceError, import_records, normalize
 from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
 from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
 from .storage import ScreenshotError, signed_screenshot_url, upload_screenshot
@@ -103,11 +104,43 @@ def records(
     return [OfficialRecordOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, waterbody=r.waterbody.name_ru, bait=r.bait.name if r.bait else None, player_name=r.player_name, record_date=r.caught_at, category=(r.raw_payload or {}).get("category"), region=(r.raw_payload or {}).get("region"), source_url=r.source_url) for r in items]
 
 
+def _admin(authorization: Annotated[str | None, Header()] = None) -> str:
+    if not authorization or authorization != f"Bearer {settings.admin_token}":
+        raise HTTPException(status_code=401, detail="invalid admin token", headers={"WWW-Authenticate": "Bearer"})
+    return "admin"
+
+
 @app.get("/api/v1/imports", response_model=list[ImportRunOut])
 def imports(db: Db, limit: int = Query(20, ge=1, le=100)) -> list[OfficialRecordImport]:
     return list(db.scalars(select(OfficialRecordImport).order_by(OfficialRecordImport.started_at.desc()).limit(limit)))
 
 
+@app.get("/api/v1/admin/imports", response_model=list[ImportRunOut])
+def admin_imports(
+    db: Db,
+    _: Annotated[str, Depends(_admin)],
+    limit: int = Query(20, ge=1, le=100),
+    offset: int = Query(0, ge=0),
+) -> list[OfficialRecordImport]:
+    query = select(OfficialRecordImport).order_by(
+        OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc()
+    ).offset(offset).limit(limit)
+    return list(db.scalars(query))
+
+
+@app.post("/api/v1/admin/imports/official-records", response_model=ImportRunOut, status_code=201)
+def admin_start_official_import(db: Db, _: Annotated[str, Depends(_admin)]) -> OfficialRecordImport:
+    try:
+        return import_records(
+            db,
+            url=settings.official_records_url,
+            region=settings.official_records_region,
+            category=settings.official_records_category,
+        )
+    except (ImportSourceError, httpx.HTTPError) as exc:
+        raise HTTPException(status_code=502, detail=f"official records import failed: {exc}") from exc
+
+
 @app.post("/api/v1/catch-reports", response_model=CatchReportCreated, status_code=201)
 def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated:
     if payload.website:
@@ -150,12 +183,6 @@ def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) ->
     return Response(status_code=204)
 
 
-def _admin(authorization: Annotated[str | None, Header()] = None) -> str:
-    if not authorization or authorization != f"Bearer {settings.admin_token}":
-        raise HTTPException(status_code=401, detail="invalid admin token", headers={"WWW-Authenticate": "Bearer"})
-    return "admin"
-
-
 @app.get("/api/v1/admin/catch-reports", response_model=list[AdminCatchReportOut])
 def admin_reports(db: Db, _: Annotated[str, Depends(_admin)], status: ModerationStatus = ModerationStatus.pending, limit: int = Query(50, ge=1, le=100)) -> list[AdminCatchReportOut]:
     reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), joinedload(CatchReport.spot), joinedload(CatchReport.bait)).where(CatchReport.source_type == SourceType.user, CatchReport.moderation_status == status).order_by(CatchReport.reported_at).limit(limit)))

apps/api/tests/test_api.py

@@ -9,7 +9,7 @@ from sqlalchemy.pool import StaticPool
 
 from app.database import Base, get_session
 from app.main import app
-from app.models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody
+from app.models import Bait, BaitKind, CatchReport, Fish, ImportStatus, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
 
 
 engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
@@ -85,6 +85,34 @@ def test_user_report_requires_moderation_before_activity() -> None:
 
 def test_admin_requires_token() -> None:
     assert client.get("/api/v1/admin/catch-reports").status_code == 401
+    assert client.get("/api/v1/admin/imports").status_code == 401
+    assert client.post("/api/v1/admin/imports/official-records").status_code == 401
+
+
+def test_admin_can_start_and_list_official_import(monkeypatch) -> None:
+    def fake_import(db: Session, **_: str) -> OfficialRecordImport:
+        run = OfficialRecordImport(
+            started_at=datetime.now(timezone.utc),
+            finished_at=datetime.now(timezone.utc),
+            status=ImportStatus.success,
+            source_url="fixture://admin",
+            rows_seen=2,
+            rows_created=2,
+            rows_updated=0,
+        )
+        db.add(run)
+        db.commit()
+        db.refresh(run)
+        return run
+
+    monkeypatch.setattr("app.main.import_records", fake_import)
+    headers = {"Authorization": "Bearer change-me-in-production"}
+    started = client.post("/api/v1/admin/imports/official-records", headers=headers)
+    assert started.status_code == 201
+    assert started.json()["source_url"] == "fixture://admin"
+    listed = client.get("/api/v1/admin/imports?limit=1&offset=0", headers=headers)
+    assert listed.status_code == 200
+    assert listed.json()[0]["id"] == started.json()["id"]
 
 
 def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:

apps/api/tests/test_importer.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from pathlib import Path
 
+import pytest
 from sqlalchemy import create_engine, func, select
 from sqlalchemy.orm import Session
 
 from app.database import Base
-from app.importer import import_records, parse_html
+from app.importer import ImportSourceError, import_records, parse_html
-from app.models import CatchReport, OfficialRecordImport, SourceType
+from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType
 
 
 FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "records_ru_sample.html"
@@ -28,3 +29,29 @@ def test_parser_and_import_are_idempotent() -> None:
         assert (second.rows_created, second.rows_updated) == (0, 2)
         assert db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.source_type == SourceType.official_record)) == 2
         assert db.scalar(select(func.count()).select_from(OfficialRecordImport)) == 2
+
+
+def test_failed_import_preserves_previous_records_and_is_logged() -> None:
+    html = FIXTURE.read_text(encoding="utf-8")
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as db:
+        import_records(db, url="fixture://records", region="RU", category="records", html=html)
+        before = db.scalar(select(func.count()).select_from(CatchReport))
+
+        with pytest.raises(ImportSourceError, match="records table not found"):
+            import_records(db, url="fixture://broken", region="RU", category="records", html="<html></html>")
+
+        assert db.scalar(select(func.count()).select_from(CatchReport)) == before
+        failed = db.scalar(select(OfficialRecordImport).where(OfficialRecordImport.status == ImportStatus.failed))
+        assert failed is not None
+        assert failed.source_url == "fixture://broken"
+        assert "records table not found" in (failed.error_summary or "")
+
+
+def test_import_rejects_changed_column_contract() -> None:
+    html = FIXTURE.read_text(encoding="utf-8").replace(
+        'class="col data"', 'class="col changed"', 1
+    )
+    with pytest.raises(ImportSourceError, match="record columns changed"):
+        parse_html(html, region="RU", category="records")

compose.yaml

@@ -35,6 +35,9 @@ services:
       S3_ACCESS_KEY: ${S3_ACCESS_KEY:-rf4-local}
       S3_SECRET_KEY: ${S3_SECRET_KEY:-rf4-local-secret}
       S3_BUCKET: ${S3_BUCKET:-catch-screenshots}
+      OFFICIAL_RECORDS_URL: ${OFFICIAL_RECORDS_URL:-https://rf4game.de/records/region/RU/}
+      OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU}
+      OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records}
     depends_on:
       db:
         condition: service_healthy

docs/ROADMAP.md

@@ -18,12 +18,12 @@
 
 ## Этап 2 — завершить официальный импорт
 
-- [ ] Добавить административный endpoint ручного запуска импорта `POST /api/v1/admin/imports/official-records`.
+- [x] Добавить административный endpoint ручного запуска импорта `POST /api/v1/admin/imports/official-records` (проверено API-тестом).
-- [ ] Привести журнал импорта к административному контракту `GET /api/v1/admin/imports` с авторизацией, пагинацией и стабильной сортировкой.
+- [x] Привести журнал импорта к административному контракту `GET /api/v1/admin/imports` с авторизацией, пагинацией и стабильной сортировкой (проверено API-тестом).
 - [ ] Добавить HTTP-кэширование источника (`ETag`/`Last-Modified`, если источник их отдаёт) и сохранить диагностические метаданные ответа.
 - [ ] Добавить планировщик импорта с безопасной частотой по умолчанию один раз в 60 минут; отдельный контейнер/процесс без дублирования запусков.
 - [ ] Проверить актуальные `robots.txt` и условия использования перед включением расписания; результат записать в `docs/data-sources.md`.
-- [ ] Добавить интеграционные тесты: повторный импорт не создаёт дубликаты, сбой источника не удаляет данные, изменение DOM завершается понятной ошибкой.
+- [x] Добавить интеграционные тесты: повторный импорт не создаёт дубликаты, сбой источника не удаляет данные, изменение DOM завершается понятной ошибкой.
 
 Критерий готовности: официальный импорт запускается вручную и по расписанию, наблюдаем, идемпотентен и безопасно переживает недоступность источника.