diff --git a/.env.example b/.env.example index 64ccfe2..aa3d589 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,6 @@ S3_PUBLIC_ENDPOINT_URL=http://localhost:9000 S3_ACCESS_KEY=rf4-local S3_SECRET_KEY=rf4-local-secret S3_BUCKET=catch-screenshots +OFFICIAL_RECORDS_URL=https://rf4game.de/records/region/RU/ +OFFICIAL_RECORDS_REGION=RU +OFFICIAL_RECORDS_CATEGORY=records diff --git a/apps/api/app/config.py b/apps/api/app/config.py index 841028e..b9754d3 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -10,6 +10,9 @@ class Settings(BaseSettings): s3_secret_key: str = "rf4-local-secret" s3_bucket: str = "catch-screenshots" screenshot_max_bytes: int = 8 * 1024 * 1024 + official_records_url: str = "https://rf4game.de/records/region/RU/" + official_records_region: str = "RU" + official_records_category: str = "records" model_config = SettingsConfigDict(env_file=".env", extra="ignore") diff --git a/apps/api/app/main.py b/apps/api/app/main.py index a748b64..5262b0c 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone from typing import Annotated, Literal from uuid import UUID +import httpx from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile from fastapi.middleware.cors import CORSMiddleware from sqlalchemy import select @@ -13,7 +14,7 @@ from sqlalchemy.orm import Session, joinedload from .activity import activity_rows from .database import get_session from .config import settings -from .importer import normalize +from .importer import ImportSourceError, import_records, normalize from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut from .storage import ScreenshotError, signed_screenshot_url, upload_screenshot @@ -103,11 +104,43 @@ def records( return [OfficialRecordOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, waterbody=r.waterbody.name_ru, bait=r.bait.name if r.bait else None, player_name=r.player_name, record_date=r.caught_at, category=(r.raw_payload or {}).get("category"), region=(r.raw_payload or {}).get("region"), source_url=r.source_url) for r in items] +def _admin(authorization: Annotated[str | None, Header()] = None) -> str: + if not authorization or authorization != f"Bearer {settings.admin_token}": + raise HTTPException(status_code=401, detail="invalid admin token", headers={"WWW-Authenticate": "Bearer"}) + return "admin" + + @app.get("/api/v1/imports", response_model=list[ImportRunOut]) def imports(db: Db, limit: int = Query(20, ge=1, le=100)) -> list[OfficialRecordImport]: return list(db.scalars(select(OfficialRecordImport).order_by(OfficialRecordImport.started_at.desc()).limit(limit))) +@app.get("/api/v1/admin/imports", response_model=list[ImportRunOut]) +def admin_imports( + db: Db, + _: Annotated[str, Depends(_admin)], + limit: int = Query(20, ge=1, le=100), + offset: int = Query(0, ge=0), +) -> list[OfficialRecordImport]: + query = select(OfficialRecordImport).order_by( + OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc() + ).offset(offset).limit(limit) + return list(db.scalars(query)) + + +@app.post("/api/v1/admin/imports/official-records", response_model=ImportRunOut, status_code=201) +def admin_start_official_import(db: Db, _: Annotated[str, Depends(_admin)]) -> OfficialRecordImport: + try: + return import_records( + db, + url=settings.official_records_url, + region=settings.official_records_region, + category=settings.official_records_category, + ) + except (ImportSourceError, httpx.HTTPError) as exc: + raise HTTPException(status_code=502, detail=f"official records import failed: {exc}") from exc + + @app.post("/api/v1/catch-reports", response_model=CatchReportCreated, status_code=201) def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated: if payload.website: @@ -150,12 +183,6 @@ def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) -> return Response(status_code=204) -def _admin(authorization: Annotated[str | None, Header()] = None) -> str: - if not authorization or authorization != f"Bearer {settings.admin_token}": - raise HTTPException(status_code=401, detail="invalid admin token", headers={"WWW-Authenticate": "Bearer"}) - return "admin" - - @app.get("/api/v1/admin/catch-reports", response_model=list[AdminCatchReportOut]) def admin_reports(db: Db, _: Annotated[str, Depends(_admin)], status: ModerationStatus = ModerationStatus.pending, limit: int = Query(50, ge=1, le=100)) -> list[AdminCatchReportOut]: reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), joinedload(CatchReport.spot), joinedload(CatchReport.bait)).where(CatchReport.source_type == SourceType.user, CatchReport.moderation_status == status).order_by(CatchReport.reported_at).limit(limit))) diff --git a/apps/api/tests/test_api.py b/apps/api/tests/test_api.py index 86bdc98..5ce0fb9 100644 --- a/apps/api/tests/test_api.py +++ b/apps/api/tests/test_api.py @@ -9,7 +9,7 @@ from sqlalchemy.pool import StaticPool from app.database import Base, get_session from app.main import app -from app.models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody +from app.models import Bait, BaitKind, CatchReport, Fish, ImportStatus, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) @@ -85,6 +85,34 @@ def test_user_report_requires_moderation_before_activity() -> None: def test_admin_requires_token() -> None: assert client.get("/api/v1/admin/catch-reports").status_code == 401 + assert client.get("/api/v1/admin/imports").status_code == 401 + assert client.post("/api/v1/admin/imports/official-records").status_code == 401 + + +def test_admin_can_start_and_list_official_import(monkeypatch) -> None: + def fake_import(db: Session, **_: str) -> OfficialRecordImport: + run = OfficialRecordImport( + started_at=datetime.now(timezone.utc), + finished_at=datetime.now(timezone.utc), + status=ImportStatus.success, + source_url="fixture://admin", + rows_seen=2, + rows_created=2, + rows_updated=0, + ) + db.add(run) + db.commit() + db.refresh(run) + return run + + monkeypatch.setattr("app.main.import_records", fake_import) + headers = {"Authorization": "Bearer change-me-in-production"} + started = client.post("/api/v1/admin/imports/official-records", headers=headers) + assert started.status_code == 201 + assert started.json()["source_url"] == "fixture://admin" + listed = client.get("/api/v1/admin/imports?limit=1&offset=0", headers=headers) + assert listed.status_code == 200 + assert listed.json()[0]["id"] == started.json()["id"] def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None: diff --git a/apps/api/tests/test_importer.py b/apps/api/tests/test_importer.py index 2bbaa5a..7ffd2d6 100644 --- a/apps/api/tests/test_importer.py +++ b/apps/api/tests/test_importer.py @@ -2,12 +2,13 @@ from __future__ import annotations from pathlib import Path +import pytest from sqlalchemy import create_engine, func, select from sqlalchemy.orm import Session from app.database import Base -from app.importer import import_records, parse_html -from app.models import CatchReport, OfficialRecordImport, SourceType +from app.importer import ImportSourceError, import_records, parse_html +from app.models import CatchReport, ImportStatus, OfficialRecordImport, SourceType FIXTURE = Path(__file__).parents[3] / "tests" / "fixtures" / "records_ru_sample.html" @@ -28,3 +29,29 @@ def test_parser_and_import_are_idempotent() -> None: assert (second.rows_created, second.rows_updated) == (0, 2) assert db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.source_type == SourceType.official_record)) == 2 assert db.scalar(select(func.count()).select_from(OfficialRecordImport)) == 2 + + +def test_failed_import_preserves_previous_records_and_is_logged() -> None: + html = FIXTURE.read_text(encoding="utf-8") + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as db: + import_records(db, url="fixture://records", region="RU", category="records", html=html) + before = db.scalar(select(func.count()).select_from(CatchReport)) + + with pytest.raises(ImportSourceError, match="records table not found"): + import_records(db, url="fixture://broken", region="RU", category="records", html="") + + assert db.scalar(select(func.count()).select_from(CatchReport)) == before + failed = db.scalar(select(OfficialRecordImport).where(OfficialRecordImport.status == ImportStatus.failed)) + assert failed is not None + assert failed.source_url == "fixture://broken" + assert "records table not found" in (failed.error_summary or "") + + +def test_import_rejects_changed_column_contract() -> None: + html = FIXTURE.read_text(encoding="utf-8").replace( + 'class="col data"', 'class="col changed"', 1 + ) + with pytest.raises(ImportSourceError, match="record columns changed"): + parse_html(html, region="RU", category="records") diff --git a/compose.yaml b/compose.yaml index 384ef7c..0cf99c2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -35,6 +35,9 @@ services: S3_ACCESS_KEY: ${S3_ACCESS_KEY:-rf4-local} S3_SECRET_KEY: ${S3_SECRET_KEY:-rf4-local-secret} S3_BUCKET: ${S3_BUCKET:-catch-screenshots} + OFFICIAL_RECORDS_URL: ${OFFICIAL_RECORDS_URL:-https://rf4game.de/records/region/RU/} + OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU} + OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records} depends_on: db: condition: service_healthy diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b326977..2601f2d 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -18,12 +18,12 @@ ## Этап 2 — завершить официальный импорт -- [ ] Добавить административный endpoint ручного запуска импорта `POST /api/v1/admin/imports/official-records`. -- [ ] Привести журнал импорта к административному контракту `GET /api/v1/admin/imports` с авторизацией, пагинацией и стабильной сортировкой. +- [x] Добавить административный endpoint ручного запуска импорта `POST /api/v1/admin/imports/official-records` (проверено API-тестом). +- [x] Привести журнал импорта к административному контракту `GET /api/v1/admin/imports` с авторизацией, пагинацией и стабильной сортировкой (проверено API-тестом). - [ ] Добавить HTTP-кэширование источника (`ETag`/`Last-Modified`, если источник их отдаёт) и сохранить диагностические метаданные ответа. - [ ] Добавить планировщик импорта с безопасной частотой по умолчанию один раз в 60 минут; отдельный контейнер/процесс без дублирования запусков. - [ ] Проверить актуальные `robots.txt` и условия использования перед включением расписания; результат записать в `docs/data-sources.md`. -- [ ] Добавить интеграционные тесты: повторный импорт не создаёт дубликаты, сбой источника не удаляет данные, изменение DOM завершается понятной ошибкой. +- [x] Добавить интеграционные тесты: повторный импорт не создаёт дубликаты, сбой источника не удаляет данные, изменение DOM завершается понятной ошибкой. Критерий готовности: официальный импорт запускается вручную и по расписанию, наблюдаем, идемпотентен и безопасно переживает недоступность источника.