diff --git a/.env.example b/.env.example index 262e4e1..6785edd 100644 --- a/.env.example +++ b/.env.example @@ -10,5 +10,7 @@ S3_BUCKET=catch-screenshots OFFICIAL_RECORDS_URL=https://rf4game.de/records/region/RU/ OFFICIAL_RECORDS_REGION=RU OFFICIAL_RECORDS_CATEGORY=records +# true только если scheduler официального импорта обязателен для readiness +OFFICIAL_IMPORT_REQUIRED=false IMPORT_INTERVAL_SECONDS=3600 RATE_LIMIT_SECRET=change-rate-limit-secret diff --git a/README.md b/README.md index d8958e0..f3eeebc 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,11 @@ docker compose up --build - сайт: ; - OpenAPI: ; -- проверка API: ; +- liveness API: ; +- readiness PostgreSQL, MinIO и импорта: ; - консоль MinIO: . -Контейнер API сам выполняет `alembic upgrade head`, затем идемпотентный seed. PostgreSQL хранит данные в именованном volume `postgres_data`, а MinIO — в `minio_data`. +Контейнер API сам выполняет `alembic upgrade head`, затем идемпотентный seed. PostgreSQL хранит данные в именованном volume `postgres_data`, а MinIO — в `minio_data`. Compose ожидает readiness PostgreSQL и MinIO перед API, а API-контейнер проверяет `/ready`. Официальный импорт по умолчанию необязателен; при включённом scheduler установите `OFFICIAL_IMPORT_REQUIRED=true`, тогда отсутствующий, неуспешный или просроченный запуск сделает readiness отрицательным. Остановка: @@ -69,7 +70,7 @@ docker compose up --build ## Что реализовано - FastAPI и SQLAlchemy 2; -- PostgreSQL 17 и миграции Alembic до `0008`; +- PostgreSQL 17 и миграции Alembic до `0009`; - идемпотентный seed с двумя точками и свежими демо-уловами; - `GET /api/v1/activity` с фильтрами периода, водоёма, рыбы, способа и сортировки; - `GET /api/v1/spots/{id}` и `/catches`; diff --git a/apps/api/app/config.py b/apps/api/app/config.py index c9cb3fa..98615c0 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -14,6 +14,7 @@ class Settings(BaseSettings): official_records_url: str = "https://rf4game.de/records/region/RU/" official_records_region: str = "RU" official_records_category: str = "records" + official_import_required: bool = False import_interval_seconds: int = Field(default=3600, ge=3600) rate_limit_secret: str = "change-rate-limit-secret" model_config = SettingsConfigDict(env_file=".env", extra="ignore") diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 16172b6..a6d4ac2 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -10,6 +10,7 @@ from uuid import UUID import httpx from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from sqlalchemy import delete, func, select, text from sqlalchemy.orm import Session, joinedload @@ -19,8 +20,9 @@ from .config import settings from .community_review import ExternalReviewError, map_observation, publish_observation, reject_observation from .importer import ImportSourceError, import_records, normalize from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody +from .readiness import readiness_report from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut -from .storage import ScreenshotError, delete_screenshot, signed_screenshot_url, upload_screenshot +from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot app = FastAPI(title="RF4 Spotter API", version="0.1.0") @@ -38,6 +40,18 @@ def health() -> dict[str, str]: return {"status": "ok"} +@app.get("/ready") +def ready(db: Db) -> JSONResponse: + is_ready, components = readiness_report( + db, storage_client(), import_required=settings.official_import_required, + import_interval_seconds=settings.import_interval_seconds, + ) + return JSONResponse( + status_code=200 if is_ready else 503, + content={"status": "ready" if is_ready else "not_ready", "components": components}, + ) + + @app.get("/api/v1/fishes", response_model=list[FishOut]) def fishes(db: Db) -> list[Fish]: return list(db.scalars(select(Fish).order_by(Fish.name_ru))) diff --git a/apps/api/app/readiness.py b/apps/api/app/readiness.py new file mode 100644 index 0000000..b49c741 --- /dev/null +++ b/apps/api/app/readiness.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy import select, text +from sqlalchemy.orm import Session + +from .models import ImportStatus, OfficialRecordImport + + +def readiness_report( + session: Session, s3: Any, *, import_required: bool, + import_interval_seconds: int, now: datetime | None = None, +) -> tuple[bool, dict[str, dict[str, object]]]: + current = now or datetime.now(timezone.utc) + components: dict[str, dict[str, object]] = {} + ready = True + + try: + session.execute(text("SELECT 1")) + components["postgresql"] = {"status": "ready"} + except Exception: + components["postgresql"] = {"status": "unavailable"} + ready = False + + try: + s3.list_buckets() + components["minio"] = {"status": "ready"} + except Exception: + components["minio"] = {"status": "unavailable"} + ready = False + + try: + latest = session.scalar(select(OfficialRecordImport).order_by( + OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc(), + ).limit(1)) + if not import_required: + components["official_import"] = { + "status": "optional", + "last_run_status": latest.status.value if latest else None, + } + elif latest is None: + components["official_import"] = {"status": "not_run"} + ready = False + else: + started = latest.started_at if latest.started_at.tzinfo else latest.started_at.replace(tzinfo=timezone.utc) + stale = started < current - timedelta(seconds=import_interval_seconds * 2) + healthy = latest.status == ImportStatus.success and not stale + components["official_import"] = { + "status": "ready" if healthy else ("stale" if stale else latest.status.value), + "last_run_status": latest.status.value, + "last_started_at": started.isoformat(), + } + ready = ready and healthy + except Exception: + components["official_import"] = {"status": "unknown"} + if import_required: + ready = False + + return ready, components diff --git a/apps/api/tests/test_api.py b/apps/api/tests/test_api.py index 017dfdb..9b31e9f 100644 --- a/apps/api/tests/test_api.py +++ b/apps/api/tests/test_api.py @@ -55,6 +55,10 @@ def test_invalid_period_is_rejected() -> None: assert client.get("/api/v1/activity?sort=unknown").status_code == 422 +def test_liveness_does_not_probe_dependencies() -> None: + assert client.get("/health").json() == {"status": "ok"} + + def test_spot_detail_and_catches() -> None: spot_id = client.get("/api/v1/activity").json()[0]["spot_id"] detail = client.get(f"/api/v1/spots/{spot_id}") diff --git a/apps/api/tests/test_readiness.py b/apps/api/tests/test_readiness.py new file mode 100644 index 0000000..02252a5 --- /dev/null +++ b/apps/api/tests/test_readiness.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from app.database import Base +from app.models import ImportStatus, OfficialRecordImport +from app.readiness import readiness_report + + +class AvailableStorage: + def list_buckets(self) -> dict[str, list[object]]: + return {"Buckets": []} + + +class UnavailableStorage: + def list_buckets(self) -> None: + raise ConnectionError("fixture unavailable") + + +def test_optional_import_does_not_block_dependencies() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as session: + ready, components = readiness_report( + session, AvailableStorage(), import_required=False, import_interval_seconds=3600, + ) + assert ready is True + assert components == { + "postgresql": {"status": "ready"}, + "minio": {"status": "ready"}, + "official_import": {"status": "optional", "last_run_status": None}, + } + + +def test_required_import_must_be_recent_and_successful() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + now = datetime.now(timezone.utc) + with Session(engine) as session: + session.add(OfficialRecordImport( + started_at=now - timedelta(minutes=30), finished_at=now - timedelta(minutes=29), + status=ImportStatus.success, source_url="fixture://records", rows_seen=1, + rows_created=1, rows_updated=0, + )) + session.commit() + ready, components = readiness_report( + session, AvailableStorage(), import_required=True, + import_interval_seconds=3600, now=now, + ) + assert ready is True + assert components["official_import"]["status"] == "ready" + + +def test_unavailable_storage_and_stale_import_fail_readiness() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + now = datetime.now(timezone.utc) + with Session(engine) as session: + session.add(OfficialRecordImport( + started_at=now - timedelta(hours=3), finished_at=now - timedelta(hours=3), + status=ImportStatus.success, source_url="fixture://records", rows_seen=1, + rows_created=1, rows_updated=0, + )) + session.commit() + ready, components = readiness_report( + session, UnavailableStorage(), import_required=True, + import_interval_seconds=3600, now=now, + ) + assert ready is False + assert components["minio"]["status"] == "unavailable" + assert components["official_import"]["status"] == "stale" diff --git a/compose.yaml b/compose.yaml index cd82758..cf1befa 100644 --- a/compose.yaml +++ b/compose.yaml @@ -22,6 +22,11 @@ services: ports: - "9000:9000" - "9001:9001" + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 volumes: - minio_data:/data @@ -40,16 +45,17 @@ services: OFFICIAL_RECORDS_URL: ${OFFICIAL_RECORDS_URL:-https://rf4game.de/records/region/RU/} OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU} OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records} + OFFICIAL_IMPORT_REQUIRED: ${OFFICIAL_IMPORT_REQUIRED:-false} RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:-change-rate-limit-secret} depends_on: db: condition: service_healthy minio: - condition: service_started + condition: service_healthy ports: - "8000:8000" healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/ready')"] interval: 5s timeout: 3s retries: 12 @@ -64,6 +70,11 @@ services: condition: service_healthy ports: - "4321:4321" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://localhost:4321').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + interval: 5s + timeout: 3s + retries: 10 importer: build: diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 69721bd..4db9647 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -53,7 +53,7 @@ ## Подготовка MVP к пилоту -- [ ] Добавить health/readiness-проверки PostgreSQL, MinIO, API и импорта; отразить их в Compose. +- [x] Добавить health/readiness-проверки PostgreSQL, MinIO, API и импорта; отразить их в Compose (`/health` без зависимостей, `/ready` с компонентами и режимом обязательного импорта). - [ ] Добавить структурированные логи без пользовательских секретов и персональных технических данных. - [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO. - [ ] Провести security-проверку admin-аутентификации, CORS, заголовков, загрузок и управления секретами. @@ -91,10 +91,10 @@ Лёгкий пакет по парсерам и фильтрам завершён. Следующий пакет готовит MVP к пилоту: -1. health/readiness PostgreSQL, MinIO, API и импорта; -2. структурированные логи без пользовательских секретов; -3. CI для тестов, Astro build, E2E и миграций; -4. backup/restore PostgreSQL и MinIO. +1. структурированные логи без пользовательских секретов; +2. CI для тестов, Astro build, E2E и миграций; +3. backup/restore PostgreSQL и MinIO; +4. security-проверка admin-аутентификации, CORS, загрузок и секретов. После каждого пункта необходимо: