diff --git a/README.md b/README.md index f42769e..b8ef3f2 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Production release отделяет Alembic от runtime: одноразовый Тяжёлый production bootstrap вынесен в отдельный ручной/еженедельный CI workflow с 30-минутным timeout и сохраняемыми diagnostics; обычный push по-прежнему использует быстрый Compose E2E. -Публичный API зафиксирован генерируемым [OpenAPI-контрактом](docs/api-contract.md): CI сравнивает `apps/api/openapi.json` с фактической схемой FastAPI, поэтому рефакторинг routers не может незаметно изменить URL, параметры или response models. Декомпозиция выполняется инкрементально: общий `Db` вынесен в dependencies, каталог рыб/водоёмов/приманок и public spot pages уже обслуживает отдельный `APIRouter`. +Публичный API зафиксирован генерируемым [OpenAPI-контрактом](docs/api-contract.md): CI сравнивает `apps/api/openapi.json` с фактической схемой FastAPI, поэтому рефакторинг routers не может незаметно изменить URL, параметры или response models. Декомпозиция выполняется инкрементально: общий `Db` вынесен в dependencies; каталог рыб/водоёмов/приманок, public spot pages, activity и чтение точек уже обслуживают отдельные `APIRouter`. После повторных ошибок scheduler увеличивает паузу экспоненциально до 24 часов и возвращается к 30 минутам после успеха. Публичная страница `/status` показывает свежесть и состояние источников без URL запросов, внутренних ошибок и другой диагностической информации. diff --git a/apps/api/app/main.py b/apps/api/app/main.py index fa6343f..28a5768 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections import Counter from datetime import datetime, timedelta, timezone from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network import hashlib @@ -21,7 +20,6 @@ from sqlalchemy import delete, func, select, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, joinedload -from .activity import activity_rows from .config import settings from .dependencies import Db from .community_review import ExternalReviewError, map_observation, publish_observation, reject_observation, suggest_aliases @@ -29,9 +27,11 @@ from .importer import ImportAlreadyRunning, ImportSourceError, import_records, n from .logging_config import configure_logging from .models import Bait, BaitKind, CatchReport, CommunityImportRun, DataSource, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody from .readiness import readiness_report +from .routers.activity import router as activity_router from .routers.catalog import router as catalog_router +from .time_utils import aware from .public_cache import public_cache -from .schemas import ActivityOut, AdminCatchReportOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, ImportRunOut, ImportRunPublicOut, ModerationUpdate, OfficialRecordOut, PaginatedActivityOut, PaginatedOfficialRecordOut, PublicObservationOut, SourceStatusOut, SpotOut +from .schemas import ActivityOut, AdminCatchReportOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, ImportRunOut, ImportRunPublicOut, ModerationUpdate, OfficialRecordOut, PaginatedOfficialRecordOut, PublicObservationOut, SourceStatusOut from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot @@ -99,103 +99,7 @@ def ready(db: Db) -> JSONResponse: app.include_router(catalog_router) - - -@app.get("/api/v1/activity", response_model=PaginatedActivityOut) -def activity( - db: Db, response: Response, hours: int = Query(24), - waterbody: str | None = None, fish: str | None = None, - method: str | None = None, - sort: Literal["activity", "confidence", "freshness"] = "activity", - limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0), -) -> PaginatedActivityOut: - if hours not in {6, 12, 24, 72}: - raise HTTPException(status_code=422, detail="hours must be one of: 6, 12, 24, 72") - response.headers["Cache-Control"] = "no-store" - generation = public_cache.generation() - cache_key = ("activity", hours, waterbody, fish, method, sort, limit, offset) - cached = public_cache.get(cache_key, settings.public_cache_seconds) - if cached is not None: - response.headers["X-Cache"] = "HIT" - return cached - rows = activity_rows(db, hours=hours, waterbody=waterbody, fish=fish, method=method) - total = len(rows) - keys = { - "activity": lambda r: (r.activity_score, r.confidence_score, r.last_confirmed_at, str(r.spot_id)), - "confidence": lambda r: (r.confidence_score, r.activity_score, r.last_confirmed_at, str(r.spot_id)), - "freshness": lambda r: (r.last_confirmed_at, r.activity_score, r.confidence_score, str(r.spot_id)), - } - rows.sort(key=keys[sort], reverse=True) - page = rows[offset:offset + limit] - response.headers["X-Cache"] = "MISS" - return public_cache.set(cache_key, PaginatedActivityOut(items=page, total=total, limit=limit, offset=offset), generation=generation) - - -def _spot_or_404(db: Session, spot_id: UUID) -> Spot: - spot = db.scalar(select(Spot).options(joinedload(Spot.waterbody)).where(Spot.id == spot_id)) - if spot is None: - raise HTTPException(status_code=404, detail="spot not found") - return spot - - -@app.get("/api/v1/spots/resolve", response_model=SpotOut) -def resolve_spot( - db: Db, waterbody: str, x: int = Query(ge=-10_000, le=10_000), - y: int = Query(ge=-10_000, le=10_000), -) -> SpotOut: - spot = db.scalar(select(Spot).options(joinedload(Spot.waterbody)).join(Spot.waterbody).where( - Waterbody.slug == waterbody, Spot.x == x, Spot.y == y, - )) - if spot is None: - raise HTTPException(status_code=404, detail="spot not found") - return spot_detail(spot.id, db) - - -@app.get("/api/v1/spots/{spot_id}", response_model=SpotOut) -def spot_detail(spot_id: UUID, db: Db) -> SpotOut: - spot = _spot_or_404(db, spot_id) - reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot.id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)))) - now = datetime.now(timezone.utc) - def count_since(delta: timedelta) -> int: - return sum(_aware(r.reported_at) >= now - delta for r in reports) - bait_counts = Counter(r.bait.name for r in reports if r.bait) - return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)]) - - -@app.get("/api/v1/spots/{spot_id}/catches", response_model=list[CatchOut]) -def spot_catches(spot_id: UUID, db: Db, limit: int = Query(50, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[CatchOut]: - _spot_or_404(db, spot_id) - reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at.desc(), CatchReport.id.desc()).offset(offset).limit(limit))) - return [CatchOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, bait=r.bait.name if r.bait else None, player_name=r.player_name, caught_at=r.caught_at, reported_at=r.reported_at, retrieve_method=r.retrieve_method, retrieve_speed=r.retrieve_speed, source_system=_report_source(r), source_url=r.source_url) for r in reports] - - -@app.get("/api/v1/spots/{spot_id}/timeline") -def spot_timeline(spot_id: UUID, db: Db) -> list[dict]: - _spot_or_404(db, spot_id) - now = datetime.now(timezone.utc) - buckets = [] - for index in range(6): - start = now - timedelta(hours=(6 - index) * 12) - end = start + timedelta(hours=12) - count = db.scalar(select(func.count()).select_from(CatchReport).where( - CatchReport.spot_id == spot_id, - CatchReport.moderation_status == ModerationStatus.approved, - CatchReport.deleted_at.is_(None), - CatchReport.reported_at >= start, CatchReport.reported_at < end, - )) or 0 - buckets.append({"start": start.isoformat(), "end": end.isoformat(), "count": count}) - return buckets - - -def _report_source(report: CatchReport) -> str: - provenance = (report.raw_payload or {}).get("provenance", {}) - if isinstance(provenance, dict) and provenance.get("source_system"): - return str(provenance["source_system"]) - if report.source_type == SourceType.official_record: - return "rf4-official" - if report.source_type == SourceType.user: - return "players" - return "manual-import" +app.include_router(activity_router) @app.get("/api/v1/community-observations", response_model=list[PublicObservationOut]) @@ -243,7 +147,7 @@ def source_status(db: Db) -> list[SourceStatusOut]: state = "waiting" elif latest.status == "failed": state = "source_changed" if "CommunityParseError" in (latest.error_summary or "") else "temporarily_limited" - elif _aware(latest.started_at) < now - timedelta(seconds=settings.community_import_interval_seconds * 2): + elif aware(latest.started_at) < now - timedelta(seconds=settings.community_import_interval_seconds * 2): state = "stale" else: state = "healthy" @@ -609,7 +513,3 @@ def _check_rate_limit(request: Request, db: Session) -> None: raise HTTPException(status_code=429, detail="too many submissions") db.add(SubmissionAttempt(client_hash=client_hash, created_at=now)) db.commit() - - -def _aware(value: datetime) -> datetime: - return value if value.tzinfo else value.replace(tzinfo=timezone.utc) diff --git a/apps/api/app/routers/activity.py b/apps/api/app/routers/activity.py new file mode 100644 index 0000000..edd4ece --- /dev/null +++ b/apps/api/app/routers/activity.py @@ -0,0 +1,100 @@ +from collections import Counter +from datetime import datetime, timedelta, timezone +from typing import Literal +from uuid import UUID + +from fastapi import APIRouter, HTTPException, Query, Response +from sqlalchemy import func, select +from sqlalchemy.orm import Session, joinedload + +from ..activity import activity_rows +from ..config import settings +from ..dependencies import Db +from ..models import CatchReport, ModerationStatus, SourceType, Spot, Waterbody +from ..public_cache import public_cache +from ..schemas import CatchOut, PaginatedActivityOut, SpotOut +from ..time_utils import aware + +router = APIRouter() + + +@router.get("/api/v1/activity", response_model=PaginatedActivityOut) +def activity(db: Db, response: Response, hours: int = Query(24), waterbody: str | None = None, + fish: str | None = None, method: str | None = None, + sort: Literal["activity", "confidence", "freshness"] = "activity", + limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0)) -> PaginatedActivityOut: + if hours not in {6, 12, 24, 72}: + raise HTTPException(status_code=422, detail="hours must be one of: 6, 12, 24, 72") + response.headers["Cache-Control"] = "no-store" + generation = public_cache.generation() + cache_key = ("activity", hours, waterbody, fish, method, sort, limit, offset) + cached = public_cache.get(cache_key, settings.public_cache_seconds) + if cached is not None: + response.headers["X-Cache"] = "HIT" + return cached + rows = activity_rows(db, hours=hours, waterbody=waterbody, fish=fish, method=method) + total = len(rows) + keys = { + "activity": lambda row: (row.activity_score, row.confidence_score, row.last_confirmed_at, str(row.spot_id)), + "confidence": lambda row: (row.confidence_score, row.activity_score, row.last_confirmed_at, str(row.spot_id)), + "freshness": lambda row: (row.last_confirmed_at, row.activity_score, row.confidence_score, str(row.spot_id)), + } + rows.sort(key=keys[sort], reverse=True) + response.headers["X-Cache"] = "MISS" + return public_cache.set(cache_key, PaginatedActivityOut(items=rows[offset:offset + limit], total=total, limit=limit, offset=offset), generation=generation) + + +def _spot_or_404(db: Session, spot_id: UUID) -> Spot: + spot = db.scalar(select(Spot).options(joinedload(Spot.waterbody)).where(Spot.id == spot_id)) + if spot is None: + raise HTTPException(status_code=404, detail="spot not found") + return spot + + +@router.get("/api/v1/spots/resolve", response_model=SpotOut) +def resolve_spot(db: Db, waterbody: str, x: int = Query(ge=-10_000, le=10_000), y: int = Query(ge=-10_000, le=10_000)) -> SpotOut: + spot = db.scalar(select(Spot).options(joinedload(Spot.waterbody)).join(Spot.waterbody).where(Waterbody.slug == waterbody, Spot.x == x, Spot.y == y)) + if spot is None: + raise HTTPException(status_code=404, detail="spot not found") + return spot_detail(spot.id, db) + + +@router.get("/api/v1/spots/{spot_id}", response_model=SpotOut) +def spot_detail(spot_id: UUID, db: Db) -> SpotOut: + spot = _spot_or_404(db, spot_id) + reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot.id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)))) + now = datetime.now(timezone.utc) + bait_counts = Counter(report.bait.name for report in reports if report.bait) + def count_since(delta: timedelta) -> int: + return sum(aware(report.reported_at) >= now - delta for report in reports) + + return SpotOut(id=spot.id, waterbody_slug=spot.waterbody.slug, waterbody=spot.waterbody.name_ru, x=spot.x, y=spot.y, description=spot.description, catches_24h=count_since(timedelta(hours=24)), catches_3d=count_since(timedelta(days=3)), catches_7d=count_since(timedelta(days=7)), top_baits=[name for name, _ in bait_counts.most_common(5)]) + + +def _report_source(report: CatchReport) -> str: + provenance = (report.raw_payload or {}).get("provenance", {}) + if isinstance(provenance, dict) and provenance.get("source_system"): + return str(provenance["source_system"]) + if report.source_type == SourceType.official_record: + return "rf4-official" + return "players" if report.source_type == SourceType.user else "manual-import" + + +@router.get("/api/v1/spots/{spot_id}/catches", response_model=list[CatchOut]) +def spot_catches(spot_id: UUID, db: Db, limit: int = Query(50, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[CatchOut]: + _spot_or_404(db, spot_id) + reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at.desc(), CatchReport.id.desc()).offset(offset).limit(limit))) + return [CatchOut(id=report.id, fish=report.fish.name_ru, weight_g=report.weight_g, bait=report.bait.name if report.bait else None, player_name=report.player_name, caught_at=report.caught_at, reported_at=report.reported_at, retrieve_method=report.retrieve_method, retrieve_speed=report.retrieve_speed, source_system=_report_source(report), source_url=report.source_url) for report in reports] + + +@router.get("/api/v1/spots/{spot_id}/timeline") +def spot_timeline(spot_id: UUID, db: Db) -> list[dict]: + _spot_or_404(db, spot_id) + now = datetime.now(timezone.utc) + buckets = [] + for index in range(6): + start = now - timedelta(hours=(6 - index) * 12) + end = start + timedelta(hours=12) + count = db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None), CatchReport.reported_at >= start, CatchReport.reported_at < end)) or 0 + buckets.append({"start": start.isoformat(), "end": end.isoformat(), "count": count}) + return buckets diff --git a/apps/api/app/time_utils.py b/apps/api/app/time_utils.py new file mode 100644 index 0000000..4849d20 --- /dev/null +++ b/apps/api/app/time_utils.py @@ -0,0 +1,5 @@ +from datetime import datetime, timezone + + +def aware(value: datetime) -> datetime: + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6e69934..acdd2aa 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -51,7 +51,7 @@ Аудит выполнен на старой базе `13e04e6`; рекомендации ниже повторно проверены по текущей ветке. Уже реализованные или неприменимые предложения не возвращаются в backlog. -- [ ] **Q11 · Декомпозиция API — в работе.** После фиксации OpenAPI catalog routes (рыбы, водоёмы, приманки, public spot pages) вынесены в первый `APIRouter`, общий `Db` — в dependencies. URL, response models и generated contract сохранены. Далее: activity/spots, submissions и admin/community отдельными пакетами. +- [ ] **Q11 · Декомпозиция API — в работе.** После фиксации OpenAPI catalog routes (рыбы, водоёмы, приманки, public spot pages) и контур activity/spots вынесены в отдельные `APIRouter`, общий `Db` — в dependencies, нормализация времени — в `time_utils`. URL, response models и generated contract сохранены. Далее: records/community status, submissions и admin отдельными пакетами. - [ ] **Q12 · Query-plan gate.** В рамках Q07 снять `EXPLAIN (ANALYZE, BUFFERS)` для activity, records, spot detail и public spot pages на реалистичном наборе данных. Существующие индексы миграции `0011_query_indexes` не дублировать; индекс с `fish_id`, SQL-агрегацию или materialized view добавлять только по измеренному плану и p95. - [x] **Q13 · Production bootstrap в CI.** Отдельный workflow запускает `deploy/test-production-bootstrap.sh` вручную или раз в неделю, а не на каждом push. Вывод bootstrap всегда сохраняется 14 дней; при падении добавляются Compose status и Playwright diagnostics. - [ ] **Q14 · Полная CSP.** Расширить текущую CSP (`frame-ancestors`, `base-uri`, `object-src`) до `default-src`, `script-src`, `style-src`, `img-src`, `connect-src` и `form-action`. Сначала инвентаризировать inline scripts/styles Astro, затем внедрить nonce/hash или безопасное вынесение; проверить report/admin/OG без ослабления до произвольных внешних origin.