perf: stabilize list queries for pilot

apps/api/alembic/versions/0011_query_indexes.py

@@ -0,0 +1,29 @@
+"""Add composite indexes for pilot list and maintenance queries."""
+from alembic import op
+
+revision = "0011"
+down_revision = "0010"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_index("ix_catch_report_activity_lookup", "catch_report", ["moderation_status", "deleted_at", "reported_at"])
+    op.create_index("ix_catch_report_spot_feed", "catch_report", ["spot_id", "moderation_status", "deleted_at", "reported_at"])
+    op.create_index("ix_catch_report_moderation_queue", "catch_report", ["source_type", "moderation_status", "deleted_at", "reported_at"])
+    op.create_index("ix_catch_report_official_records", "catch_report", ["source_type", "caught_at", "weight_g"])
+    op.create_index("ix_official_import_source_status_started", "official_record_import", ["source_url", "status", "started_at"])
+    op.create_index("ix_external_observation_review_queue", "external_observation", ["status", "source_system", "last_seen_at"])
+    op.create_index("ix_submission_attempt_client_created", "submission_attempt", ["client_hash", "created_at"])
+    op.create_index("ix_moderation_event_created_at", "moderation_event", ["created_at"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_moderation_event_created_at", table_name="moderation_event")
+    op.drop_index("ix_submission_attempt_client_created", table_name="submission_attempt")
+    op.drop_index("ix_external_observation_review_queue", table_name="external_observation")
+    op.drop_index("ix_official_import_source_status_started", table_name="official_record_import")
+    op.drop_index("ix_catch_report_official_records", table_name="catch_report")
+    op.drop_index("ix_catch_report_moderation_queue", table_name="catch_report")
+    op.drop_index("ix_catch_report_spot_feed", table_name="catch_report")
+    op.drop_index("ix_catch_report_activity_lookup", table_name="catch_report")

apps/api/app/main.py

@@ -94,18 +94,18 @@ def ready(db: Db) -> JSONResponse:
 
 
 @app.get("/api/v1/fishes", response_model=list[FishOut])
-def fishes(db: Db) -> list[Fish]:
-    return list(db.scalars(select(Fish).order_by(Fish.name_ru)))
+def fishes(db: Db, limit: int = Query(200, ge=1, le=500), offset: int = Query(0, ge=0)) -> list[Fish]:
+    return list(db.scalars(select(Fish).order_by(Fish.name_ru, Fish.id).offset(offset).limit(limit)))
 
 
 @app.get("/api/v1/waterbodies", response_model=list[WaterbodyOut])
-def waterbodies(db: Db) -> list[Waterbody]:
-    return list(db.scalars(select(Waterbody).order_by(Waterbody.name_ru)))
+def waterbodies(db: Db, limit: int = Query(200, ge=1, le=500), offset: int = Query(0, ge=0)) -> list[Waterbody]:
+    return list(db.scalars(select(Waterbody).order_by(Waterbody.name_ru, Waterbody.id).offset(offset).limit(limit)))
 
 
 @app.get("/api/v1/baits", response_model=list[BaitOut])
-def baits(db: Db) -> list[Bait]:
-    return list(db.scalars(select(Bait).order_by(Bait.name)))
+def baits(db: Db, limit: int = Query(200, ge=1, le=500), offset: int = Query(0, ge=0)) -> list[Bait]:
+    return list(db.scalars(select(Bait).order_by(Bait.name, Bait.id).offset(offset).limit(limit)))
 
 
 @app.get("/api/v1/activity", response_model=list[ActivityOut])
@@ -119,7 +119,11 @@ def activity(
     if hours not in {6, 12, 24, 72}:
         raise HTTPException(status_code=422, detail="hours must be one of: 6, 12, 24, 72")
     rows = activity_rows(db, hours=hours, waterbody=waterbody, fish=fish, method=method)
-    keys = {"activity": lambda r: r.activity_score, "confidence": lambda r: r.confidence_score, "freshness": lambda r: r.last_confirmed_at}
+    keys = {
+        "activity": lambda r: (r.activity_score, r.confidence_score, r.last_confirmed_at, str(r.spot_id)),
+        "confidence": lambda r: (r.confidence_score, r.activity_score, r.last_confirmed_at, str(r.spot_id)),
+        "freshness": lambda r: (r.last_confirmed_at, r.activity_score, r.confidence_score, str(r.spot_id)),
+    }
     rows.sort(key=keys[sort], reverse=True)
     return rows[offset:offset + limit]
 
@@ -145,7 +149,7 @@ def spot_detail(spot_id: UUID, db: Db) -> SpotOut:
 @app.get("/api/v1/spots/{spot_id}/catches", response_model=list[CatchOut])
 def spot_catches(spot_id: UUID, db: Db, limit: int = Query(50, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[CatchOut]:
     _spot_or_404(db, spot_id)
-    reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at.desc()).offset(offset).limit(limit)))
+    reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.bait)).where(CatchReport.spot_id == spot_id, CatchReport.moderation_status == ModerationStatus.approved, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at.desc(), CatchReport.id.desc()).offset(offset).limit(limit)))
     return [CatchOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, bait=r.bait.name if r.bait else None, player_name=r.player_name, caught_at=r.caught_at, reported_at=r.reported_at, retrieve_method=r.retrieve_method, retrieve_speed=r.retrieve_speed) for r in reports]
 
 
@@ -160,9 +164,9 @@ def records(
         query = query.join(CatchReport.fish).where(Fish.slug == fish)
     if waterbody:
         query = query.join(CatchReport.waterbody).where(Waterbody.slug == waterbody)
-    items = list(db.scalars(query.order_by(CatchReport.caught_at.desc(), CatchReport.weight_g.desc()).offset(offset).limit(limit)))
     if category:
-        items = [item for item in items if (item.raw_payload or {}).get("category") == category]
+        query = query.where(CatchReport.raw_payload["category"].as_string() == category)
+    items = list(db.scalars(query.order_by(CatchReport.caught_at.desc(), CatchReport.weight_g.desc(), CatchReport.id.desc()).offset(offset).limit(limit)))
     return [OfficialRecordOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, waterbody=r.waterbody.name_ru, bait=r.bait.name if r.bait else None, player_name=r.player_name, record_date=r.caught_at, category=(r.raw_payload or {}).get("category"), region=(r.raw_payload or {}).get("region"), source_url=r.source_url) for r in items]
 
 
@@ -174,8 +178,8 @@ def _admin(authorization: Annotated[str | None, Header()] = None) -> str:
 
 
 @app.get("/api/v1/imports", response_model=list[ImportRunOut])
-def imports(db: Db, limit: int = Query(20, ge=1, le=100)) -> list[OfficialRecordImport]:
-    return list(db.scalars(select(OfficialRecordImport).order_by(OfficialRecordImport.started_at.desc()).limit(limit)))
+def imports(db: Db, limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[OfficialRecordImport]:
+    return list(db.scalars(select(OfficialRecordImport).order_by(OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc()).offset(offset).limit(limit)))
 
 
 @app.get("/api/v1/admin/imports", response_model=list[ImportRunOut])
@@ -221,7 +225,8 @@ def _external_out(item: ExternalObservation) -> ExternalObservationOut:
 
 @app.get("/api/v1/admin/external-observations", response_model=list[ExternalObservationOut])
 def admin_external_observations(
-    db: Db, _: Annotated[str, Depends(_admin)], status: str | None = None,
+    db: Db, _: Annotated[str, Depends(_admin)],
+    status: Literal["staged", "mapped", "ready", "published", "rejected"] | None = None,
     source_system: str | None = None, limit: int = Query(50, ge=1, le=200), offset: int = Query(0, ge=0),
 ) -> list[ExternalObservationOut]:
     query = select(ExternalObservation).options(
@@ -231,7 +236,7 @@ def admin_external_observations(
         query = query.where(ExternalObservation.status == status)
     if source_system:
         query = query.where(ExternalObservation.source_system == source_system)
-    items = db.scalars(query.order_by(ExternalObservation.last_seen_at.desc()).offset(offset).limit(limit))
+    items = db.scalars(query.order_by(ExternalObservation.last_seen_at.desc(), ExternalObservation.id.desc()).offset(offset).limit(limit))
     return [_external_out(item) for item in items]
 
 
@@ -332,8 +337,8 @@ def add_screenshot(
 
 
 @app.get("/api/v1/admin/catch-reports", response_model=list[AdminCatchReportOut])
-def admin_reports(db: Db, _: Annotated[str, Depends(_admin)], status: ModerationStatus = ModerationStatus.pending, limit: int = Query(50, ge=1, le=100)) -> list[AdminCatchReportOut]:
-    reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), joinedload(CatchReport.spot), joinedload(CatchReport.bait)).where(CatchReport.source_type == SourceType.user, CatchReport.moderation_status == status, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at).limit(limit)))
+def admin_reports(db: Db, _: Annotated[str, Depends(_admin)], status: ModerationStatus = ModerationStatus.pending, limit: int = Query(50, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[AdminCatchReportOut]:
+    reports = list(db.scalars(select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), joinedload(CatchReport.spot), joinedload(CatchReport.bait)).where(CatchReport.source_type == SourceType.user, CatchReport.moderation_status == status, CatchReport.deleted_at.is_(None)).order_by(CatchReport.reported_at, CatchReport.id).offset(offset).limit(limit)))
     return [AdminCatchReportOut(id=r.id, fish=r.fish.name_ru, waterbody=r.waterbody.name_ru, coordinates=f"{r.spot.x}:{r.spot.y}" if r.spot else "—", weight_g=r.weight_g, bait=r.bait.name if r.bait else None, player_name=r.player_name, reported_at=r.reported_at, moderation_status=r.moderation_status.value, comment=(r.raw_payload or {}).get("comment"), screenshot_url=signed_screenshot_url(r.screenshot_key) if r.screenshot_key else None) for r in reports]
 
 

apps/api/tests/test_api.py

@@ -56,6 +56,14 @@ def test_invalid_period_is_rejected() -> None:
     assert client.get("/api/v1/activity?sort=unknown").status_code == 422
 
 
+def test_list_pagination_and_filter_validation() -> None:
+    assert client.get("/api/v1/fishes?limit=0").status_code == 422
+    assert client.get("/api/v1/fishes?limit=1&offset=0").status_code == 200
+    headers = {"Authorization": "Bearer change-me-in-production"}
+    assert client.get("/api/v1/admin/external-observations?status=unknown", headers=headers).status_code == 422
+    assert client.get("/api/v1/admin/catch-reports?offset=-1", headers=headers).status_code == 422
+
+
 def test_liveness_does_not_probe_dependencies() -> None:
     response = client.get("/health?token=must-not-be-logged")
     assert response.json() == {"status": "ok"}
@@ -80,6 +88,22 @@ def test_records_list_is_empty_before_import() -> None:
     assert response.json() == []
 
 
+def test_record_category_filter_is_applied_before_pagination() -> None:
+    with Session(engine) as db:
+        fish = db.scalar(select(Fish).where(Fish.slug == "pike"))
+        waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == "test-lake"))
+        now = datetime.now(timezone.utc) - timedelta(days=30)
+        db.add_all([
+            CatchReport(fish=fish, waterbody=waterbody, weight_g=9000, caught_at=now, reported_at=now, source_type=SourceType.official_record, source_confidence=100, moderation_status=ModerationStatus.approved, raw_payload={"category": "other"}),
+            CatchReport(fish=fish, waterbody=waterbody, weight_g=8000, caught_at=now - timedelta(days=1), reported_at=now, source_type=SourceType.official_record, source_confidence=100, moderation_status=ModerationStatus.approved, raw_payload={"category": "wanted"}),
+        ])
+        db.commit()
+    response = client.get("/api/v1/records?category=wanted&limit=1")
+    assert response.status_code == 200
+    assert len(response.json()) == 1
+    assert response.json()[0]["category"] == "wanted"
+
+
 def test_user_report_requires_moderation_before_activity() -> None:
     created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 77, "y": 88, "weight_g": 5500, "bait_name": "Новая приманка", "player_name": "Reporter"})
     assert created.status_code == 201