fix: audit P0-P1 — T03-T07, D01-D08, U01-U02 (#1788956171115)

apps/api/app/activity.py

@@ -56,7 +56,7 @@ def activity_rows(
         activity = round(55 * min(1, weighted / 12) + 25 * min(1, len(players) / 6) + 20 * min(1, trophies / 3))
         average_confidence = sum(r.source_confidence for r in items) / len(items)
         confidence = round(45 * min(1, len(items) / 10) + 35 * min(1, len(players) / 5) + 20 * average_confidence / 100)
-        latest = max(_aware(r.caught_at or r.reported_at) for r in items)
+        latest = max(_aware(r.reported_at) for r in items)
         baits = Counter(r.bait.name for r in items if r.bait)
         freshness_text = _freshness_text(now - latest)
         result.append(ActivityOut(

apps/api/app/community_importer.py

@@ -9,7 +9,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from .community_review import publish_observation
-from .models import DataSource, ExternalEntityAlias, ExternalObservation, ModerationStatus
+from .models import DataSource, ExternalEntityAlias, ExternalObservation, ModerationStatus, Waterbody
 
 
 SOURCE_DEFAULTS = {
@@ -107,7 +107,6 @@ def _auto_publish(session: Session, observation: ExternalObservation) -> bool:
         observation.status not in {"staged", "mapped", "ready"}
         or not observation.source.enabled
         or observation.fish_external_id is None
-        or observation.waterbody_external_id is None
         or observation.x is None
         or observation.y is None
         or observation.weight_g is None
@@ -118,15 +117,27 @@ def _auto_publish(session: Session, observation: ExternalObservation) -> bool:
         ExternalEntityAlias.entity_type == "fish",
         ExternalEntityAlias.external_id == observation.fish_external_id,
     ))
-    waterbody_alias = session.scalar(select(ExternalEntityAlias).where(
-        ExternalEntityAlias.source_system == observation.source_system,
-        ExternalEntityAlias.entity_type == "waterbody",
-        ExternalEntityAlias.external_id == observation.waterbody_external_id,
-    ))
-    if fish_alias is None or fish_alias.fish is None or waterbody_alias is None or waterbody_alias.waterbody is None:
+    if fish_alias is None or fish_alias.fish is None:
+        return False
+    # Waterbody: prefer external alias, fall back to exact name match
+    waterbody = None
+    if observation.waterbody_external_id is not None:
+        waterbody_alias = session.scalar(select(ExternalEntityAlias).where(
+            ExternalEntityAlias.source_system == observation.source_system,
+            ExternalEntityAlias.entity_type == "waterbody",
+            ExternalEntityAlias.external_id == observation.waterbody_external_id,
+        ))
+        if waterbody_alias and waterbody_alias.waterbody:
+            waterbody = waterbody_alias.waterbody
+    if waterbody is None:
+        # Fallback: exact name match
+        waterbody = session.scalar(
+            select(Waterbody).where(Waterbody.name_ru == observation.waterbody_name)
+        )
+    if waterbody is None:
         return False
     observation.fish = fish_alias.fish
-    observation.waterbody = waterbody_alias.waterbody
+    observation.waterbody = waterbody
     observation.status = "ready"
     observation.review_note = "Automatically matched by previously reviewed source aliases"
     publish_observation(session, observation)

apps/api/app/community_scheduler.py

@@ -27,12 +27,18 @@ def retry_delay(statuses: list[str]) -> int:
     return min(settings.community_import_interval_seconds * (2 ** max(0, failures - 1)), MAX_BACKOFF_SECONDS)
 
 def configured_sources():
+    with SessionLocal() as session:
+        enabled_keys = {
+            s.key for s in session.scalars(select(DataSource).where(DataSource.enabled.is_(True)))
+        }
     return {
-        "rf4db": SOURCES["rf4db"],
-        "rf4stat-fishing": SOURCES["rf4stat-fishing"],
-        "rf4stat-post": (SOURCES["rf4stat-posts"][0], SOURCES["rf4stat-posts"][1]),
-        "rf4map": (settings.rf4map_point_url, parse_rf4map_point),
-        "rf4posts-spot": (settings.rf4posts_spot_url, parse_rf4posts_spot),
+        k: v for k, v in {
+            "rf4db": SOURCES["rf4db"],
+            "rf4stat-fishing": SOURCES["rf4stat-fishing"],
+            "rf4stat-post": (SOURCES["rf4stat-posts"][0], SOURCES["rf4stat-posts"][1]),
+            "rf4map": (settings.rf4map_point_url, parse_rf4map_point),
+            "rf4posts-spot": (settings.rf4posts_spot_url, parse_rf4posts_spot),
+        }.items() if k in enabled_keys
     }
 
 def oldest_site_source(source_system: str, latest_by_source: dict[str, datetime]) -> str:

apps/api/app/main.py

@@ -27,7 +27,7 @@ from .logging_config import configure_logging
 from .models import Bait, BaitKind, CatchReport, CommunityImportRun, DataSource, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
 from .readiness import readiness_report
 from .public_cache import public_cache
-from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, PublicObservationOut, SourceStatusOut, SpotOut, WaterbodyOut
+from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ImportRunPublicOut, ModerationUpdate, OfficialRecordOut, PaginatedActivityOut, PublicObservationOut, SourceStatusOut, SpotOut, WaterbodyOut
 from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot
 
 
@@ -87,6 +87,7 @@ def ready(db: Db) -> JSONResponse:
     is_ready, components = readiness_report(
         db, storage_client(), import_required=settings.official_import_required,
         import_interval_seconds=settings.import_interval_seconds,
+        community_import_interval_seconds=settings.community_import_interval_seconds,
     )
     return JSONResponse(
         status_code=200 if is_ready else 503,
@@ -120,14 +121,14 @@ def public_spot_pages(db: Db, limit: int = Query(500, ge=1, le=500), offset: int
         (f"/spots/{water}-{x}x{y}", f"/waterbodies/{water}/{fish}")]
 
 
-@app.get("/api/v1/activity", response_model=list[ActivityOut])
+@app.get("/api/v1/activity", response_model=PaginatedActivityOut)
 def activity(
     db: Db, response: Response, hours: int = Query(24),
     waterbody: str | None = None, fish: str | None = None,
     method: str | None = None,
     sort: Literal["activity", "confidence", "freshness"] = "activity",
     limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0),
-) -> list[ActivityOut]:
+) -> PaginatedActivityOut:
     if hours not in {6, 12, 24, 72}:
         raise HTTPException(status_code=422, detail="hours must be one of: 6, 12, 24, 72")
     response.headers["Cache-Control"] = "no-store"
@@ -138,14 +139,16 @@ def activity(
         response.headers["X-Cache"] = "HIT"
         return cached
     rows = activity_rows(db, hours=hours, waterbody=waterbody, fish=fish, method=method)
+    total = len(rows)
     keys = {
         "activity": lambda r: (r.activity_score, r.confidence_score, r.last_confirmed_at, str(r.spot_id)),
         "confidence": lambda r: (r.confidence_score, r.activity_score, r.last_confirmed_at, str(r.spot_id)),
         "freshness": lambda r: (r.last_confirmed_at, r.activity_score, r.confidence_score, str(r.spot_id)),
     }
     rows.sort(key=keys[sort], reverse=True)
+    page = rows[offset:offset + limit]
     response.headers["X-Cache"] = "MISS"
-    return public_cache.set(cache_key, rows[offset:offset + limit], generation=generation)
+    return public_cache.set(cache_key, PaginatedActivityOut(items=page, total=total, limit=limit, offset=offset), generation=generation)
 
 
 def _spot_or_404(db: Session, spot_id: UUID) -> Spot:
@@ -218,19 +221,18 @@ def _report_source(report: CatchReport) -> str:
 @app.get("/api/v1/community-observations", response_model=list[PublicObservationOut])
 def community_observations(
     db: Db, limit: int = Query(12, ge=1, le=50), offset: int = Query(0, ge=0),
+    waterbody: str | None = None, fish: str | None = None,
 ) -> list[PublicObservationOut]:
-    items = list(db.scalars(
-        select(ExternalObservation)
-        .join(ExternalObservation.source)
-        .options(joinedload(ExternalObservation.source))
-        .where(
-            ExternalObservation.catch_report_id.is_(None),
-            ExternalObservation.status != "rejected",
-            DataSource.enabled.is_(True),
-        )
-        .order_by(ExternalObservation.last_seen_at.desc(), ExternalObservation.id.desc())
-        .offset(offset).limit(limit)
-    ))
+    query = select(ExternalObservation).join(ExternalObservation.source).options(joinedload(ExternalObservation.source)).where(
+        ExternalObservation.catch_report_id.is_(None),
+        ExternalObservation.status != "rejected",
+        DataSource.enabled.is_(True),
+    )
+    if waterbody:
+        query = query.where(ExternalObservation.waterbody_name == waterbody)
+    if fish:
+        query = query.where(ExternalObservation.fish_name == fish)
+    items = list(db.scalars(query.order_by(ExternalObservation.last_seen_at.desc(), ExternalObservation.id.desc()).offset(offset).limit(limit)))
     result: list[PublicObservationOut] = []
     for item in items:
         missing = []
@@ -318,7 +320,7 @@ def admin_diagnostics(db: Db, _: Annotated[str, Depends(_admin)]) -> JSONRespons
     return JSONResponse(payload, headers={"Content-Disposition": "attachment; filename=rf4spotter-diagnostics.json"})
 
 
-@app.get("/api/v1/imports", response_model=list[ImportRunOut])
+@app.get("/api/v1/imports", response_model=list[ImportRunPublicOut])
 def imports(db: Db, limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0)) -> list[OfficialRecordImport]:
     return list(db.scalars(select(OfficialRecordImport).order_by(OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc()).offset(offset).limit(limit)))
 
@@ -448,7 +450,7 @@ def admin_reject_external_observation(
 def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportAccepted:
     if payload.website:
         raise HTTPException(status_code=400, detail="invalid submission")
-    _check_rate_limit(request.client.host if request.client else "unknown", db)
+    _check_rate_limit(request, db)
     fish = db.scalar(select(Fish).where(Fish.slug == payload.fish_slug))
     waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == payload.waterbody_slug))
     if fish is None or waterbody is None:
@@ -536,9 +538,14 @@ def delete_report(report_id: UUID, db: Db, moderator: Annotated[str, Depends(_ad
     return Response(status_code=204)
 
 
-def _check_rate_limit(client: str, db: Session) -> None:
+def _check_rate_limit(request: Request, db: Session) -> None:
     now = datetime.now(timezone.utc)
     cutoff = now - timedelta(minutes=10)
+    # Extract real client IP from forwarded headers
+    client = request.client.host if request.client else "unknown"
+    forwarded = request.headers.get("x-forwarded-for")
+    if forwarded:
+        client = forwarded.split(",")[0].strip()
     client_hash = hmac.new(settings.rate_limit_secret.encode(), client.encode(), hashlib.sha256).hexdigest()
     if db.get_bind().dialect.name == "postgresql":
         lock_key = int(client_hash[:16], 16) & 0x7FFF_FFFF_FFFF_FFFF

apps/api/app/readiness.py

@@ -6,12 +6,13 @@ from typing import Any
 from sqlalchemy import select, text
 from sqlalchemy.orm import Session
 
-from .models import ImportStatus, OfficialRecordImport
+from .models import CommunityImportRun, ImportStatus, OfficialRecordImport
 
 
 def readiness_report(
     session: Session, s3: Any, *, import_required: bool,
-    import_interval_seconds: int, now: datetime | None = None,
+    import_interval_seconds: int, community_import_interval_seconds: int = 1800,
+    now: datetime | None = None,
 ) -> tuple[bool, dict[str, dict[str, object]]]:
     current = now or datetime.now(timezone.utc)
     components: dict[str, dict[str, object]] = {}
@@ -58,4 +59,26 @@ def readiness_report(
         if import_required:
             ready = False
 
+    # Check community scheduler: look for recent import runs
+    try:
+        latest_community = session.scalar(
+            select(CommunityImportRun)
+            .order_by(CommunityImportRun.started_at.desc())
+            .limit(1)
+        )
+        if latest_community is None:
+            components["community_scheduler"] = {"status": "not_started"}
+        else:
+            started = latest_community.started_at
+            if started.tzinfo is None:
+                started = started.replace(tzinfo=timezone.utc)
+            stale = started < current - timedelta(seconds=community_import_interval_seconds * 2)
+            healthy = latest_community.status == "success" and not stale
+            components["community_scheduler"] = {
+                "status": "ready" if healthy else ("stale" if stale else latest_community.status),
+                "last_started_at": started.isoformat(),
+            }
+    except Exception:
+        components["community_scheduler"] = {"status": "unknown"}
+
     return ready, components

apps/api/app/schemas.py

@@ -50,6 +50,13 @@ class ActivityOut(BaseModel):
     sources: list[str]
 
 
+class PaginatedActivityOut(BaseModel):
+    items: list[ActivityOut]
+    total: int
+    limit: int
+    offset: int
+
+
 class CatchOut(BaseModel):
     id: UUID
     fish: str
@@ -125,6 +132,18 @@ class ImportRunOut(BaseModel):
     not_modified: bool
 
 
+class ImportRunPublicOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: UUID
+    started_at: datetime
+    finished_at: datetime | None
+    status: str
+    rows_seen: int
+    rows_created: int
+    rows_updated: int
+    not_modified: bool
+
+
 class CatchReportCreate(BaseModel):
     fish_slug: str
     waterbody_slug: str