Persist privacy-safe submission rate limits

apps/api/app/config.py

@@ -15,6 +15,7 @@ class Settings(BaseSettings):
     official_records_region: str = "RU"
     official_records_category: str = "records"
     import_interval_seconds: int = Field(default=3600, ge=3600)
+    rate_limit_secret: str = "change-rate-limit-secret"
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
 
 

apps/api/app/main.py

@@ -1,21 +1,23 @@
 from __future__ import annotations
 
-from collections import Counter, defaultdict, deque
+from collections import Counter
 from datetime import datetime, timedelta, timezone
+import hashlib
+import hmac
 from typing import Annotated, Literal
 from uuid import UUID
 
 import httpx
 from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
-from sqlalchemy import select
+from sqlalchemy import delete, func, select, text
 from sqlalchemy.orm import Session, joinedload
 
 from .activity import activity_rows
 from .database import get_session
 from .config import settings
 from .importer import ImportSourceError, import_records, normalize
-from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
+from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
 from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
 from .storage import ScreenshotError, delete_screenshot, signed_screenshot_url, upload_screenshot
 
@@ -28,7 +30,6 @@ app.add_middleware(
     allow_headers=["Authorization", "Content-Type"],
 )
 Db = Annotated[Session, Depends(get_session)]
-_submissions: dict[str, deque[datetime]] = defaultdict(deque)
 
 
 @app.get("/health")
@@ -150,7 +151,7 @@ def admin_start_official_import(db: Db, _: Annotated[str, Depends(_admin)]) -> O
 def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated:
     if payload.website:
         raise HTTPException(status_code=400, detail="invalid submission")
-    _check_rate_limit(request.client.host if request.client else "unknown")
+    _check_rate_limit(request.client.host if request.client else "unknown", db)
     fish = db.scalar(select(Fish).where(Fish.slug == payload.fish_slug))
     waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == payload.waterbody_slug))
     if fish is None or waterbody is None:
@@ -228,14 +229,20 @@ def delete_report(report_id: UUID, db: Db, moderator: Annotated[str, Depends(_ad
     return Response(status_code=204)
 
 
-def _check_rate_limit(client: str) -> None:
+def _check_rate_limit(client: str, db: Session) -> None:
     now = datetime.now(timezone.utc)
-    recent = _submissions[client]
-    while recent and recent[0] < now - timedelta(minutes=10):
-        recent.popleft()
-    if len(recent) >= 5:
+    cutoff = now - timedelta(minutes=10)
+    client_hash = hmac.new(settings.rate_limit_secret.encode(), client.encode(), hashlib.sha256).hexdigest()
+    if db.get_bind().dialect.name == "postgresql":
+        lock_key = int(client_hash[:16], 16) & 0x7FFF_FFFF_FFFF_FFFF
+        db.execute(text("SELECT pg_advisory_xact_lock(:lock_key)"), {"lock_key": lock_key})
+    db.execute(delete(SubmissionAttempt).where(SubmissionAttempt.created_at < now - timedelta(days=1)))
+    recent = db.scalar(select(func.count()).select_from(SubmissionAttempt).where(SubmissionAttempt.client_hash == client_hash, SubmissionAttempt.created_at >= cutoff)) or 0
+    if recent >= 5:
+        db.commit()
         raise HTTPException(status_code=429, detail="too many submissions")
-    recent.append(now)
+    db.add(SubmissionAttempt(client_hash=client_hash, created_at=now))
+    db.commit()
 
 
 def _aware(value: datetime) -> datetime:

apps/api/app/models.py

@@ -129,3 +129,10 @@ class ModerationEvent(Base):
     moderator: Mapped[str] = mapped_column(String(100))
     reason: Mapped[str | None] = mapped_column(Text)
     catch_report: Mapped[CatchReport] = relationship()
+
+
+class SubmissionAttempt(Base):
+    __tablename__ = "submission_attempt"
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+    client_hash: Mapped[str] = mapped_column(String(64), index=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)