From b73580c720a95288e5edaa7b03660359367768f2 Mon Sep 17 00:00:00 2001 From: IK Date: Thu, 3 Sep 2026 08:16:12 +0700 Subject: [PATCH] Persist privacy-safe submission rate limits --- .env.example | 1 + .../versions/0007_persistent_rate_limit.py | 25 ++++++++++++++++ apps/api/app/config.py | 1 + apps/api/app/main.py | 29 ++++++++++++------- apps/api/app/models.py | 7 +++++ apps/api/tests/test_rate_limit.py | 27 +++++++++++++++++ compose.yaml | 1 + docs/ROADMAP.md | 2 +- 8 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 apps/api/alembic/versions/0007_persistent_rate_limit.py create mode 100644 apps/api/tests/test_rate_limit.py diff --git a/.env.example b/.env.example index 8499841..262e4e1 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,4 @@ OFFICIAL_RECORDS_URL=https://rf4game.de/records/region/RU/ OFFICIAL_RECORDS_REGION=RU OFFICIAL_RECORDS_CATEGORY=records IMPORT_INTERVAL_SECONDS=3600 +RATE_LIMIT_SECRET=change-rate-limit-secret diff --git a/apps/api/alembic/versions/0007_persistent_rate_limit.py b/apps/api/alembic/versions/0007_persistent_rate_limit.py new file mode 100644 index 0000000..9178e14 --- /dev/null +++ b/apps/api/alembic/versions/0007_persistent_rate_limit.py @@ -0,0 +1,25 @@ +"""Persistent privacy-preserving submission rate limit.""" +from alembic import op +import sqlalchemy as sa + +revision = "0007" +down_revision = "0006" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "submission_attempt", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("client_hash", sa.String(64), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index("ix_submission_attempt_client_hash", "submission_attempt", ["client_hash"]) + op.create_index("ix_submission_attempt_created_at", "submission_attempt", ["created_at"]) + + +def downgrade() -> None: + op.drop_index("ix_submission_attempt_created_at", table_name="submission_attempt") + op.drop_index("ix_submission_attempt_client_hash", table_name="submission_attempt") + op.drop_table("submission_attempt") diff --git a/apps/api/app/config.py b/apps/api/app/config.py index 28a428a..c9cb3fa 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -15,6 +15,7 @@ class Settings(BaseSettings): official_records_region: str = "RU" official_records_category: str = "records" import_interval_seconds: int = Field(default=3600, ge=3600) + rate_limit_secret: str = "change-rate-limit-secret" model_config = SettingsConfigDict(env_file=".env", extra="ignore") diff --git a/apps/api/app/main.py b/apps/api/app/main.py index cf35704..77e93a3 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -1,21 +1,23 @@ from __future__ import annotations -from collections import Counter, defaultdict, deque +from collections import Counter from datetime import datetime, timedelta, timezone +import hashlib +import hmac from typing import Annotated, Literal from uuid import UUID import httpx from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, Response, UploadFile from fastapi.middleware.cors import CORSMiddleware -from sqlalchemy import select +from sqlalchemy import delete, func, select, text from sqlalchemy.orm import Session, joinedload from .activity import activity_rows from .database import get_session from .config import settings from .importer import ImportSourceError, import_records, normalize -from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody +from .models import Bait, BaitKind, CatchReport, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut from .storage import ScreenshotError, delete_screenshot, signed_screenshot_url, upload_screenshot @@ -28,7 +30,6 @@ app.add_middleware( allow_headers=["Authorization", "Content-Type"], ) Db = Annotated[Session, Depends(get_session)] -_submissions: dict[str, deque[datetime]] = defaultdict(deque) @app.get("/health") @@ -150,7 +151,7 @@ def admin_start_official_import(db: Db, _: Annotated[str, Depends(_admin)]) -> O def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated: if payload.website: raise HTTPException(status_code=400, detail="invalid submission") - _check_rate_limit(request.client.host if request.client else "unknown") + _check_rate_limit(request.client.host if request.client else "unknown", db) fish = db.scalar(select(Fish).where(Fish.slug == payload.fish_slug)) waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == payload.waterbody_slug)) if fish is None or waterbody is None: @@ -228,14 +229,20 @@ def delete_report(report_id: UUID, db: Db, moderator: Annotated[str, Depends(_ad return Response(status_code=204) -def _check_rate_limit(client: str) -> None: +def _check_rate_limit(client: str, db: Session) -> None: now = datetime.now(timezone.utc) - recent = _submissions[client] - while recent and recent[0] < now - timedelta(minutes=10): - recent.popleft() - if len(recent) >= 5: + cutoff = now - timedelta(minutes=10) + client_hash = hmac.new(settings.rate_limit_secret.encode(), client.encode(), hashlib.sha256).hexdigest() + if db.get_bind().dialect.name == "postgresql": + lock_key = int(client_hash[:16], 16) & 0x7FFF_FFFF_FFFF_FFFF + db.execute(text("SELECT pg_advisory_xact_lock(:lock_key)"), {"lock_key": lock_key}) + db.execute(delete(SubmissionAttempt).where(SubmissionAttempt.created_at < now - timedelta(days=1))) + recent = db.scalar(select(func.count()).select_from(SubmissionAttempt).where(SubmissionAttempt.client_hash == client_hash, SubmissionAttempt.created_at >= cutoff)) or 0 + if recent >= 5: + db.commit() raise HTTPException(status_code=429, detail="too many submissions") - recent.append(now) + db.add(SubmissionAttempt(client_hash=client_hash, created_at=now)) + db.commit() def _aware(value: datetime) -> datetime: diff --git a/apps/api/app/models.py b/apps/api/app/models.py index 6d109a8..4281069 100644 --- a/apps/api/app/models.py +++ b/apps/api/app/models.py @@ -129,3 +129,10 @@ class ModerationEvent(Base): moderator: Mapped[str] = mapped_column(String(100)) reason: Mapped[str | None] = mapped_column(Text) catch_report: Mapped[CatchReport] = relationship() + + +class SubmissionAttempt(Base): + __tablename__ = "submission_attempt" + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + client_hash: Mapped[str] = mapped_column(String(64), index=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) diff --git a/apps/api/tests/test_rate_limit.py b/apps/api/tests/test_rate_limit.py new file mode 100644 index 0000000..4c4dabc --- /dev/null +++ b/apps/api/tests/test_rate_limit.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session + +from app.database import Base +from app.main import _check_rate_limit +from app.models import SubmissionAttempt + + +def test_rate_limit_is_persistent_and_does_not_store_raw_client() -> None: + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + with Session(engine) as db: + for _ in range(5): + _check_rate_limit("203.0.113.42", db) + with pytest.raises(HTTPException) as blocked: + _check_rate_limit("203.0.113.42", db) + assert blocked.value.status_code == 429 + attempts = list(db.scalars(select(SubmissionAttempt))) + assert len(attempts) == 5 + assert all(item.client_hash != "203.0.113.42" and len(item.client_hash) == 64 for item in attempts) + assert all(item.created_at.replace(tzinfo=timezone.utc) <= datetime.now(timezone.utc) for item in attempts) diff --git a/compose.yaml b/compose.yaml index 857a508..02826be 100644 --- a/compose.yaml +++ b/compose.yaml @@ -38,6 +38,7 @@ services: OFFICIAL_RECORDS_URL: ${OFFICIAL_RECORDS_URL:-https://rf4game.de/records/region/RU/} OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU} OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records} + RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:-change-rate-limit-secret} depends_on: db: condition: service_healthy diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 7037602..0881e43 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -32,7 +32,7 @@ - [x] Добавить административный веб-интерфейс очереди модерации поверх существующего API (`/admin/moderation`, токен только в памяти страницы). - [x] Показать скриншот, данные улова и причину решения; реализовать действия «одобрить» и «отклонить» (проверено в браузере на desktop и 390 px). - [x] Добавить удаление пользовательского сообщения администратором с аудитом действия (обезличивание записи, удаление объекта MinIO, миграция `0006`). -- [ ] Заменить in-memory rate limit на общее хранилище, пригодное для нескольких API-процессов и перезапусков. +- [x] Заменить in-memory rate limit на общее хранилище, пригодное для нескольких API-процессов и перезапусков (PostgreSQL, HMAC-отпечаток без хранения исходного IP, миграция `0007`). - [ ] Валидировать одновременно содержимое, MIME, расширение и лимит изображения; добавить тесты каждого отказа. - [ ] Добавить сквозной тест: отправка → pending → модерация → появление одобренного улова в публичной статистике. - [ ] Добавить понятные состояния успеха и ошибок загрузки в форму, включая отдельную ошибку скриншота без потери уже созданной заявки.