From 870d9cc7f995e50fc6bd29328fbed8331151610c Mon Sep 17 00:00:00 2001 From: IK Date: Sun, 6 Sep 2026 14:08:07 +0700 Subject: [PATCH] feat: harden production data and backups --- README.md | 2 +- .../versions/0010_screenshot_upload_token.py | 16 +++++++ apps/api/app/config.py | 3 ++ apps/api/app/main.py | 21 ++++++--- apps/api/app/models.py | 1 + apps/api/app/schemas.py | 4 ++ apps/api/app/seed.py | 45 ++++++++++++++----- apps/api/tests/test_api.py | 9 ++-- apps/api/tests/test_config.py | 2 + apps/api/tests/test_seed.py | 27 +++++++++++ apps/web/src/pages/api/report-screenshot.ts | 9 ++-- apps/web/src/pages/api/report.ts | 19 +++++--- compose.production.yaml | 9 ++++ deploy/README.md | 25 ++++++++--- deploy/backup.sh | 26 +++++++++++ deploy/restore.sh | 27 +++++++++++ docs/ROADMAP.md | 6 +-- 17 files changed, 215 insertions(+), 36 deletions(-) create mode 100644 apps/api/alembic/versions/0010_screenshot_upload_token.py create mode 100644 apps/api/tests/test_seed.py create mode 100755 deploy/backup.sh create mode 100755 deploy/restore.sh diff --git a/README.md b/README.md index c72bb36..7468008 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ RF4 Spotter — неофициальный сервис свежих точек Подробный план и актуальные чекбоксы находятся в [`docs/ROADMAP.md`](docs/ROADMAP.md). Результаты проверки интерфейса и пять приоритетных UX-пакетов описаны в [`docs/UI_UX_AUDIT.md`](docs/UI_UX_AUDIT.md). -Production-контур для домена `rf4spotter.ru`, TLS, секреты и команды первого запуска описаны в [`deploy/README.md`](deploy/README.md). Он использует отдельный `compose.production.yaml`; локальный `compose.yaml` остаётся средой разработки. +Production-контур для домена `rf4spotter.ru`, TLS, секреты, backup/restore и команды первого запуска описаны в [`deploy/README.md`](deploy/README.md). Он использует отдельный `compose.production.yaml`; локальный `compose.yaml` остаётся средой разработки. Production seed добавляет только справочники — демонстрационные уловы отключены. Gitea Actions workflow `.gitea/workflows/ci.yml` на каждый push и pull request проверяет Python, миграции на чистой PostgreSQL, Astro build и полный Compose/Playwright-сценарий. При падении E2E сохраняются логи контейнеров и Playwright-артефакты. diff --git a/apps/api/alembic/versions/0010_screenshot_upload_token.py b/apps/api/alembic/versions/0010_screenshot_upload_token.py new file mode 100644 index 0000000..86e5fab --- /dev/null +++ b/apps/api/alembic/versions/0010_screenshot_upload_token.py @@ -0,0 +1,16 @@ +"""Protect pending screenshot uploads with a one-time token.""" +from alembic import op +import sqlalchemy as sa + +revision = "0010" +down_revision = "0009" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("catch_report", sa.Column("screenshot_upload_token_hash", sa.String(64))) + + +def downgrade() -> None: + op.drop_column("catch_report", "screenshot_upload_token_hash") diff --git a/apps/api/app/config.py b/apps/api/app/config.py index 13b95db..6172b1e 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -16,6 +16,7 @@ class Settings(BaseSettings): official_records_region: str = "RU" official_records_category: str = "records" official_import_required: bool = False + seed_demo_data: bool = True import_interval_seconds: int = Field(default=3600, ge=3600) rate_limit_secret: str = "change-rate-limit-secret" log_level: str = "INFO" @@ -39,6 +40,8 @@ class Settings(BaseSettings): raise ValueError("production CORS_ORIGINS must contain only HTTPS origins") if not self.s3_public_endpoint_url.startswith("https://"): raise ValueError("production S3_PUBLIC_ENDPOINT_URL must use HTTPS") + if self.seed_demo_data: + raise ValueError("SEED_DEMO_DATA must be false in production") return self diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 864bd67..d7e8104 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone import hashlib import hmac import logging +import secrets import time as time_module from typing import Annotated, Literal from uuid import UUID @@ -25,7 +26,7 @@ from .importer import ImportSourceError, import_records, normalize from .logging_config import configure_logging from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody from .readiness import readiness_report -from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut +from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot @@ -273,8 +274,8 @@ def admin_reject_external_observation( raise HTTPException(status_code=409, detail=str(exc)) from exc -@app.post("/api/v1/catch-reports", response_model=CatchReportCreated, status_code=201) -def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated: +@app.post("/api/v1/catch-reports", response_model=CatchReportAccepted, status_code=201) +def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportAccepted: if payload.website: raise HTTPException(status_code=400, detail="invalid submission") _check_rate_limit(request.client.host if request.client else "unknown", db) @@ -293,17 +294,24 @@ def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> if bait is None: bait = Bait(name=payload.bait_name.strip(), normalized_name=key, kind=BaitKind.unknown) db.add(bait) - report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None) + upload_token = secrets.token_urlsafe(32) + report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None, screenshot_upload_token_hash=hashlib.sha256(upload_token.encode()).hexdigest()) db.add(report) db.commit() - return CatchReportCreated(id=report.id, moderation_status=report.moderation_status.value) + return CatchReportAccepted(id=report.id, moderation_status=report.moderation_status.value, screenshot_upload_token=upload_token) @app.post("/api/v1/catch-reports/{report_id}/screenshot", status_code=204, response_class=Response) -def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) -> Response: +def add_screenshot( + report_id: UUID, db: Db, screenshot: UploadFile = File(), + upload_token: Annotated[str | None, Header(alias="X-Upload-Token")] = None, +) -> Response: report = db.get(CatchReport, report_id) if report is None or report.source_type != SourceType.user or report.moderation_status != ModerationStatus.pending: raise HTTPException(status_code=404, detail="pending catch report not found") + supplied_hash = hashlib.sha256((upload_token or "").encode()).hexdigest() + if not report.screenshot_upload_token_hash or not hmac.compare_digest(report.screenshot_upload_token_hash, supplied_hash): + raise HTTPException(status_code=401, detail="invalid screenshot upload token") if report.screenshot_key: raise HTTPException(status_code=409, detail="screenshot already uploaded") raw = screenshot.file.read(settings.screenshot_max_bytes + 1) @@ -311,6 +319,7 @@ def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) -> report.screenshot_key = upload_screenshot(raw, filename=screenshot.filename, content_type=screenshot.content_type) except ScreenshotError as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc + report.screenshot_upload_token_hash = None db.commit() return Response(status_code=204) diff --git a/apps/api/app/models.py b/apps/api/app/models.py index baee951..e54b155 100644 --- a/apps/api/app/models.py +++ b/apps/api/app/models.py @@ -92,6 +92,7 @@ class CatchReport(Base): source_confidence: Mapped[int] moderation_status: Mapped[ModerationStatus] = mapped_column(Enum(ModerationStatus)) screenshot_key: Mapped[str | None] = mapped_column(Text) + screenshot_upload_token_hash: Mapped[str | None] = mapped_column(String(64)) deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) raw_payload: Mapped[dict | None] = mapped_column(JSON) fish: Mapped[Fish] = relationship() diff --git a/apps/api/app/schemas.py b/apps/api/app/schemas.py index 0381ed9..df2f0ff 100644 --- a/apps/api/app/schemas.py +++ b/apps/api/app/schemas.py @@ -137,6 +137,10 @@ class CatchReportCreated(BaseModel): moderation_status: str +class CatchReportAccepted(CatchReportCreated): + screenshot_upload_token: str + + class AdminCatchReportOut(BaseModel): id: UUID fish: str diff --git a/apps/api/app/seed.py b/apps/api/app/seed.py index 36f53be..e7fffa6 100644 --- a/apps/api/app/seed.py +++ b/apps/api/app/seed.py @@ -5,6 +5,7 @@ from uuid import UUID from sqlalchemy import select +from .config import settings from .database import SessionLocal from .models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody @@ -23,19 +24,38 @@ IDS = { def seed() -> None: with SessionLocal.begin() as db: - if db.scalar(select(Fish.id).limit(1)) is not None: + def entity(model, key: str, value: str, **values): + item = db.scalar(select(model).where(getattr(model, key) == value)) + if item is None: + item = model(**values) + db.add(item) + db.flush() + return item + + vyunok = entity(Waterbody, "slug", "vyunok", id=IDS["vyunok"], slug="vyunok", name_ru="Вьюнок", unlock_level=1) + kuori = entity(Waterbody, "slug", "kuori", id=IDS["kuori"], slug="kuori", name_ru="Куори", unlock_level=16) + pike = entity(Fish, "slug", "pike", id=IDS["pike"], slug="pike", name_ru="Щука", trophy_weight_g=10_000) + trout = entity(Fish, "slug", "lake-trout", id=IDS["trout"], slug="lake-trout", name_ru="Озёрная форель", trophy_weight_g=10_000) + spiker = entity(Bait, "normalized_name", "spiker #2 01-015", id=IDS["spiker"], name="Spiker #2 01-015", normalized_name="spiker #2 01-015", kind=BaitKind.lure) + shad = entity(Bait, "normalized_name", "salmon t1 shad 12 005", id=IDS["shad"], name="Salmon T1 Shad 12 005", normalized_name="salmon t1 shad 12 005", kind=BaitKind.lure) + + def spot(waterbody: Waterbody, identity: str, x: int, y: int, description: str) -> Spot: + item = db.scalar(select(Spot).where(Spot.waterbody_id == waterbody.id, Spot.x == x, Spot.y == y)) + if item is None: + item = Spot(id=IDS[identity], waterbody=waterbody, x=x, y=y, description=description) + db.add(item) + db.flush() + return item + + spot1 = spot(vyunok, "spot1", 110, 103, "Кромка травы у северного берега") + spot2 = spot(kuori, "spot2", 85, 92, "Свальчик в глубину") + if not settings.seed_demo_data: return - vyunok = Waterbody(id=IDS["vyunok"], slug="vyunok", name_ru="Вьюнок", unlock_level=1) - kuori = Waterbody(id=IDS["kuori"], slug="kuori", name_ru="Куори", unlock_level=16) - pike = Fish(id=IDS["pike"], slug="pike", name_ru="Щука", trophy_weight_g=10_000) - trout = Fish(id=IDS["trout"], slug="lake-trout", name_ru="Озёрная форель", trophy_weight_g=10_000) - spiker = Bait(id=IDS["spiker"], name="Spiker #2 01-015", normalized_name="spiker #2 01-015", kind=BaitKind.lure) - shad = Bait(id=IDS["shad"], name="Salmon T1 Shad 12 005", normalized_name="salmon t1 shad 12 005", kind=BaitKind.lure) - spot1 = Spot(id=IDS["spot1"], waterbody=vyunok, x=110, y=103, description="Кромка травы у северного берега") - spot2 = Spot(id=IDS["spot2"], waterbody=kuori, x=85, y=92, description="Свальчик в глубину") - db.add_all([vyunok, kuori, pike, trout, spiker, shad, spot1, spot2]) now = datetime.now(timezone.utc) for index in range(12): + external_id = f"seed:pike:{index}" + if db.scalar(select(CatchReport.id).where(CatchReport.source_external_id == external_id)): + continue db.add(CatchReport( fish=pike, spot=spot1, waterbody=vyunok, bait=spiker, weight_g=2_600 + index * 480, fishing_method="spinning", @@ -43,9 +63,13 @@ def seed() -> None: caught_at=now - timedelta(minutes=25 + index * 47), reported_at=now - timedelta(minutes=20 + index * 47), player_name=f"DemoPlayer{index % 7 + 1}", source_type=SourceType.manual_import, + source_external_id=external_id, source_confidence=80 + index % 3 * 5, moderation_status=ModerationStatus.approved, )) for index in range(5): + external_id = f"seed:trout:{index}" + if db.scalar(select(CatchReport.id).where(CatchReport.source_external_id == external_id)): + continue db.add(CatchReport( fish=trout, spot=spot2, waterbody=kuori, bait=shad, weight_g=4_200 + index * 900, fishing_method="spinning", @@ -53,6 +77,7 @@ def seed() -> None: caught_at=now - timedelta(hours=2 + index * 4), reported_at=now - timedelta(hours=2 + index * 4), player_name=f"DemoAngler{index + 1}", source_type=SourceType.manual_import, + source_external_id=external_id, source_confidence=85, moderation_status=ModerationStatus.approved, )) diff --git a/apps/api/tests/test_api.py b/apps/api/tests/test_api.py index 449d8ef..7127cc0 100644 --- a/apps/api/tests/test_api.py +++ b/apps/api/tests/test_api.py @@ -187,10 +187,13 @@ def test_admin_can_start_and_list_official_import(monkeypatch) -> None: def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None: created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 91, "y": 92, "weight_g": 4200}).json() monkeypatch.setattr("app.main.upload_screenshot", lambda raw, **metadata: "reports/test.jpg" if raw == b"image-bytes" and metadata == {"filename": "catch.jpg", "content_type": "image/jpeg"} else "unexpected") - response = client.post(f"/api/v1/catch-reports/{created['id']}/screenshot", files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}) + upload_url = f"/api/v1/catch-reports/{created['id']}/screenshot" + assert client.post(upload_url, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}).status_code == 401 + assert client.post(upload_url, headers={"X-Upload-Token": "wrong"}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}).status_code == 401 + response = client.post(upload_url, headers={"X-Upload-Token": created["screenshot_upload_token"]}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}) assert response.status_code == 204 - duplicate = client.post(f"/api/v1/catch-reports/{created['id']}/screenshot", files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}) - assert duplicate.status_code == 409 + reused = client.post(upload_url, headers={"X-Upload-Token": created["screenshot_upload_token"]}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}) + assert reused.status_code == 401 def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None: diff --git a/apps/api/tests/test_config.py b/apps/api/tests/test_config.py index a5b45ac..e00ab4e 100644 --- a/apps/api/tests/test_config.py +++ b/apps/api/tests/test_config.py @@ -13,6 +13,7 @@ def production_settings(**changes) -> Settings: "s3_secret_key": "s" * 32, "s3_public_endpoint_url": "https://files.rf4spotter.ru", "cors_origins": ["https://rf4spotter.ru"], + "seed_demo_data": False, } return Settings(**(values | changes)) @@ -30,6 +31,7 @@ def test_production_settings_accept_real_domains_and_secrets() -> None: ("s3_secret_key", "rf4-local-secret"), ("cors_origins", ["http://rf4spotter.ru"]), ("s3_public_endpoint_url", "http://files.rf4spotter.ru"), + ("seed_demo_data", True), ]) def test_production_settings_reject_insecure_values(field: str, value: object) -> None: with pytest.raises(ValidationError): diff --git a/apps/api/tests/test_seed.py b/apps/api/tests/test_seed.py new file mode 100644 index 0000000..90b9c8e --- /dev/null +++ b/apps/api/tests/test_seed.py @@ -0,0 +1,27 @@ +from sqlalchemy import create_engine, func, select +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool + +from app.database import Base +from app.models import CatchReport, Fish, Spot, Waterbody +from app import seed as seed_module + + +def test_seed_repairs_partial_database_and_is_idempotent(monkeypatch) -> None: + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + with Session(engine) as db: + db.add(Fish(slug="pike", name_ru="Щука", trophy_weight_g=10_000)) + db.commit() + + monkeypatch.setattr(seed_module, "SessionLocal", sessions) + monkeypatch.setattr(seed_module.settings, "seed_demo_data", False) + seed_module.seed() + seed_module.seed() + + with Session(engine) as db: + assert db.scalar(select(func.count()).select_from(Fish)) == 2 + assert db.scalar(select(func.count()).select_from(Waterbody)) == 2 + assert db.scalar(select(func.count()).select_from(Spot)) == 2 + assert db.scalar(select(func.count()).select_from(CatchReport)) == 0 diff --git a/apps/web/src/pages/api/report-screenshot.ts b/apps/web/src/pages/api/report-screenshot.ts index 358c6c0..4e076b1 100644 --- a/apps/web/src/pages/api/report-screenshot.ts +++ b/apps/web/src/pages/api/report-screenshot.ts @@ -2,14 +2,17 @@ import type { APIRoute } from "astro"; const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000"; -export const POST: APIRoute = async ({ request, redirect }) => { +export const POST: APIRoute = async ({ request, redirect, cookies }) => { const form = await request.formData(); const reportId = String(form.get("report_id") || ""); const screenshot = form.get("screenshot"); - if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(reportId) || !(screenshot instanceof File) || screenshot.size === 0) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); + const cookieName = `rf4-upload-${reportId}`; + const uploadToken = cookies.get(cookieName)?.value; + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(reportId) || !uploadToken || !(screenshot instanceof File) || screenshot.size === 0) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); try { const upload = new FormData(); upload.set("screenshot", screenshot); - const response = await fetch(`${base}/api/v1/catch-reports/${reportId}/screenshot`, { method: "POST", body: upload }); + const response = await fetch(`${base}/api/v1/catch-reports/${reportId}/screenshot`, { method: "POST", headers:{"X-Upload-Token":uploadToken}, body: upload }); + if (response.ok) cookies.delete(cookieName, {path:"/"}); return redirect(response.ok ? "/report?state=screenshot_sent" : `/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); } catch { return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); } }; diff --git a/apps/web/src/pages/api/report.ts b/apps/web/src/pages/api/report.ts index 8917343..18e8ec9 100644 --- a/apps/web/src/pages/api/report.ts +++ b/apps/web/src/pages/api/report.ts @@ -1,23 +1,32 @@ import type { APIRoute } from "astro"; const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000"; -export const POST: APIRoute = async ({ request, redirect }) => { +export const POST: APIRoute = async ({ request, redirect, cookies }) => { const form = await request.formData(); const text = (name: string) => String(form.get(name) || "").trim() || null; const number = (name: string) => text(name) ? Number(text(name)) : null; const payload = { fish_slug: String(form.get("fish_slug") || ""), waterbody_slug: String(form.get("waterbody_slug") || ""), x: Number(form.get("x")), y: Number(form.get("y")), weight_g: Number(form.get("weight_g")), bait_name: text("bait_name"), fishing_method: text("fishing_method"), retrieve_method: text("retrieve_method"), retrieve_speed: number("retrieve_speed"), player_name: text("player_name"), comment: text("comment"), website: String(form.get("website") || "") }; let createdId: string | null = null; + let uploadToken: string | null = null; try { const response = await fetch(`${base}/api/v1/catch-reports`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) }); if (!response.ok) return redirect("/report?state=create_error", 303); - const created = await response.json(); + const created = await response.json() as { id?: unknown; screenshot_upload_token?: unknown }; + if (typeof created.id !== "string" || typeof created.screenshot_upload_token !== "string") return redirect("/report?state=create_error", 303); createdId = created.id; + uploadToken = created.screenshot_upload_token; const screenshot = form.get("screenshot"); if (screenshot instanceof File && screenshot.size > 0) { const upload = new FormData(); upload.set("screenshot", screenshot); - const uploaded = await fetch(`${base}/api/v1/catch-reports/${created.id}/screenshot`, { method: "POST", body: upload }); - if (!uploaded.ok) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(created.id)}`, 303); + const uploaded = await fetch(`${base}/api/v1/catch-reports/${created.id}/screenshot`, { method: "POST", headers: {"X-Upload-Token": created.screenshot_upload_token}, body: upload }); + if (!uploaded.ok) { + cookies.set(`rf4-upload-${created.id}`, created.screenshot_upload_token, {httpOnly:true, sameSite:"strict", secure:import.meta.env.PROD, path:"/", maxAge:3600}); + return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(created.id)}`, 303); + } } return redirect("/report?state=sent", 303); } - catch { return redirect(createdId ? `/report?state=screenshot_error&report_id=${encodeURIComponent(createdId)}` : "/report?state=create_error", 303); } + catch { + if (createdId && uploadToken) cookies.set(`rf4-upload-${createdId}`, uploadToken, {httpOnly:true, sameSite:"strict", secure:import.meta.env.PROD, path:"/", maxAge:3600}); + return redirect(createdId ? `/report?state=screenshot_error&report_id=${encodeURIComponent(createdId)}` : "/report?state=create_error", 303); + } }; diff --git a/compose.production.yaml b/compose.production.yaml index c36b105..e07cdeb 100644 --- a/compose.production.yaml +++ b/compose.production.yaml @@ -86,6 +86,7 @@ services: OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU} OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records} OFFICIAL_IMPORT_REQUIRED: ${OFFICIAL_IMPORT_REQUIRED:-false} + SEED_DEMO_DATA: "false" IMPORT_INTERVAL_SECONDS: ${IMPORT_INTERVAL_SECONDS:-3600} RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:?Set RATE_LIMIT_SECRET} LOG_LEVEL: ${LOG_LEVEL:-INFO} @@ -122,6 +123,14 @@ services: resources: limits: {cpus: "0.75", memory: 512M} + storage-tool: + image: alpine:3.22 + profiles: [tools] + volumes: + - minio_data:/data + - ${BACKUP_DIRECTORY:-./backups}:/backup + security_opt: [no-new-privileges:true] + networks: edge: backend: diff --git a/deploy/README.md b/deploy/README.md index 2fc70ca..8ea75e8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -32,7 +32,7 @@ curl -fsS https://rf4spotter.ru/health curl -fsS https://rf4spotter.ru/ready ``` -API-контейнер перед стартом применяет Alembic-миграции и запускает seed. До альфы демонстрационные уловы следует убрать отдельной задачей; справочники должны остаться. +API-контейнер перед стартом применяет Alembic-миграции и запускает идемпотентный seed. В production он добавляет только минимальные справочники и точки; демонстрационные уловы жёстко отключены настройкой `SEED_DEMO_DATA=false`. Проверка TLS и маршрутизации: @@ -65,11 +65,26 @@ curl -fsS https://rf4spotter.ru/ready Перед обновлением со сменой схемы обязателен backup PostgreSQL. Не удаляйте volumes и не используйте `down -v`. -## 6. Что ещё блокирует приглашение альфа-пользователей +## 6. Резервное копирование и восстановление -- проверенный backup/restore PostgreSQL и MinIO; -- production-safe seed без демонстрационных уловов; -- одноразовый секрет для повторной загрузки скриншота вместо одного UUID; +Храните копии вне диска приложения. Скрипт создаёт PostgreSQL dump, архив MinIO и контрольные суммы в новом каталоге с UTC-временем: + +```bash +./deploy/backup.sh /srv/rf4-backups +``` + +Восстановление заменяет содержимое PostgreSQL и MinIO данными из выбранной копии, временно останавливая API, web и MinIO. Это намеренно защищённая подтверждением операция: + +```bash +CONFIRM_RESTORE=rf4-spotter ./deploy/restore.sh /srv/rf4-backups/20260906T120000Z +curl -fsS https://rf4spotter.ru/ready +``` + +Перед приглашением пользователей проведите учебное восстановление на отдельном сервере или с отдельным Compose project name. Одной только успешной архивации недостаточно. После проверки настройте ежедневный запуск `backup.sh`, шифрование/выгрузку копий во внешнее хранилище и уведомление при ошибке; храните минимум 7 ежедневных и 4 еженедельных копии. + +## 7. Что ещё блокирует приглашение альфа-пользователей + +- учебное восстановление PostgreSQL и MinIO на изолированном контуре; - проверка полного bootstrap на пустых volumes; - политика хранения и удаления пользовательских данных; - базовый мониторинг `/ready`, диска и срока TLS-сертификата. diff --git a/deploy/backup.sh b/deploy/backup.sh new file mode 100755 index 0000000..ec8f739 --- /dev/null +++ b/deploy/backup.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 ABSOLUTE_BACKUP_DIRECTORY" >&2 + exit 2 +fi + +case "$1" in + /*) ;; + *) echo "Backup directory must be an absolute path" >&2; exit 2 ;; +esac + +backup_root=${1%/} +stamp=$(date -u +%Y%m%dT%H%M%SZ) +target="$backup_root/$stamp" +mkdir -p "$target" +chmod 700 "$target" +export BACKUP_DIRECTORY="$backup_root" + +compose="docker compose --env-file .env.production -f compose.production.yaml" +$compose exec -T db sh -c 'pg_dump --format=custom --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$target/postgres.dump" +$compose run --rm storage-tool tar -czf "/backup/$stamp/minio.tar.gz" -C /data . +sha256sum "$target/postgres.dump" "$target/minio.tar.gz" > "$target/SHA256SUMS" +chmod 600 "$target/postgres.dump" "$target/minio.tar.gz" "$target/SHA256SUMS" +echo "Backup created: $target" diff --git a/deploy/restore.sh b/deploy/restore.sh new file mode 100755 index 0000000..00048a5 --- /dev/null +++ b/deploy/restore.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +if [ "$#" -ne 1 ] || [ "${CONFIRM_RESTORE:-}" != "rf4-spotter" ]; then + echo "Usage: CONFIRM_RESTORE=rf4-spotter $0 ABSOLUTE_BACKUP_DIRECTORY/TIMESTAMP" >&2 + exit 2 +fi + +case "$1" in + /*) ;; + *) echo "Backup directory must be an absolute path" >&2; exit 2 ;; +esac + +source_dir=${1%/} +test -f "$source_dir/postgres.dump" +test -f "$source_dir/minio.tar.gz" +(cd "$source_dir" && sha256sum -c SHA256SUMS) +backup_root=$(dirname "$source_dir") +stamp=$(basename "$source_dir") +export BACKUP_DIRECTORY="$backup_root" + +compose="docker compose --env-file .env.production -f compose.production.yaml" +$compose stop api web minio +$compose exec -T db sh -c 'pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$source_dir/postgres.dump" +$compose run --rm storage-tool sh -c 'find /data -mindepth 1 -delete && tar -xzf "/backup/'"$stamp"'/minio.tar.gz" -C /data' +$compose up -d minio api web proxy +echo "Restore completed from: $source_dir" diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e6b94fc..f645ade 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -60,13 +60,13 @@ - [x] Добавить структурированные JSON-логи без пользовательских секретов и персональных технических данных (whitelist полей, redaction, request ID; Uvicorn access-log отключён). - [x] Добавить Gitea Actions CI: backend tests, Astro check/build, E2E и применение всех миграций на чистой PostgreSQL; сохранять логи Compose и Playwright-артефакты при падении (`.gitea/workflows/ci.yml`). - [ ] Добавить отдельный тест полного bootstrap: пустые volumes → миграции → seed → readiness → основной E2E. -- [ ] Сделать seed устойчивым к частично заполненной БД и покрыть повторный/частичный запуск тестом; текущая реализация прекращает работу при наличии любой рыбы. +- [x] Сделать seed устойчивым к частично заполненной БД: справочники досеиваются независимо, демо-уловы идемпотентны и принудительно отключены в production; повторный/частичный запуск покрыт конфигурационными и интеграционными проверками. - [ ] Проверить списочные API по требованию раздела 12: пагинация, предсказуемая сортировка и валидация фильтров для справочников, импортов, модерации и внешнего staging. - [ ] Проверить необходимые индексы PostgreSQL и планы запросов для activity, модерации, дедупликации и очистки rate limit; зафиксировать допустимый бюджет запросов пилота. - [ ] Провести security-проверку admin-аутентификации, CORS, security headers, загрузок и управления секретами; вынести допустимые origins в конфигурацию и исключить демонстрационные секреты в production-режиме. -- [ ] Проверить авторизацию повторной загрузки скриншота: один UUID pending-заявки не должен быть достаточным полномочием для изменения чужой записи. +- [x] Проверить авторизацию повторной загрузки скриншота: используется отдельный одноразовый случайный токен, в БД хранится только SHA-256, UUID заявки недостаточно. - [ ] Определить сроки хранения ников, исходных payload, staging-наблюдений, moderation events и submission attempts; добавить документированную очистку/анонимизацию. -- [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO; проверить восстановление на отдельных временных volumes. +- [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO: скрипты, контрольные суммы и runbook готовы; остаётся учебное восстановление на отдельных временных volumes. - [ ] Проверить доступность интерфейса: клавиатура, focus states, контраст, подписи полей и семантика таблиц/карточек. - [ ] Провести Lighthouse-проверку основных страниц и устранить критические проблемы производительности. - [x] Провести UI/UX-аудит desktop/mobile и сформировать приоритетный план (`docs/UI_UX_AUDIT.md`).