feat: harden production data and backups
CI / backend-and-migrations (push) Canceled after 0s
CI / astro-build (push) Canceled after 0s
CI / compose-e2e (push) Canceled after 0s

This commit is contained in:
ik
2026-09-06 14:08:07 +07:00
parent a4bd395856
commit 870d9cc7f9
17 changed files with 215 additions and 36 deletions
+1 -1
View File
@@ -14,7 +14,7 @@ RF4 Spotter — неофициальный сервис свежих точек
Подробный план и актуальные чекбоксы находятся в [`docs/ROADMAP.md`](docs/ROADMAP.md). Результаты проверки интерфейса и пять приоритетных UX-пакетов описаны в [`docs/UI_UX_AUDIT.md`](docs/UI_UX_AUDIT.md). Подробный план и актуальные чекбоксы находятся в [`docs/ROADMAP.md`](docs/ROADMAP.md). Результаты проверки интерфейса и пять приоритетных UX-пакетов описаны в [`docs/UI_UX_AUDIT.md`](docs/UI_UX_AUDIT.md).
Production-контур для домена `rf4spotter.ru`, TLS, секреты и команды первого запуска описаны в [`deploy/README.md`](deploy/README.md). Он использует отдельный `compose.production.yaml`; локальный `compose.yaml` остаётся средой разработки. Production-контур для домена `rf4spotter.ru`, TLS, секреты, backup/restore и команды первого запуска описаны в [`deploy/README.md`](deploy/README.md). Он использует отдельный `compose.production.yaml`; локальный `compose.yaml` остаётся средой разработки. Production seed добавляет только справочники — демонстрационные уловы отключены.
Gitea Actions workflow `.gitea/workflows/ci.yml` на каждый push и pull request проверяет Python, миграции на чистой PostgreSQL, Astro build и полный Compose/Playwright-сценарий. При падении E2E сохраняются логи контейнеров и Playwright-артефакты. Gitea Actions workflow `.gitea/workflows/ci.yml` на каждый push и pull request проверяет Python, миграции на чистой PostgreSQL, Astro build и полный Compose/Playwright-сценарий. При падении E2E сохраняются логи контейнеров и Playwright-артефакты.
@@ -0,0 +1,16 @@
"""Protect pending screenshot uploads with a one-time token."""
from alembic import op
import sqlalchemy as sa
revision = "0010"
down_revision = "0009"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("catch_report", sa.Column("screenshot_upload_token_hash", sa.String(64)))
def downgrade() -> None:
op.drop_column("catch_report", "screenshot_upload_token_hash")
+3
View File
@@ -16,6 +16,7 @@ class Settings(BaseSettings):
official_records_region: str = "RU" official_records_region: str = "RU"
official_records_category: str = "records" official_records_category: str = "records"
official_import_required: bool = False official_import_required: bool = False
seed_demo_data: bool = True
import_interval_seconds: int = Field(default=3600, ge=3600) import_interval_seconds: int = Field(default=3600, ge=3600)
rate_limit_secret: str = "change-rate-limit-secret" rate_limit_secret: str = "change-rate-limit-secret"
log_level: str = "INFO" log_level: str = "INFO"
@@ -39,6 +40,8 @@ class Settings(BaseSettings):
raise ValueError("production CORS_ORIGINS must contain only HTTPS origins") raise ValueError("production CORS_ORIGINS must contain only HTTPS origins")
if not self.s3_public_endpoint_url.startswith("https://"): if not self.s3_public_endpoint_url.startswith("https://"):
raise ValueError("production S3_PUBLIC_ENDPOINT_URL must use HTTPS") raise ValueError("production S3_PUBLIC_ENDPOINT_URL must use HTTPS")
if self.seed_demo_data:
raise ValueError("SEED_DEMO_DATA must be false in production")
return self return self
+15 -6
View File
@@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone
import hashlib import hashlib
import hmac import hmac
import logging import logging
import secrets
import time as time_module import time as time_module
from typing import Annotated, Literal from typing import Annotated, Literal
from uuid import UUID from uuid import UUID
@@ -25,7 +26,7 @@ from .importer import ImportSourceError, import_records, normalize
from .logging_config import configure_logging from .logging_config import configure_logging
from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
from .readiness import readiness_report from .readiness import readiness_report
from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot
@@ -273,8 +274,8 @@ def admin_reject_external_observation(
raise HTTPException(status_code=409, detail=str(exc)) from exc raise HTTPException(status_code=409, detail=str(exc)) from exc
@app.post("/api/v1/catch-reports", response_model=CatchReportCreated, status_code=201) @app.post("/api/v1/catch-reports", response_model=CatchReportAccepted, status_code=201)
def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated: def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportAccepted:
if payload.website: if payload.website:
raise HTTPException(status_code=400, detail="invalid submission") raise HTTPException(status_code=400, detail="invalid submission")
_check_rate_limit(request.client.host if request.client else "unknown", db) _check_rate_limit(request.client.host if request.client else "unknown", db)
@@ -293,17 +294,24 @@ def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) ->
if bait is None: if bait is None:
bait = Bait(name=payload.bait_name.strip(), normalized_name=key, kind=BaitKind.unknown) bait = Bait(name=payload.bait_name.strip(), normalized_name=key, kind=BaitKind.unknown)
db.add(bait) db.add(bait)
report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None) upload_token = secrets.token_urlsafe(32)
report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None, screenshot_upload_token_hash=hashlib.sha256(upload_token.encode()).hexdigest())
db.add(report) db.add(report)
db.commit() db.commit()
return CatchReportCreated(id=report.id, moderation_status=report.moderation_status.value) return CatchReportAccepted(id=report.id, moderation_status=report.moderation_status.value, screenshot_upload_token=upload_token)
@app.post("/api/v1/catch-reports/{report_id}/screenshot", status_code=204, response_class=Response) @app.post("/api/v1/catch-reports/{report_id}/screenshot", status_code=204, response_class=Response)
def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) -> Response: def add_screenshot(
report_id: UUID, db: Db, screenshot: UploadFile = File(),
upload_token: Annotated[str | None, Header(alias="X-Upload-Token")] = None,
) -> Response:
report = db.get(CatchReport, report_id) report = db.get(CatchReport, report_id)
if report is None or report.source_type != SourceType.user or report.moderation_status != ModerationStatus.pending: if report is None or report.source_type != SourceType.user or report.moderation_status != ModerationStatus.pending:
raise HTTPException(status_code=404, detail="pending catch report not found") raise HTTPException(status_code=404, detail="pending catch report not found")
supplied_hash = hashlib.sha256((upload_token or "").encode()).hexdigest()
if not report.screenshot_upload_token_hash or not hmac.compare_digest(report.screenshot_upload_token_hash, supplied_hash):
raise HTTPException(status_code=401, detail="invalid screenshot upload token")
if report.screenshot_key: if report.screenshot_key:
raise HTTPException(status_code=409, detail="screenshot already uploaded") raise HTTPException(status_code=409, detail="screenshot already uploaded")
raw = screenshot.file.read(settings.screenshot_max_bytes + 1) raw = screenshot.file.read(settings.screenshot_max_bytes + 1)
@@ -311,6 +319,7 @@ def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) ->
report.screenshot_key = upload_screenshot(raw, filename=screenshot.filename, content_type=screenshot.content_type) report.screenshot_key = upload_screenshot(raw, filename=screenshot.filename, content_type=screenshot.content_type)
except ScreenshotError as exc: except ScreenshotError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc raise HTTPException(status_code=422, detail=str(exc)) from exc
report.screenshot_upload_token_hash = None
db.commit() db.commit()
return Response(status_code=204) return Response(status_code=204)
+1
View File
@@ -92,6 +92,7 @@ class CatchReport(Base):
source_confidence: Mapped[int] source_confidence: Mapped[int]
moderation_status: Mapped[ModerationStatus] = mapped_column(Enum(ModerationStatus)) moderation_status: Mapped[ModerationStatus] = mapped_column(Enum(ModerationStatus))
screenshot_key: Mapped[str | None] = mapped_column(Text) screenshot_key: Mapped[str | None] = mapped_column(Text)
screenshot_upload_token_hash: Mapped[str | None] = mapped_column(String(64))
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
raw_payload: Mapped[dict | None] = mapped_column(JSON) raw_payload: Mapped[dict | None] = mapped_column(JSON)
fish: Mapped[Fish] = relationship() fish: Mapped[Fish] = relationship()
+4
View File
@@ -137,6 +137,10 @@ class CatchReportCreated(BaseModel):
moderation_status: str moderation_status: str
class CatchReportAccepted(CatchReportCreated):
screenshot_upload_token: str
class AdminCatchReportOut(BaseModel): class AdminCatchReportOut(BaseModel):
id: UUID id: UUID
fish: str fish: str
+35 -10
View File
@@ -5,6 +5,7 @@ from uuid import UUID
from sqlalchemy import select from sqlalchemy import select
from .config import settings
from .database import SessionLocal from .database import SessionLocal
from .models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody from .models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody
@@ -23,19 +24,38 @@ IDS = {
def seed() -> None: def seed() -> None:
with SessionLocal.begin() as db: with SessionLocal.begin() as db:
if db.scalar(select(Fish.id).limit(1)) is not None: def entity(model, key: str, value: str, **values):
item = db.scalar(select(model).where(getattr(model, key) == value))
if item is None:
item = model(**values)
db.add(item)
db.flush()
return item
vyunok = entity(Waterbody, "slug", "vyunok", id=IDS["vyunok"], slug="vyunok", name_ru="Вьюнок", unlock_level=1)
kuori = entity(Waterbody, "slug", "kuori", id=IDS["kuori"], slug="kuori", name_ru="Куори", unlock_level=16)
pike = entity(Fish, "slug", "pike", id=IDS["pike"], slug="pike", name_ru="Щука", trophy_weight_g=10_000)
trout = entity(Fish, "slug", "lake-trout", id=IDS["trout"], slug="lake-trout", name_ru="Озёрная форель", trophy_weight_g=10_000)
spiker = entity(Bait, "normalized_name", "spiker #2 01-015", id=IDS["spiker"], name="Spiker #2 01-015", normalized_name="spiker #2 01-015", kind=BaitKind.lure)
shad = entity(Bait, "normalized_name", "salmon t1 shad 12 005", id=IDS["shad"], name="Salmon T1 Shad 12 005", normalized_name="salmon t1 shad 12 005", kind=BaitKind.lure)
def spot(waterbody: Waterbody, identity: str, x: int, y: int, description: str) -> Spot:
item = db.scalar(select(Spot).where(Spot.waterbody_id == waterbody.id, Spot.x == x, Spot.y == y))
if item is None:
item = Spot(id=IDS[identity], waterbody=waterbody, x=x, y=y, description=description)
db.add(item)
db.flush()
return item
spot1 = spot(vyunok, "spot1", 110, 103, "Кромка травы у северного берега")
spot2 = spot(kuori, "spot2", 85, 92, "Свальчик в глубину")
if not settings.seed_demo_data:
return return
vyunok = Waterbody(id=IDS["vyunok"], slug="vyunok", name_ru="Вьюнок", unlock_level=1)
kuori = Waterbody(id=IDS["kuori"], slug="kuori", name_ru="Куори", unlock_level=16)
pike = Fish(id=IDS["pike"], slug="pike", name_ru="Щука", trophy_weight_g=10_000)
trout = Fish(id=IDS["trout"], slug="lake-trout", name_ru="Озёрная форель", trophy_weight_g=10_000)
spiker = Bait(id=IDS["spiker"], name="Spiker #2 01-015", normalized_name="spiker #2 01-015", kind=BaitKind.lure)
shad = Bait(id=IDS["shad"], name="Salmon T1 Shad 12 005", normalized_name="salmon t1 shad 12 005", kind=BaitKind.lure)
spot1 = Spot(id=IDS["spot1"], waterbody=vyunok, x=110, y=103, description="Кромка травы у северного берега")
spot2 = Spot(id=IDS["spot2"], waterbody=kuori, x=85, y=92, description="Свальчик в глубину")
db.add_all([vyunok, kuori, pike, trout, spiker, shad, spot1, spot2])
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
for index in range(12): for index in range(12):
external_id = f"seed:pike:{index}"
if db.scalar(select(CatchReport.id).where(CatchReport.source_external_id == external_id)):
continue
db.add(CatchReport( db.add(CatchReport(
fish=pike, spot=spot1, waterbody=vyunok, bait=spiker, fish=pike, spot=spot1, waterbody=vyunok, bait=spiker,
weight_g=2_600 + index * 480, fishing_method="spinning", weight_g=2_600 + index * 480, fishing_method="spinning",
@@ -43,9 +63,13 @@ def seed() -> None:
caught_at=now - timedelta(minutes=25 + index * 47), caught_at=now - timedelta(minutes=25 + index * 47),
reported_at=now - timedelta(minutes=20 + index * 47), reported_at=now - timedelta(minutes=20 + index * 47),
player_name=f"DemoPlayer{index % 7 + 1}", source_type=SourceType.manual_import, player_name=f"DemoPlayer{index % 7 + 1}", source_type=SourceType.manual_import,
source_external_id=external_id,
source_confidence=80 + index % 3 * 5, moderation_status=ModerationStatus.approved, source_confidence=80 + index % 3 * 5, moderation_status=ModerationStatus.approved,
)) ))
for index in range(5): for index in range(5):
external_id = f"seed:trout:{index}"
if db.scalar(select(CatchReport.id).where(CatchReport.source_external_id == external_id)):
continue
db.add(CatchReport( db.add(CatchReport(
fish=trout, spot=spot2, waterbody=kuori, bait=shad, fish=trout, spot=spot2, waterbody=kuori, bait=shad,
weight_g=4_200 + index * 900, fishing_method="spinning", weight_g=4_200 + index * 900, fishing_method="spinning",
@@ -53,6 +77,7 @@ def seed() -> None:
caught_at=now - timedelta(hours=2 + index * 4), caught_at=now - timedelta(hours=2 + index * 4),
reported_at=now - timedelta(hours=2 + index * 4), reported_at=now - timedelta(hours=2 + index * 4),
player_name=f"DemoAngler{index + 1}", source_type=SourceType.manual_import, player_name=f"DemoAngler{index + 1}", source_type=SourceType.manual_import,
source_external_id=external_id,
source_confidence=85, moderation_status=ModerationStatus.approved, source_confidence=85, moderation_status=ModerationStatus.approved,
)) ))
+6 -3
View File
@@ -187,10 +187,13 @@ def test_admin_can_start_and_list_official_import(monkeypatch) -> None:
def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None: def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:
created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 91, "y": 92, "weight_g": 4200}).json() created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 91, "y": 92, "weight_g": 4200}).json()
monkeypatch.setattr("app.main.upload_screenshot", lambda raw, **metadata: "reports/test.jpg" if raw == b"image-bytes" and metadata == {"filename": "catch.jpg", "content_type": "image/jpeg"} else "unexpected") monkeypatch.setattr("app.main.upload_screenshot", lambda raw, **metadata: "reports/test.jpg" if raw == b"image-bytes" and metadata == {"filename": "catch.jpg", "content_type": "image/jpeg"} else "unexpected")
response = client.post(f"/api/v1/catch-reports/{created['id']}/screenshot", files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}) upload_url = f"/api/v1/catch-reports/{created['id']}/screenshot"
assert client.post(upload_url, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}).status_code == 401
assert client.post(upload_url, headers={"X-Upload-Token": "wrong"}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}).status_code == 401
response = client.post(upload_url, headers={"X-Upload-Token": created["screenshot_upload_token"]}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")})
assert response.status_code == 204 assert response.status_code == 204
duplicate = client.post(f"/api/v1/catch-reports/{created['id']}/screenshot", files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}) reused = client.post(upload_url, headers={"X-Upload-Token": created["screenshot_upload_token"]}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")})
assert duplicate.status_code == 409 assert reused.status_code == 401
def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None: def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None:
+2
View File
@@ -13,6 +13,7 @@ def production_settings(**changes) -> Settings:
"s3_secret_key": "s" * 32, "s3_secret_key": "s" * 32,
"s3_public_endpoint_url": "https://files.rf4spotter.ru", "s3_public_endpoint_url": "https://files.rf4spotter.ru",
"cors_origins": ["https://rf4spotter.ru"], "cors_origins": ["https://rf4spotter.ru"],
"seed_demo_data": False,
} }
return Settings(**(values | changes)) return Settings(**(values | changes))
@@ -30,6 +31,7 @@ def test_production_settings_accept_real_domains_and_secrets() -> None:
("s3_secret_key", "rf4-local-secret"), ("s3_secret_key", "rf4-local-secret"),
("cors_origins", ["http://rf4spotter.ru"]), ("cors_origins", ["http://rf4spotter.ru"]),
("s3_public_endpoint_url", "http://files.rf4spotter.ru"), ("s3_public_endpoint_url", "http://files.rf4spotter.ru"),
("seed_demo_data", True),
]) ])
def test_production_settings_reject_insecure_values(field: str, value: object) -> None: def test_production_settings_reject_insecure_values(field: str, value: object) -> None:
with pytest.raises(ValidationError): with pytest.raises(ValidationError):
+27
View File
@@ -0,0 +1,27 @@
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
from app.database import Base
from app.models import CatchReport, Fish, Spot, Waterbody
from app import seed as seed_module
def test_seed_repairs_partial_database_and_is_idempotent(monkeypatch) -> None:
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
Base.metadata.create_all(engine)
sessions = sessionmaker(bind=engine, expire_on_commit=False)
with Session(engine) as db:
db.add(Fish(slug="pike", name_ru="Щука", trophy_weight_g=10_000))
db.commit()
monkeypatch.setattr(seed_module, "SessionLocal", sessions)
monkeypatch.setattr(seed_module.settings, "seed_demo_data", False)
seed_module.seed()
seed_module.seed()
with Session(engine) as db:
assert db.scalar(select(func.count()).select_from(Fish)) == 2
assert db.scalar(select(func.count()).select_from(Waterbody)) == 2
assert db.scalar(select(func.count()).select_from(Spot)) == 2
assert db.scalar(select(func.count()).select_from(CatchReport)) == 0
+6 -3
View File
@@ -2,14 +2,17 @@ import type { APIRoute } from "astro";
const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000"; const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000";
export const POST: APIRoute = async ({ request, redirect }) => { export const POST: APIRoute = async ({ request, redirect, cookies }) => {
const form = await request.formData(); const form = await request.formData();
const reportId = String(form.get("report_id") || ""); const reportId = String(form.get("report_id") || "");
const screenshot = form.get("screenshot"); const screenshot = form.get("screenshot");
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(reportId) || !(screenshot instanceof File) || screenshot.size === 0) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); const cookieName = `rf4-upload-${reportId}`;
const uploadToken = cookies.get(cookieName)?.value;
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(reportId) || !uploadToken || !(screenshot instanceof File) || screenshot.size === 0) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303);
try { try {
const upload = new FormData(); upload.set("screenshot", screenshot); const upload = new FormData(); upload.set("screenshot", screenshot);
const response = await fetch(`${base}/api/v1/catch-reports/${reportId}/screenshot`, { method: "POST", body: upload }); const response = await fetch(`${base}/api/v1/catch-reports/${reportId}/screenshot`, { method: "POST", headers:{"X-Upload-Token":uploadToken}, body: upload });
if (response.ok) cookies.delete(cookieName, {path:"/"});
return redirect(response.ok ? "/report?state=screenshot_sent" : `/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); return redirect(response.ok ? "/report?state=screenshot_sent" : `/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303);
} catch { return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); } } catch { return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); }
}; };
+14 -5
View File
@@ -1,23 +1,32 @@
import type { APIRoute } from "astro"; import type { APIRoute } from "astro";
const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000"; const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000";
export const POST: APIRoute = async ({ request, redirect }) => { export const POST: APIRoute = async ({ request, redirect, cookies }) => {
const form = await request.formData(); const form = await request.formData();
const text = (name: string) => String(form.get(name) || "").trim() || null; const text = (name: string) => String(form.get(name) || "").trim() || null;
const number = (name: string) => text(name) ? Number(text(name)) : null; const number = (name: string) => text(name) ? Number(text(name)) : null;
const payload = { fish_slug: String(form.get("fish_slug") || ""), waterbody_slug: String(form.get("waterbody_slug") || ""), x: Number(form.get("x")), y: Number(form.get("y")), weight_g: Number(form.get("weight_g")), bait_name: text("bait_name"), fishing_method: text("fishing_method"), retrieve_method: text("retrieve_method"), retrieve_speed: number("retrieve_speed"), player_name: text("player_name"), comment: text("comment"), website: String(form.get("website") || "") }; const payload = { fish_slug: String(form.get("fish_slug") || ""), waterbody_slug: String(form.get("waterbody_slug") || ""), x: Number(form.get("x")), y: Number(form.get("y")), weight_g: Number(form.get("weight_g")), bait_name: text("bait_name"), fishing_method: text("fishing_method"), retrieve_method: text("retrieve_method"), retrieve_speed: number("retrieve_speed"), player_name: text("player_name"), comment: text("comment"), website: String(form.get("website") || "") };
let createdId: string | null = null; let createdId: string | null = null;
let uploadToken: string | null = null;
try { try {
const response = await fetch(`${base}/api/v1/catch-reports`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) }); const response = await fetch(`${base}/api/v1/catch-reports`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) });
if (!response.ok) return redirect("/report?state=create_error", 303); if (!response.ok) return redirect("/report?state=create_error", 303);
const created = await response.json(); const created = await response.json() as { id?: unknown; screenshot_upload_token?: unknown };
if (typeof created.id !== "string" || typeof created.screenshot_upload_token !== "string") return redirect("/report?state=create_error", 303);
createdId = created.id; createdId = created.id;
uploadToken = created.screenshot_upload_token;
const screenshot = form.get("screenshot"); const screenshot = form.get("screenshot");
if (screenshot instanceof File && screenshot.size > 0) { if (screenshot instanceof File && screenshot.size > 0) {
const upload = new FormData(); upload.set("screenshot", screenshot); const upload = new FormData(); upload.set("screenshot", screenshot);
const uploaded = await fetch(`${base}/api/v1/catch-reports/${created.id}/screenshot`, { method: "POST", body: upload }); const uploaded = await fetch(`${base}/api/v1/catch-reports/${created.id}/screenshot`, { method: "POST", headers: {"X-Upload-Token": created.screenshot_upload_token}, body: upload });
if (!uploaded.ok) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(created.id)}`, 303); if (!uploaded.ok) {
cookies.set(`rf4-upload-${created.id}`, created.screenshot_upload_token, {httpOnly:true, sameSite:"strict", secure:import.meta.env.PROD, path:"/", maxAge:3600});
return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(created.id)}`, 303);
}
} }
return redirect("/report?state=sent", 303); return redirect("/report?state=sent", 303);
} }
catch { return redirect(createdId ? `/report?state=screenshot_error&report_id=${encodeURIComponent(createdId)}` : "/report?state=create_error", 303); } catch {
if (createdId && uploadToken) cookies.set(`rf4-upload-${createdId}`, uploadToken, {httpOnly:true, sameSite:"strict", secure:import.meta.env.PROD, path:"/", maxAge:3600});
return redirect(createdId ? `/report?state=screenshot_error&report_id=${encodeURIComponent(createdId)}` : "/report?state=create_error", 303);
}
}; };
+9
View File
@@ -86,6 +86,7 @@ services:
OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU} OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU}
OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records} OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records}
OFFICIAL_IMPORT_REQUIRED: ${OFFICIAL_IMPORT_REQUIRED:-false} OFFICIAL_IMPORT_REQUIRED: ${OFFICIAL_IMPORT_REQUIRED:-false}
SEED_DEMO_DATA: "false"
IMPORT_INTERVAL_SECONDS: ${IMPORT_INTERVAL_SECONDS:-3600} IMPORT_INTERVAL_SECONDS: ${IMPORT_INTERVAL_SECONDS:-3600}
RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:?Set RATE_LIMIT_SECRET} RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:?Set RATE_LIMIT_SECRET}
LOG_LEVEL: ${LOG_LEVEL:-INFO} LOG_LEVEL: ${LOG_LEVEL:-INFO}
@@ -122,6 +123,14 @@ services:
resources: resources:
limits: {cpus: "0.75", memory: 512M} limits: {cpus: "0.75", memory: 512M}
storage-tool:
image: alpine:3.22
profiles: [tools]
volumes:
- minio_data:/data
- ${BACKUP_DIRECTORY:-./backups}:/backup
security_opt: [no-new-privileges:true]
networks: networks:
edge: edge:
backend: backend:
+20 -5
View File
@@ -32,7 +32,7 @@ curl -fsS https://rf4spotter.ru/health
curl -fsS https://rf4spotter.ru/ready curl -fsS https://rf4spotter.ru/ready
``` ```
API-контейнер перед стартом применяет Alembic-миграции и запускает seed. До альфы демонстрационные уловы следует убрать отдельной задачей; справочники должны остаться. API-контейнер перед стартом применяет Alembic-миграции и запускает идемпотентный seed. В production он добавляет только минимальные справочники и точки; демонстрационные уловы жёстко отключены настройкой `SEED_DEMO_DATA=false`.
Проверка TLS и маршрутизации: Проверка TLS и маршрутизации:
@@ -65,11 +65,26 @@ curl -fsS https://rf4spotter.ru/ready
Перед обновлением со сменой схемы обязателен backup PostgreSQL. Не удаляйте volumes и не используйте `down -v`. Перед обновлением со сменой схемы обязателен backup PostgreSQL. Не удаляйте volumes и не используйте `down -v`.
## 6. Что ещё блокирует приглашение альфа-пользователей ## 6. Резервное копирование и восстановление
- проверенный backup/restore PostgreSQL и MinIO; Храните копии вне диска приложения. Скрипт создаёт PostgreSQL dump, архив MinIO и контрольные суммы в новом каталоге с UTC-временем:
- production-safe seed без демонстрационных уловов;
- одноразовый секрет для повторной загрузки скриншота вместо одного UUID; ```bash
./deploy/backup.sh /srv/rf4-backups
```
Восстановление заменяет содержимое PostgreSQL и MinIO данными из выбранной копии, временно останавливая API, web и MinIO. Это намеренно защищённая подтверждением операция:
```bash
CONFIRM_RESTORE=rf4-spotter ./deploy/restore.sh /srv/rf4-backups/20260906T120000Z
curl -fsS https://rf4spotter.ru/ready
```
Перед приглашением пользователей проведите учебное восстановление на отдельном сервере или с отдельным Compose project name. Одной только успешной архивации недостаточно. После проверки настройте ежедневный запуск `backup.sh`, шифрование/выгрузку копий во внешнее хранилище и уведомление при ошибке; храните минимум 7 ежедневных и 4 еженедельных копии.
## 7. Что ещё блокирует приглашение альфа-пользователей
- учебное восстановление PostgreSQL и MinIO на изолированном контуре;
- проверка полного bootstrap на пустых volumes; - проверка полного bootstrap на пустых volumes;
- политика хранения и удаления пользовательских данных; - политика хранения и удаления пользовательских данных;
- базовый мониторинг `/ready`, диска и срока TLS-сертификата. - базовый мониторинг `/ready`, диска и срока TLS-сертификата.
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ]; then
echo "Usage: $0 ABSOLUTE_BACKUP_DIRECTORY" >&2
exit 2
fi
case "$1" in
/*) ;;
*) echo "Backup directory must be an absolute path" >&2; exit 2 ;;
esac
backup_root=${1%/}
stamp=$(date -u +%Y%m%dT%H%M%SZ)
target="$backup_root/$stamp"
mkdir -p "$target"
chmod 700 "$target"
export BACKUP_DIRECTORY="$backup_root"
compose="docker compose --env-file .env.production -f compose.production.yaml"
$compose exec -T db sh -c 'pg_dump --format=custom --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$target/postgres.dump"
$compose run --rm storage-tool tar -czf "/backup/$stamp/minio.tar.gz" -C /data .
sha256sum "$target/postgres.dump" "$target/minio.tar.gz" > "$target/SHA256SUMS"
chmod 600 "$target/postgres.dump" "$target/minio.tar.gz" "$target/SHA256SUMS"
echo "Backup created: $target"
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ] || [ "${CONFIRM_RESTORE:-}" != "rf4-spotter" ]; then
echo "Usage: CONFIRM_RESTORE=rf4-spotter $0 ABSOLUTE_BACKUP_DIRECTORY/TIMESTAMP" >&2
exit 2
fi
case "$1" in
/*) ;;
*) echo "Backup directory must be an absolute path" >&2; exit 2 ;;
esac
source_dir=${1%/}
test -f "$source_dir/postgres.dump"
test -f "$source_dir/minio.tar.gz"
(cd "$source_dir" && sha256sum -c SHA256SUMS)
backup_root=$(dirname "$source_dir")
stamp=$(basename "$source_dir")
export BACKUP_DIRECTORY="$backup_root"
compose="docker compose --env-file .env.production -f compose.production.yaml"
$compose stop api web minio
$compose exec -T db sh -c 'pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$source_dir/postgres.dump"
$compose run --rm storage-tool sh -c 'find /data -mindepth 1 -delete && tar -xzf "/backup/'"$stamp"'/minio.tar.gz" -C /data'
$compose up -d minio api web proxy
echo "Restore completed from: $source_dir"
+3 -3
View File
@@ -60,13 +60,13 @@
- [x] Добавить структурированные JSON-логи без пользовательских секретов и персональных технических данных (whitelist полей, redaction, request ID; Uvicorn access-log отключён). - [x] Добавить структурированные JSON-логи без пользовательских секретов и персональных технических данных (whitelist полей, redaction, request ID; Uvicorn access-log отключён).
- [x] Добавить Gitea Actions CI: backend tests, Astro check/build, E2E и применение всех миграций на чистой PostgreSQL; сохранять логи Compose и Playwright-артефакты при падении (`.gitea/workflows/ci.yml`). - [x] Добавить Gitea Actions CI: backend tests, Astro check/build, E2E и применение всех миграций на чистой PostgreSQL; сохранять логи Compose и Playwright-артефакты при падении (`.gitea/workflows/ci.yml`).
- [ ] Добавить отдельный тест полного bootstrap: пустые volumes → миграции → seed → readiness → основной E2E. - [ ] Добавить отдельный тест полного bootstrap: пустые volumes → миграции → seed → readiness → основной E2E.
- [ ] Сделать seed устойчивым к частично заполненной БД и покрыть повторный/частичный запуск тестом; текущая реализация прекращает работу при наличии любой рыбы. - [x] Сделать seed устойчивым к частично заполненной БД: справочники досеиваются независимо, демо-уловы идемпотентны и принудительно отключены в production; повторный/частичный запуск покрыт конфигурационными и интеграционными проверками.
- [ ] Проверить списочные API по требованию раздела 12: пагинация, предсказуемая сортировка и валидация фильтров для справочников, импортов, модерации и внешнего staging. - [ ] Проверить списочные API по требованию раздела 12: пагинация, предсказуемая сортировка и валидация фильтров для справочников, импортов, модерации и внешнего staging.
- [ ] Проверить необходимые индексы PostgreSQL и планы запросов для activity, модерации, дедупликации и очистки rate limit; зафиксировать допустимый бюджет запросов пилота. - [ ] Проверить необходимые индексы PostgreSQL и планы запросов для activity, модерации, дедупликации и очистки rate limit; зафиксировать допустимый бюджет запросов пилота.
- [ ] Провести security-проверку admin-аутентификации, CORS, security headers, загрузок и управления секретами; вынести допустимые origins в конфигурацию и исключить демонстрационные секреты в production-режиме. - [ ] Провести security-проверку admin-аутентификации, CORS, security headers, загрузок и управления секретами; вынести допустимые origins в конфигурацию и исключить демонстрационные секреты в production-режиме.
- [ ] Проверить авторизацию повторной загрузки скриншота: один UUID pending-заявки не должен быть достаточным полномочием для изменения чужой записи. - [x] Проверить авторизацию повторной загрузки скриншота: используется отдельный одноразовый случайный токен, в БД хранится только SHA-256, UUID заявки недостаточно.
- [ ] Определить сроки хранения ников, исходных payload, staging-наблюдений, moderation events и submission attempts; добавить документированную очистку/анонимизацию. - [ ] Определить сроки хранения ников, исходных payload, staging-наблюдений, moderation events и submission attempts; добавить документированную очистку/анонимизацию.
- [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO; проверить восстановление на отдельных временных volumes. - [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO: скрипты, контрольные суммы и runbook готовы; остаётся учебное восстановление на отдельных временных volumes.
- [ ] Проверить доступность интерфейса: клавиатура, focus states, контраст, подписи полей и семантика таблиц/карточек. - [ ] Проверить доступность интерфейса: клавиатура, focus states, контраст, подписи полей и семантика таблиц/карточек.
- [ ] Провести Lighthouse-проверку основных страниц и устранить критические проблемы производительности. - [ ] Провести Lighthouse-проверку основных страниц и устранить критические проблемы производительности.
- [x] Провести UI/UX-аудит desktop/mobile и сформировать приоритетный план (`docs/UI_UX_AUDIT.md`). - [x] Провести UI/UX-аудит desktop/mobile и сформировать приоритетный план (`docs/UI_UX_AUDIT.md`).