feat: harden production data and backups
This commit is contained in:
@@ -14,7 +14,7 @@ RF4 Spotter — неофициальный сервис свежих точек
|
|||||||
|
|
||||||
Подробный план и актуальные чекбоксы находятся в [`docs/ROADMAP.md`](docs/ROADMAP.md). Результаты проверки интерфейса и пять приоритетных UX-пакетов описаны в [`docs/UI_UX_AUDIT.md`](docs/UI_UX_AUDIT.md).
|
Подробный план и актуальные чекбоксы находятся в [`docs/ROADMAP.md`](docs/ROADMAP.md). Результаты проверки интерфейса и пять приоритетных UX-пакетов описаны в [`docs/UI_UX_AUDIT.md`](docs/UI_UX_AUDIT.md).
|
||||||
|
|
||||||
Production-контур для домена `rf4spotter.ru`, TLS, секреты и команды первого запуска описаны в [`deploy/README.md`](deploy/README.md). Он использует отдельный `compose.production.yaml`; локальный `compose.yaml` остаётся средой разработки.
|
Production-контур для домена `rf4spotter.ru`, TLS, секреты, backup/restore и команды первого запуска описаны в [`deploy/README.md`](deploy/README.md). Он использует отдельный `compose.production.yaml`; локальный `compose.yaml` остаётся средой разработки. Production seed добавляет только справочники — демонстрационные уловы отключены.
|
||||||
|
|
||||||
Gitea Actions workflow `.gitea/workflows/ci.yml` на каждый push и pull request проверяет Python, миграции на чистой PostgreSQL, Astro build и полный Compose/Playwright-сценарий. При падении E2E сохраняются логи контейнеров и Playwright-артефакты.
|
Gitea Actions workflow `.gitea/workflows/ci.yml` на каждый push и pull request проверяет Python, миграции на чистой PostgreSQL, Astro build и полный Compose/Playwright-сценарий. При падении E2E сохраняются логи контейнеров и Playwright-артефакты.
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
"""Protect pending screenshot uploads with a one-time token."""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "0010"
|
||||||
|
down_revision = "0009"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("catch_report", sa.Column("screenshot_upload_token_hash", sa.String(64)))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("catch_report", "screenshot_upload_token_hash")
|
||||||
@@ -16,6 +16,7 @@ class Settings(BaseSettings):
|
|||||||
official_records_region: str = "RU"
|
official_records_region: str = "RU"
|
||||||
official_records_category: str = "records"
|
official_records_category: str = "records"
|
||||||
official_import_required: bool = False
|
official_import_required: bool = False
|
||||||
|
seed_demo_data: bool = True
|
||||||
import_interval_seconds: int = Field(default=3600, ge=3600)
|
import_interval_seconds: int = Field(default=3600, ge=3600)
|
||||||
rate_limit_secret: str = "change-rate-limit-secret"
|
rate_limit_secret: str = "change-rate-limit-secret"
|
||||||
log_level: str = "INFO"
|
log_level: str = "INFO"
|
||||||
@@ -39,6 +40,8 @@ class Settings(BaseSettings):
|
|||||||
raise ValueError("production CORS_ORIGINS must contain only HTTPS origins")
|
raise ValueError("production CORS_ORIGINS must contain only HTTPS origins")
|
||||||
if not self.s3_public_endpoint_url.startswith("https://"):
|
if not self.s3_public_endpoint_url.startswith("https://"):
|
||||||
raise ValueError("production S3_PUBLIC_ENDPOINT_URL must use HTTPS")
|
raise ValueError("production S3_PUBLIC_ENDPOINT_URL must use HTTPS")
|
||||||
|
if self.seed_demo_data:
|
||||||
|
raise ValueError("SEED_DEMO_DATA must be false in production")
|
||||||
return self
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+15
-6
@@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone
|
|||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import logging
|
import logging
|
||||||
|
import secrets
|
||||||
import time as time_module
|
import time as time_module
|
||||||
from typing import Annotated, Literal
|
from typing import Annotated, Literal
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
@@ -25,7 +26,7 @@ from .importer import ImportSourceError, import_records, normalize
|
|||||||
from .logging_config import configure_logging
|
from .logging_config import configure_logging
|
||||||
from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
|
from .models import Bait, BaitKind, CatchReport, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
|
||||||
from .readiness import readiness_report
|
from .readiness import readiness_report
|
||||||
from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
|
from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ModerationUpdate, OfficialRecordOut, SpotOut, WaterbodyOut
|
||||||
from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot
|
from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot
|
||||||
|
|
||||||
|
|
||||||
@@ -273,8 +274,8 @@ def admin_reject_external_observation(
|
|||||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/v1/catch-reports", response_model=CatchReportCreated, status_code=201)
|
@app.post("/api/v1/catch-reports", response_model=CatchReportAccepted, status_code=201)
|
||||||
def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportCreated:
|
def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportAccepted:
|
||||||
if payload.website:
|
if payload.website:
|
||||||
raise HTTPException(status_code=400, detail="invalid submission")
|
raise HTTPException(status_code=400, detail="invalid submission")
|
||||||
_check_rate_limit(request.client.host if request.client else "unknown", db)
|
_check_rate_limit(request.client.host if request.client else "unknown", db)
|
||||||
@@ -293,17 +294,24 @@ def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) ->
|
|||||||
if bait is None:
|
if bait is None:
|
||||||
bait = Bait(name=payload.bait_name.strip(), normalized_name=key, kind=BaitKind.unknown)
|
bait = Bait(name=payload.bait_name.strip(), normalized_name=key, kind=BaitKind.unknown)
|
||||||
db.add(bait)
|
db.add(bait)
|
||||||
report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None)
|
upload_token = secrets.token_urlsafe(32)
|
||||||
|
report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None, screenshot_upload_token_hash=hashlib.sha256(upload_token.encode()).hexdigest())
|
||||||
db.add(report)
|
db.add(report)
|
||||||
db.commit()
|
db.commit()
|
||||||
return CatchReportCreated(id=report.id, moderation_status=report.moderation_status.value)
|
return CatchReportAccepted(id=report.id, moderation_status=report.moderation_status.value, screenshot_upload_token=upload_token)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/v1/catch-reports/{report_id}/screenshot", status_code=204, response_class=Response)
|
@app.post("/api/v1/catch-reports/{report_id}/screenshot", status_code=204, response_class=Response)
|
||||||
def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) -> Response:
|
def add_screenshot(
|
||||||
|
report_id: UUID, db: Db, screenshot: UploadFile = File(),
|
||||||
|
upload_token: Annotated[str | None, Header(alias="X-Upload-Token")] = None,
|
||||||
|
) -> Response:
|
||||||
report = db.get(CatchReport, report_id)
|
report = db.get(CatchReport, report_id)
|
||||||
if report is None or report.source_type != SourceType.user or report.moderation_status != ModerationStatus.pending:
|
if report is None or report.source_type != SourceType.user or report.moderation_status != ModerationStatus.pending:
|
||||||
raise HTTPException(status_code=404, detail="pending catch report not found")
|
raise HTTPException(status_code=404, detail="pending catch report not found")
|
||||||
|
supplied_hash = hashlib.sha256((upload_token or "").encode()).hexdigest()
|
||||||
|
if not report.screenshot_upload_token_hash or not hmac.compare_digest(report.screenshot_upload_token_hash, supplied_hash):
|
||||||
|
raise HTTPException(status_code=401, detail="invalid screenshot upload token")
|
||||||
if report.screenshot_key:
|
if report.screenshot_key:
|
||||||
raise HTTPException(status_code=409, detail="screenshot already uploaded")
|
raise HTTPException(status_code=409, detail="screenshot already uploaded")
|
||||||
raw = screenshot.file.read(settings.screenshot_max_bytes + 1)
|
raw = screenshot.file.read(settings.screenshot_max_bytes + 1)
|
||||||
@@ -311,6 +319,7 @@ def add_screenshot(report_id: UUID, db: Db, screenshot: UploadFile = File()) ->
|
|||||||
report.screenshot_key = upload_screenshot(raw, filename=screenshot.filename, content_type=screenshot.content_type)
|
report.screenshot_key = upload_screenshot(raw, filename=screenshot.filename, content_type=screenshot.content_type)
|
||||||
except ScreenshotError as exc:
|
except ScreenshotError as exc:
|
||||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||||
|
report.screenshot_upload_token_hash = None
|
||||||
db.commit()
|
db.commit()
|
||||||
return Response(status_code=204)
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ class CatchReport(Base):
|
|||||||
source_confidence: Mapped[int]
|
source_confidence: Mapped[int]
|
||||||
moderation_status: Mapped[ModerationStatus] = mapped_column(Enum(ModerationStatus))
|
moderation_status: Mapped[ModerationStatus] = mapped_column(Enum(ModerationStatus))
|
||||||
screenshot_key: Mapped[str | None] = mapped_column(Text)
|
screenshot_key: Mapped[str | None] = mapped_column(Text)
|
||||||
|
screenshot_upload_token_hash: Mapped[str | None] = mapped_column(String(64))
|
||||||
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
raw_payload: Mapped[dict | None] = mapped_column(JSON)
|
raw_payload: Mapped[dict | None] = mapped_column(JSON)
|
||||||
fish: Mapped[Fish] = relationship()
|
fish: Mapped[Fish] = relationship()
|
||||||
|
|||||||
@@ -137,6 +137,10 @@ class CatchReportCreated(BaseModel):
|
|||||||
moderation_status: str
|
moderation_status: str
|
||||||
|
|
||||||
|
|
||||||
|
class CatchReportAccepted(CatchReportCreated):
|
||||||
|
screenshot_upload_token: str
|
||||||
|
|
||||||
|
|
||||||
class AdminCatchReportOut(BaseModel):
|
class AdminCatchReportOut(BaseModel):
|
||||||
id: UUID
|
id: UUID
|
||||||
fish: str
|
fish: str
|
||||||
|
|||||||
+35
-10
@@ -5,6 +5,7 @@ from uuid import UUID
|
|||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from .config import settings
|
||||||
from .database import SessionLocal
|
from .database import SessionLocal
|
||||||
from .models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody
|
from .models import Bait, BaitKind, CatchReport, Fish, ModerationStatus, SourceType, Spot, Waterbody
|
||||||
|
|
||||||
@@ -23,19 +24,38 @@ IDS = {
|
|||||||
|
|
||||||
def seed() -> None:
|
def seed() -> None:
|
||||||
with SessionLocal.begin() as db:
|
with SessionLocal.begin() as db:
|
||||||
if db.scalar(select(Fish.id).limit(1)) is not None:
|
def entity(model, key: str, value: str, **values):
|
||||||
|
item = db.scalar(select(model).where(getattr(model, key) == value))
|
||||||
|
if item is None:
|
||||||
|
item = model(**values)
|
||||||
|
db.add(item)
|
||||||
|
db.flush()
|
||||||
|
return item
|
||||||
|
|
||||||
|
vyunok = entity(Waterbody, "slug", "vyunok", id=IDS["vyunok"], slug="vyunok", name_ru="Вьюнок", unlock_level=1)
|
||||||
|
kuori = entity(Waterbody, "slug", "kuori", id=IDS["kuori"], slug="kuori", name_ru="Куори", unlock_level=16)
|
||||||
|
pike = entity(Fish, "slug", "pike", id=IDS["pike"], slug="pike", name_ru="Щука", trophy_weight_g=10_000)
|
||||||
|
trout = entity(Fish, "slug", "lake-trout", id=IDS["trout"], slug="lake-trout", name_ru="Озёрная форель", trophy_weight_g=10_000)
|
||||||
|
spiker = entity(Bait, "normalized_name", "spiker #2 01-015", id=IDS["spiker"], name="Spiker #2 01-015", normalized_name="spiker #2 01-015", kind=BaitKind.lure)
|
||||||
|
shad = entity(Bait, "normalized_name", "salmon t1 shad 12 005", id=IDS["shad"], name="Salmon T1 Shad 12 005", normalized_name="salmon t1 shad 12 005", kind=BaitKind.lure)
|
||||||
|
|
||||||
|
def spot(waterbody: Waterbody, identity: str, x: int, y: int, description: str) -> Spot:
|
||||||
|
item = db.scalar(select(Spot).where(Spot.waterbody_id == waterbody.id, Spot.x == x, Spot.y == y))
|
||||||
|
if item is None:
|
||||||
|
item = Spot(id=IDS[identity], waterbody=waterbody, x=x, y=y, description=description)
|
||||||
|
db.add(item)
|
||||||
|
db.flush()
|
||||||
|
return item
|
||||||
|
|
||||||
|
spot1 = spot(vyunok, "spot1", 110, 103, "Кромка травы у северного берега")
|
||||||
|
spot2 = spot(kuori, "spot2", 85, 92, "Свальчик в глубину")
|
||||||
|
if not settings.seed_demo_data:
|
||||||
return
|
return
|
||||||
vyunok = Waterbody(id=IDS["vyunok"], slug="vyunok", name_ru="Вьюнок", unlock_level=1)
|
|
||||||
kuori = Waterbody(id=IDS["kuori"], slug="kuori", name_ru="Куори", unlock_level=16)
|
|
||||||
pike = Fish(id=IDS["pike"], slug="pike", name_ru="Щука", trophy_weight_g=10_000)
|
|
||||||
trout = Fish(id=IDS["trout"], slug="lake-trout", name_ru="Озёрная форель", trophy_weight_g=10_000)
|
|
||||||
spiker = Bait(id=IDS["spiker"], name="Spiker #2 01-015", normalized_name="spiker #2 01-015", kind=BaitKind.lure)
|
|
||||||
shad = Bait(id=IDS["shad"], name="Salmon T1 Shad 12 005", normalized_name="salmon t1 shad 12 005", kind=BaitKind.lure)
|
|
||||||
spot1 = Spot(id=IDS["spot1"], waterbody=vyunok, x=110, y=103, description="Кромка травы у северного берега")
|
|
||||||
spot2 = Spot(id=IDS["spot2"], waterbody=kuori, x=85, y=92, description="Свальчик в глубину")
|
|
||||||
db.add_all([vyunok, kuori, pike, trout, spiker, shad, spot1, spot2])
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
for index in range(12):
|
for index in range(12):
|
||||||
|
external_id = f"seed:pike:{index}"
|
||||||
|
if db.scalar(select(CatchReport.id).where(CatchReport.source_external_id == external_id)):
|
||||||
|
continue
|
||||||
db.add(CatchReport(
|
db.add(CatchReport(
|
||||||
fish=pike, spot=spot1, waterbody=vyunok, bait=spiker,
|
fish=pike, spot=spot1, waterbody=vyunok, bait=spiker,
|
||||||
weight_g=2_600 + index * 480, fishing_method="spinning",
|
weight_g=2_600 + index * 480, fishing_method="spinning",
|
||||||
@@ -43,9 +63,13 @@ def seed() -> None:
|
|||||||
caught_at=now - timedelta(minutes=25 + index * 47),
|
caught_at=now - timedelta(minutes=25 + index * 47),
|
||||||
reported_at=now - timedelta(minutes=20 + index * 47),
|
reported_at=now - timedelta(minutes=20 + index * 47),
|
||||||
player_name=f"DemoPlayer{index % 7 + 1}", source_type=SourceType.manual_import,
|
player_name=f"DemoPlayer{index % 7 + 1}", source_type=SourceType.manual_import,
|
||||||
|
source_external_id=external_id,
|
||||||
source_confidence=80 + index % 3 * 5, moderation_status=ModerationStatus.approved,
|
source_confidence=80 + index % 3 * 5, moderation_status=ModerationStatus.approved,
|
||||||
))
|
))
|
||||||
for index in range(5):
|
for index in range(5):
|
||||||
|
external_id = f"seed:trout:{index}"
|
||||||
|
if db.scalar(select(CatchReport.id).where(CatchReport.source_external_id == external_id)):
|
||||||
|
continue
|
||||||
db.add(CatchReport(
|
db.add(CatchReport(
|
||||||
fish=trout, spot=spot2, waterbody=kuori, bait=shad,
|
fish=trout, spot=spot2, waterbody=kuori, bait=shad,
|
||||||
weight_g=4_200 + index * 900, fishing_method="spinning",
|
weight_g=4_200 + index * 900, fishing_method="spinning",
|
||||||
@@ -53,6 +77,7 @@ def seed() -> None:
|
|||||||
caught_at=now - timedelta(hours=2 + index * 4),
|
caught_at=now - timedelta(hours=2 + index * 4),
|
||||||
reported_at=now - timedelta(hours=2 + index * 4),
|
reported_at=now - timedelta(hours=2 + index * 4),
|
||||||
player_name=f"DemoAngler{index + 1}", source_type=SourceType.manual_import,
|
player_name=f"DemoAngler{index + 1}", source_type=SourceType.manual_import,
|
||||||
|
source_external_id=external_id,
|
||||||
source_confidence=85, moderation_status=ModerationStatus.approved,
|
source_confidence=85, moderation_status=ModerationStatus.approved,
|
||||||
))
|
))
|
||||||
|
|
||||||
|
|||||||
@@ -187,10 +187,13 @@ def test_admin_can_start_and_list_official_import(monkeypatch) -> None:
|
|||||||
def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:
|
def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:
|
||||||
created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 91, "y": 92, "weight_g": 4200}).json()
|
created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 91, "y": 92, "weight_g": 4200}).json()
|
||||||
monkeypatch.setattr("app.main.upload_screenshot", lambda raw, **metadata: "reports/test.jpg" if raw == b"image-bytes" and metadata == {"filename": "catch.jpg", "content_type": "image/jpeg"} else "unexpected")
|
monkeypatch.setattr("app.main.upload_screenshot", lambda raw, **metadata: "reports/test.jpg" if raw == b"image-bytes" and metadata == {"filename": "catch.jpg", "content_type": "image/jpeg"} else "unexpected")
|
||||||
response = client.post(f"/api/v1/catch-reports/{created['id']}/screenshot", files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")})
|
upload_url = f"/api/v1/catch-reports/{created['id']}/screenshot"
|
||||||
|
assert client.post(upload_url, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}).status_code == 401
|
||||||
|
assert client.post(upload_url, headers={"X-Upload-Token": "wrong"}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")}).status_code == 401
|
||||||
|
response = client.post(upload_url, headers={"X-Upload-Token": created["screenshot_upload_token"]}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")})
|
||||||
assert response.status_code == 204
|
assert response.status_code == 204
|
||||||
duplicate = client.post(f"/api/v1/catch-reports/{created['id']}/screenshot", files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")})
|
reused = client.post(upload_url, headers={"X-Upload-Token": created["screenshot_upload_token"]}, files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")})
|
||||||
assert duplicate.status_code == 409
|
assert reused.status_code == 401
|
||||||
|
|
||||||
|
|
||||||
def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None:
|
def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None:
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ def production_settings(**changes) -> Settings:
|
|||||||
"s3_secret_key": "s" * 32,
|
"s3_secret_key": "s" * 32,
|
||||||
"s3_public_endpoint_url": "https://files.rf4spotter.ru",
|
"s3_public_endpoint_url": "https://files.rf4spotter.ru",
|
||||||
"cors_origins": ["https://rf4spotter.ru"],
|
"cors_origins": ["https://rf4spotter.ru"],
|
||||||
|
"seed_demo_data": False,
|
||||||
}
|
}
|
||||||
return Settings(**(values | changes))
|
return Settings(**(values | changes))
|
||||||
|
|
||||||
@@ -30,6 +31,7 @@ def test_production_settings_accept_real_domains_and_secrets() -> None:
|
|||||||
("s3_secret_key", "rf4-local-secret"),
|
("s3_secret_key", "rf4-local-secret"),
|
||||||
("cors_origins", ["http://rf4spotter.ru"]),
|
("cors_origins", ["http://rf4spotter.ru"]),
|
||||||
("s3_public_endpoint_url", "http://files.rf4spotter.ru"),
|
("s3_public_endpoint_url", "http://files.rf4spotter.ru"),
|
||||||
|
("seed_demo_data", True),
|
||||||
])
|
])
|
||||||
def test_production_settings_reject_insecure_values(field: str, value: object) -> None:
|
def test_production_settings_reject_insecure_values(field: str, value: object) -> None:
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
from sqlalchemy import create_engine, func, select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
from sqlalchemy.pool import StaticPool
|
||||||
|
|
||||||
|
from app.database import Base
|
||||||
|
from app.models import CatchReport, Fish, Spot, Waterbody
|
||||||
|
from app import seed as seed_module
|
||||||
|
|
||||||
|
|
||||||
|
def test_seed_repairs_partial_database_and_is_idempotent(monkeypatch) -> None:
|
||||||
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
sessions = sessionmaker(bind=engine, expire_on_commit=False)
|
||||||
|
with Session(engine) as db:
|
||||||
|
db.add(Fish(slug="pike", name_ru="Щука", trophy_weight_g=10_000))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
monkeypatch.setattr(seed_module, "SessionLocal", sessions)
|
||||||
|
monkeypatch.setattr(seed_module.settings, "seed_demo_data", False)
|
||||||
|
seed_module.seed()
|
||||||
|
seed_module.seed()
|
||||||
|
|
||||||
|
with Session(engine) as db:
|
||||||
|
assert db.scalar(select(func.count()).select_from(Fish)) == 2
|
||||||
|
assert db.scalar(select(func.count()).select_from(Waterbody)) == 2
|
||||||
|
assert db.scalar(select(func.count()).select_from(Spot)) == 2
|
||||||
|
assert db.scalar(select(func.count()).select_from(CatchReport)) == 0
|
||||||
@@ -2,14 +2,17 @@ import type { APIRoute } from "astro";
|
|||||||
|
|
||||||
const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000";
|
const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000";
|
||||||
|
|
||||||
export const POST: APIRoute = async ({ request, redirect }) => {
|
export const POST: APIRoute = async ({ request, redirect, cookies }) => {
|
||||||
const form = await request.formData();
|
const form = await request.formData();
|
||||||
const reportId = String(form.get("report_id") || "");
|
const reportId = String(form.get("report_id") || "");
|
||||||
const screenshot = form.get("screenshot");
|
const screenshot = form.get("screenshot");
|
||||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(reportId) || !(screenshot instanceof File) || screenshot.size === 0) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303);
|
const cookieName = `rf4-upload-${reportId}`;
|
||||||
|
const uploadToken = cookies.get(cookieName)?.value;
|
||||||
|
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(reportId) || !uploadToken || !(screenshot instanceof File) || screenshot.size === 0) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303);
|
||||||
try {
|
try {
|
||||||
const upload = new FormData(); upload.set("screenshot", screenshot);
|
const upload = new FormData(); upload.set("screenshot", screenshot);
|
||||||
const response = await fetch(`${base}/api/v1/catch-reports/${reportId}/screenshot`, { method: "POST", body: upload });
|
const response = await fetch(`${base}/api/v1/catch-reports/${reportId}/screenshot`, { method: "POST", headers:{"X-Upload-Token":uploadToken}, body: upload });
|
||||||
|
if (response.ok) cookies.delete(cookieName, {path:"/"});
|
||||||
return redirect(response.ok ? "/report?state=screenshot_sent" : `/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303);
|
return redirect(response.ok ? "/report?state=screenshot_sent" : `/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303);
|
||||||
} catch { return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); }
|
} catch { return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(reportId)}`, 303); }
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,23 +1,32 @@
|
|||||||
import type { APIRoute } from "astro";
|
import type { APIRoute } from "astro";
|
||||||
const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000";
|
const base = process.env.API_INTERNAL_URL || import.meta.env.API_INTERNAL_URL || "http://localhost:8000";
|
||||||
export const POST: APIRoute = async ({ request, redirect }) => {
|
export const POST: APIRoute = async ({ request, redirect, cookies }) => {
|
||||||
const form = await request.formData();
|
const form = await request.formData();
|
||||||
const text = (name: string) => String(form.get(name) || "").trim() || null;
|
const text = (name: string) => String(form.get(name) || "").trim() || null;
|
||||||
const number = (name: string) => text(name) ? Number(text(name)) : null;
|
const number = (name: string) => text(name) ? Number(text(name)) : null;
|
||||||
const payload = { fish_slug: String(form.get("fish_slug") || ""), waterbody_slug: String(form.get("waterbody_slug") || ""), x: Number(form.get("x")), y: Number(form.get("y")), weight_g: Number(form.get("weight_g")), bait_name: text("bait_name"), fishing_method: text("fishing_method"), retrieve_method: text("retrieve_method"), retrieve_speed: number("retrieve_speed"), player_name: text("player_name"), comment: text("comment"), website: String(form.get("website") || "") };
|
const payload = { fish_slug: String(form.get("fish_slug") || ""), waterbody_slug: String(form.get("waterbody_slug") || ""), x: Number(form.get("x")), y: Number(form.get("y")), weight_g: Number(form.get("weight_g")), bait_name: text("bait_name"), fishing_method: text("fishing_method"), retrieve_method: text("retrieve_method"), retrieve_speed: number("retrieve_speed"), player_name: text("player_name"), comment: text("comment"), website: String(form.get("website") || "") };
|
||||||
let createdId: string | null = null;
|
let createdId: string | null = null;
|
||||||
|
let uploadToken: string | null = null;
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${base}/api/v1/catch-reports`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) });
|
const response = await fetch(`${base}/api/v1/catch-reports`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(payload) });
|
||||||
if (!response.ok) return redirect("/report?state=create_error", 303);
|
if (!response.ok) return redirect("/report?state=create_error", 303);
|
||||||
const created = await response.json();
|
const created = await response.json() as { id?: unknown; screenshot_upload_token?: unknown };
|
||||||
|
if (typeof created.id !== "string" || typeof created.screenshot_upload_token !== "string") return redirect("/report?state=create_error", 303);
|
||||||
createdId = created.id;
|
createdId = created.id;
|
||||||
|
uploadToken = created.screenshot_upload_token;
|
||||||
const screenshot = form.get("screenshot");
|
const screenshot = form.get("screenshot");
|
||||||
if (screenshot instanceof File && screenshot.size > 0) {
|
if (screenshot instanceof File && screenshot.size > 0) {
|
||||||
const upload = new FormData(); upload.set("screenshot", screenshot);
|
const upload = new FormData(); upload.set("screenshot", screenshot);
|
||||||
const uploaded = await fetch(`${base}/api/v1/catch-reports/${created.id}/screenshot`, { method: "POST", body: upload });
|
const uploaded = await fetch(`${base}/api/v1/catch-reports/${created.id}/screenshot`, { method: "POST", headers: {"X-Upload-Token": created.screenshot_upload_token}, body: upload });
|
||||||
if (!uploaded.ok) return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(created.id)}`, 303);
|
if (!uploaded.ok) {
|
||||||
|
cookies.set(`rf4-upload-${created.id}`, created.screenshot_upload_token, {httpOnly:true, sameSite:"strict", secure:import.meta.env.PROD, path:"/", maxAge:3600});
|
||||||
|
return redirect(`/report?state=screenshot_error&report_id=${encodeURIComponent(created.id)}`, 303);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return redirect("/report?state=sent", 303);
|
return redirect("/report?state=sent", 303);
|
||||||
}
|
}
|
||||||
catch { return redirect(createdId ? `/report?state=screenshot_error&report_id=${encodeURIComponent(createdId)}` : "/report?state=create_error", 303); }
|
catch {
|
||||||
|
if (createdId && uploadToken) cookies.set(`rf4-upload-${createdId}`, uploadToken, {httpOnly:true, sameSite:"strict", secure:import.meta.env.PROD, path:"/", maxAge:3600});
|
||||||
|
return redirect(createdId ? `/report?state=screenshot_error&report_id=${encodeURIComponent(createdId)}` : "/report?state=create_error", 303);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ services:
|
|||||||
OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU}
|
OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU}
|
||||||
OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records}
|
OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records}
|
||||||
OFFICIAL_IMPORT_REQUIRED: ${OFFICIAL_IMPORT_REQUIRED:-false}
|
OFFICIAL_IMPORT_REQUIRED: ${OFFICIAL_IMPORT_REQUIRED:-false}
|
||||||
|
SEED_DEMO_DATA: "false"
|
||||||
IMPORT_INTERVAL_SECONDS: ${IMPORT_INTERVAL_SECONDS:-3600}
|
IMPORT_INTERVAL_SECONDS: ${IMPORT_INTERVAL_SECONDS:-3600}
|
||||||
RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:?Set RATE_LIMIT_SECRET}
|
RATE_LIMIT_SECRET: ${RATE_LIMIT_SECRET:?Set RATE_LIMIT_SECRET}
|
||||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||||
@@ -122,6 +123,14 @@ services:
|
|||||||
resources:
|
resources:
|
||||||
limits: {cpus: "0.75", memory: 512M}
|
limits: {cpus: "0.75", memory: 512M}
|
||||||
|
|
||||||
|
storage-tool:
|
||||||
|
image: alpine:3.22
|
||||||
|
profiles: [tools]
|
||||||
|
volumes:
|
||||||
|
- minio_data:/data
|
||||||
|
- ${BACKUP_DIRECTORY:-./backups}:/backup
|
||||||
|
security_opt: [no-new-privileges:true]
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
edge:
|
edge:
|
||||||
backend:
|
backend:
|
||||||
|
|||||||
+20
-5
@@ -32,7 +32,7 @@ curl -fsS https://rf4spotter.ru/health
|
|||||||
curl -fsS https://rf4spotter.ru/ready
|
curl -fsS https://rf4spotter.ru/ready
|
||||||
```
|
```
|
||||||
|
|
||||||
API-контейнер перед стартом применяет Alembic-миграции и запускает seed. До альфы демонстрационные уловы следует убрать отдельной задачей; справочники должны остаться.
|
API-контейнер перед стартом применяет Alembic-миграции и запускает идемпотентный seed. В production он добавляет только минимальные справочники и точки; демонстрационные уловы жёстко отключены настройкой `SEED_DEMO_DATA=false`.
|
||||||
|
|
||||||
Проверка TLS и маршрутизации:
|
Проверка TLS и маршрутизации:
|
||||||
|
|
||||||
@@ -65,11 +65,26 @@ curl -fsS https://rf4spotter.ru/ready
|
|||||||
|
|
||||||
Перед обновлением со сменой схемы обязателен backup PostgreSQL. Не удаляйте volumes и не используйте `down -v`.
|
Перед обновлением со сменой схемы обязателен backup PostgreSQL. Не удаляйте volumes и не используйте `down -v`.
|
||||||
|
|
||||||
## 6. Что ещё блокирует приглашение альфа-пользователей
|
## 6. Резервное копирование и восстановление
|
||||||
|
|
||||||
- проверенный backup/restore PostgreSQL и MinIO;
|
Храните копии вне диска приложения. Скрипт создаёт PostgreSQL dump, архив MinIO и контрольные суммы в новом каталоге с UTC-временем:
|
||||||
- production-safe seed без демонстрационных уловов;
|
|
||||||
- одноразовый секрет для повторной загрузки скриншота вместо одного UUID;
|
```bash
|
||||||
|
./deploy/backup.sh /srv/rf4-backups
|
||||||
|
```
|
||||||
|
|
||||||
|
Восстановление заменяет содержимое PostgreSQL и MinIO данными из выбранной копии, временно останавливая API, web и MinIO. Это намеренно защищённая подтверждением операция:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CONFIRM_RESTORE=rf4-spotter ./deploy/restore.sh /srv/rf4-backups/20260906T120000Z
|
||||||
|
curl -fsS https://rf4spotter.ru/ready
|
||||||
|
```
|
||||||
|
|
||||||
|
Перед приглашением пользователей проведите учебное восстановление на отдельном сервере или с отдельным Compose project name. Одной только успешной архивации недостаточно. После проверки настройте ежедневный запуск `backup.sh`, шифрование/выгрузку копий во внешнее хранилище и уведомление при ошибке; храните минимум 7 ежедневных и 4 еженедельных копии.
|
||||||
|
|
||||||
|
## 7. Что ещё блокирует приглашение альфа-пользователей
|
||||||
|
|
||||||
|
- учебное восстановление PostgreSQL и MinIO на изолированном контуре;
|
||||||
- проверка полного bootstrap на пустых volumes;
|
- проверка полного bootstrap на пустых volumes;
|
||||||
- политика хранения и удаления пользовательских данных;
|
- политика хранения и удаления пользовательских данных;
|
||||||
- базовый мониторинг `/ready`, диска и срока TLS-сертификата.
|
- базовый мониторинг `/ready`, диска и срока TLS-сертификата.
|
||||||
|
|||||||
Executable
+26
@@ -0,0 +1,26 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if [ "$#" -ne 1 ]; then
|
||||||
|
echo "Usage: $0 ABSOLUTE_BACKUP_DIRECTORY" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
/*) ;;
|
||||||
|
*) echo "Backup directory must be an absolute path" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
backup_root=${1%/}
|
||||||
|
stamp=$(date -u +%Y%m%dT%H%M%SZ)
|
||||||
|
target="$backup_root/$stamp"
|
||||||
|
mkdir -p "$target"
|
||||||
|
chmod 700 "$target"
|
||||||
|
export BACKUP_DIRECTORY="$backup_root"
|
||||||
|
|
||||||
|
compose="docker compose --env-file .env.production -f compose.production.yaml"
|
||||||
|
$compose exec -T db sh -c 'pg_dump --format=custom --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$target/postgres.dump"
|
||||||
|
$compose run --rm storage-tool tar -czf "/backup/$stamp/minio.tar.gz" -C /data .
|
||||||
|
sha256sum "$target/postgres.dump" "$target/minio.tar.gz" > "$target/SHA256SUMS"
|
||||||
|
chmod 600 "$target/postgres.dump" "$target/minio.tar.gz" "$target/SHA256SUMS"
|
||||||
|
echo "Backup created: $target"
|
||||||
Executable
+27
@@ -0,0 +1,27 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if [ "$#" -ne 1 ] || [ "${CONFIRM_RESTORE:-}" != "rf4-spotter" ]; then
|
||||||
|
echo "Usage: CONFIRM_RESTORE=rf4-spotter $0 ABSOLUTE_BACKUP_DIRECTORY/TIMESTAMP" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$1" in
|
||||||
|
/*) ;;
|
||||||
|
*) echo "Backup directory must be an absolute path" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
source_dir=${1%/}
|
||||||
|
test -f "$source_dir/postgres.dump"
|
||||||
|
test -f "$source_dir/minio.tar.gz"
|
||||||
|
(cd "$source_dir" && sha256sum -c SHA256SUMS)
|
||||||
|
backup_root=$(dirname "$source_dir")
|
||||||
|
stamp=$(basename "$source_dir")
|
||||||
|
export BACKUP_DIRECTORY="$backup_root"
|
||||||
|
|
||||||
|
compose="docker compose --env-file .env.production -f compose.production.yaml"
|
||||||
|
$compose stop api web minio
|
||||||
|
$compose exec -T db sh -c 'pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$source_dir/postgres.dump"
|
||||||
|
$compose run --rm storage-tool sh -c 'find /data -mindepth 1 -delete && tar -xzf "/backup/'"$stamp"'/minio.tar.gz" -C /data'
|
||||||
|
$compose up -d minio api web proxy
|
||||||
|
echo "Restore completed from: $source_dir"
|
||||||
+3
-3
@@ -60,13 +60,13 @@
|
|||||||
- [x] Добавить структурированные JSON-логи без пользовательских секретов и персональных технических данных (whitelist полей, redaction, request ID; Uvicorn access-log отключён).
|
- [x] Добавить структурированные JSON-логи без пользовательских секретов и персональных технических данных (whitelist полей, redaction, request ID; Uvicorn access-log отключён).
|
||||||
- [x] Добавить Gitea Actions CI: backend tests, Astro check/build, E2E и применение всех миграций на чистой PostgreSQL; сохранять логи Compose и Playwright-артефакты при падении (`.gitea/workflows/ci.yml`).
|
- [x] Добавить Gitea Actions CI: backend tests, Astro check/build, E2E и применение всех миграций на чистой PostgreSQL; сохранять логи Compose и Playwright-артефакты при падении (`.gitea/workflows/ci.yml`).
|
||||||
- [ ] Добавить отдельный тест полного bootstrap: пустые volumes → миграции → seed → readiness → основной E2E.
|
- [ ] Добавить отдельный тест полного bootstrap: пустые volumes → миграции → seed → readiness → основной E2E.
|
||||||
- [ ] Сделать seed устойчивым к частично заполненной БД и покрыть повторный/частичный запуск тестом; текущая реализация прекращает работу при наличии любой рыбы.
|
- [x] Сделать seed устойчивым к частично заполненной БД: справочники досеиваются независимо, демо-уловы идемпотентны и принудительно отключены в production; повторный/частичный запуск покрыт конфигурационными и интеграционными проверками.
|
||||||
- [ ] Проверить списочные API по требованию раздела 12: пагинация, предсказуемая сортировка и валидация фильтров для справочников, импортов, модерации и внешнего staging.
|
- [ ] Проверить списочные API по требованию раздела 12: пагинация, предсказуемая сортировка и валидация фильтров для справочников, импортов, модерации и внешнего staging.
|
||||||
- [ ] Проверить необходимые индексы PostgreSQL и планы запросов для activity, модерации, дедупликации и очистки rate limit; зафиксировать допустимый бюджет запросов пилота.
|
- [ ] Проверить необходимые индексы PostgreSQL и планы запросов для activity, модерации, дедупликации и очистки rate limit; зафиксировать допустимый бюджет запросов пилота.
|
||||||
- [ ] Провести security-проверку admin-аутентификации, CORS, security headers, загрузок и управления секретами; вынести допустимые origins в конфигурацию и исключить демонстрационные секреты в production-режиме.
|
- [ ] Провести security-проверку admin-аутентификации, CORS, security headers, загрузок и управления секретами; вынести допустимые origins в конфигурацию и исключить демонстрационные секреты в production-режиме.
|
||||||
- [ ] Проверить авторизацию повторной загрузки скриншота: один UUID pending-заявки не должен быть достаточным полномочием для изменения чужой записи.
|
- [x] Проверить авторизацию повторной загрузки скриншота: используется отдельный одноразовый случайный токен, в БД хранится только SHA-256, UUID заявки недостаточно.
|
||||||
- [ ] Определить сроки хранения ников, исходных payload, staging-наблюдений, moderation events и submission attempts; добавить документированную очистку/анонимизацию.
|
- [ ] Определить сроки хранения ников, исходных payload, staging-наблюдений, moderation events и submission attempts; добавить документированную очистку/анонимизацию.
|
||||||
- [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO; проверить восстановление на отдельных временных volumes.
|
- [ ] Добавить резервное копирование и документированное восстановление PostgreSQL и MinIO: скрипты, контрольные суммы и runbook готовы; остаётся учебное восстановление на отдельных временных volumes.
|
||||||
- [ ] Проверить доступность интерфейса: клавиатура, focus states, контраст, подписи полей и семантика таблиц/карточек.
|
- [ ] Проверить доступность интерфейса: клавиатура, focus states, контраст, подписи полей и семантика таблиц/карточек.
|
||||||
- [ ] Провести Lighthouse-проверку основных страниц и устранить критические проблемы производительности.
|
- [ ] Провести Lighthouse-проверку основных страниц и устранить критические проблемы производительности.
|
||||||
- [x] Провести UI/UX-аудит desktop/mobile и сформировать приоритетный план (`docs/UI_UX_AUDIT.md`).
|
- [x] Провести UI/UX-аудит desktop/mobile и сформировать приоритетный план (`docs/UI_UX_AUDIT.md`).
|
||||||
|
|||||||
Reference in New Issue
Block a user