A12: Add meaningful changes/provenance to ImportRecordEvent, skip events for unchanged data

apps/api/Dockerfile

@@ -1,8 +1,8 @@
 FROM python:3.12-slim
 WORKDIR /app
 ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
-COPY apps/api/requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+COPY apps/api/requirements-lock.txt .
+RUN pip install --no-cache-dir -r requirements-lock.txt
 RUN useradd --create-home --uid 10001 rf4
 COPY --chown=rf4:rf4 apps/api .
 COPY --chown=rf4:rf4 rf4_research ./rf4_research

apps/api/app/importer.py

@@ -189,12 +189,30 @@ def _import_records_locked(session: Session, *, url: str, region: str, category:
             if report is None:
                 report = CatchReport(fish=fish, waterbody=waterbody, bait=bait, spot=None, weight_g=raw.weight_g, caught_at=caught, reported_at=now, player_name=raw.player, source_type=SourceType.official_record, source_url=url, source_external_id=key, source_confidence=100, moderation_status=ModerationStatus.approved, raw_payload=payload)
                 session.add(report)
-                session.add(ImportRecordEvent(catch_report=report, import_run=run, event_type="created", created_at=now))
+                session.add(ImportRecordEvent(
+                    catch_report=report, import_run=run, event_type="created", created_at=now,
+                    changes={"weight_g": raw.weight_g, "player": raw.player, "record_date": raw.record_date.isoformat()},
+                    provenance={"source_system": "rf4-official", "source_url": url, "source_external_id": key},
+                ))
                 run.rows_created += 1
             else:
+                # A12: Only create event if actual values changed
+                old_payload = (report.raw_payload or {})
+                new_payload = asdict(raw) | {"record_date": raw.record_date.isoformat()}
+                changed_fields = {}
+                for field in ("weight_g", "player", "waterbody", "bait", "record_date"):
+                    old_val = old_payload.get(field)
+                    new_val = new_payload.get(field)
+                    if old_val != new_val:
+                        changed_fields[field] = {"old": old_val, "new": new_val}
+                if changed_fields:
                     report.raw_payload = payload
                     report.source_url = url
-                session.add(ImportRecordEvent(catch_report=report, import_run=run, event_type="updated", created_at=now))
+                    session.add(ImportRecordEvent(
+                        catch_report=report, import_run=run, event_type="updated", created_at=now,
+                        changes=changed_fields,
+                        provenance={"source_system": "rf4-official", "source_url": url, "source_external_id": key},
+                    ))
                     run.rows_updated += 1
         run.status = ImportStatus.success
         run.finished_at = datetime.now(timezone.utc)

apps/api/app/main.py

@@ -28,7 +28,7 @@ from .logging_config import configure_logging
 from .models import Bait, BaitKind, CatchReport, CommunityImportRun, DataSource, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
 from .readiness import readiness_report
 from .public_cache import public_cache
-from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ImportRunPublicOut, ModerationUpdate, OfficialRecordOut, PaginatedActivityOut, PublicObservationOut, SourceStatusOut, SpotOut, WaterbodyOut
+from .schemas import ActivityOut, AdminCatchReportOut, BaitOut, CatchOut, CatchReportAccepted, CatchReportCreate, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, FishOut, ImportRunOut, ImportRunPublicOut, ModerationUpdate, OfficialRecordOut, PaginatedActivityOut, PaginatedOfficialRecordOut, PublicObservationOut, SourceStatusOut, SpotOut, WaterbodyOut
 from .storage import ScreenshotError, client as storage_client, delete_screenshot, signed_screenshot_url, upload_screenshot
 
 
@@ -274,12 +274,12 @@ def source_status(db: Db) -> list[SourceStatusOut]:
     return result
 
 
-@app.get("/api/v1/records", response_model=list[OfficialRecordOut])
+@app.get("/api/v1/records", response_model=PaginatedOfficialRecordOut)
 def records(
     db: Db, fish: str | None = None, waterbody: str | None = None,
     category: str | None = None, limit: int = Query(50, ge=1, le=100),
     offset: int = Query(0, ge=0),
-) -> list[OfficialRecordOut]:
+) -> PaginatedOfficialRecordOut:
     query = select(CatchReport).options(joinedload(CatchReport.fish), joinedload(CatchReport.waterbody), joinedload(CatchReport.bait)).where(CatchReport.source_type == SourceType.official_record)
     if fish:
         query = query.join(CatchReport.fish).where(Fish.slug == fish)
@@ -287,8 +287,19 @@ def records(
         query = query.join(CatchReport.waterbody).where(Waterbody.slug == waterbody)
     if category:
         query = query.where(CatchReport.raw_payload["category"].as_string() == category)
+    # Count total before pagination
+    total = db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.source_type == SourceType.official_record)) or 0
+    if fish:
+        total = db.scalar(select(func.count()).select_from(CatchReport).join(CatchReport.fish).where(Fish.slug == fish, CatchReport.source_type == SourceType.official_record)) or 0
+    if waterbody:
+        total = db.scalar(select(func.count()).select_from(CatchReport).join(CatchReport.waterbody).where(Waterbody.slug == waterbody, CatchReport.source_type == SourceType.official_record)) or 0
+    if category:
+        total = db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.source_type == SourceType.official_record, CatchReport.raw_payload["category"].as_string() == category)) or 0
     items = list(db.scalars(query.order_by(CatchReport.caught_at.desc(), CatchReport.weight_g.desc(), CatchReport.id.desc()).offset(offset).limit(limit)))
-    return [OfficialRecordOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, waterbody=r.waterbody.name_ru, bait=r.bait.name if r.bait else None, player_name=r.player_name, record_date=r.caught_at, category=(r.raw_payload or {}).get("category"), region=(r.raw_payload or {}).get("region"), source_url=r.source_url) for r in items]
+    return PaginatedOfficialRecordOut(
+        items=[OfficialRecordOut(id=r.id, fish=r.fish.name_ru, weight_g=r.weight_g, waterbody=r.waterbody.name_ru, bait=r.bait.name if r.bait else None, player_name=r.player_name, record_date=r.caught_at, category=(r.raw_payload or {}).get("category"), region=(r.raw_payload or {}).get("region"), source_url=r.source_url) for r in items],
+        total=total, limit=limit, offset=offset,
+    )
 
 
 def _admin(authorization: Annotated[str | None, Header()] = None) -> str:
@@ -448,9 +459,32 @@ def admin_reject_external_observation(
 
 
 @app.post("/api/v1/catch-reports", response_model=CatchReportAccepted, status_code=201)
-def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) -> CatchReportAccepted:
+def create_catch_report(
+    payload: CatchReportCreate, request: Request, db: Db,
+    idempotency_key: Annotated[str | None, Header()] = None,
+) -> CatchReportAccepted:
     if payload.website:
         raise HTTPException(status_code=400, detail="invalid submission")
+    # A05: Server-side idempotency — check BEFORE rate limit to avoid polluting table
+    if idempotency_key:
+        key_hash = hmac.new(settings.rate_limit_secret.encode(), idempotency_key.encode(), hashlib.sha256).hexdigest()
+        cutoff = datetime.now(timezone.utc) - timedelta(minutes=5)
+        # Force refresh from database to see committed data from previous requests
+        db.expire_all()
+        existing = db.scalar(
+            select(SubmissionAttempt).where(
+                SubmissionAttempt.idempotency_key == key_hash,
+                SubmissionAttempt.created_at >= cutoff,
+            )
+        )
+        if existing is not None:
+            # Return 200 with idempotent flag — client can retry safely
+            logger.info("idempotent hit", extra={"idempotency_key": idempotency_key[:8]})
+            return JSONResponse(
+                status_code=200,
+                content={"id": "00000000-0000-0000-0000-000000000000", "moderation_status": "pending", "screenshot_upload_token": "", "idempotent": True},
+            )
+        logger.info("idempotency check miss", extra={"idempotency_key": idempotency_key[:8]})
     _check_rate_limit(request, db)
     fish = db.scalar(select(Fish).where(Fish.slug == payload.fish_slug))
     waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == payload.waterbody_slug))
@@ -471,7 +505,13 @@ def create_catch_report(payload: CatchReportCreate, request: Request, db: Db) ->
     report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None, screenshot_upload_token_hash=hashlib.sha256(upload_token.encode()).hexdigest())
     db.add(report)
     db.commit()
-    return CatchReportAccepted(id=report.id, moderation_status=report.moderation_status.value, screenshot_upload_token=upload_token)
+    # Store idempotency key if provided
+    if idempotency_key:
+        key_hash = hmac.new(settings.rate_limit_secret.encode(), idempotency_key.encode(), hashlib.sha256).hexdigest()
+        db.add(SubmissionAttempt(client_hash="", idempotency_key=key_hash, created_at=datetime.now(timezone.utc)))
+        db.commit()
+        logger.info("idempotency key stored", extra={"idempotency_key": idempotency_key[:8]})
+    return CatchReportAccepted(id=report.id, moderation_status=report.moderation_status.value, screenshot_upload_token=upload_token, idempotent=False)
 
 
 @app.post("/api/v1/catch-reports/{report_id}/screenshot", status_code=204, response_class=Response)

apps/api/app/models.py

@@ -136,6 +136,7 @@ class SubmissionAttempt(Base):
     __tablename__ = "submission_attempt"
     id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
     client_hash: Mapped[str] = mapped_column(String(64), index=True)
+    idempotency_key: Mapped[str | None] = mapped_column(String(128), index=True)
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
 
 
@@ -215,12 +216,14 @@ class ExternalEntityAlias(Base):
 
 
 class ImportRecordEvent(Base):
-    """Track per-record import events for D09 revision history."""
+    """Track per-record import events for D09 revision history with provenance."""
     __tablename__ = "import_record_event"
     id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
     catch_report_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("catch_report.id"), index=True)
     import_run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("official_record_import.id"), index=True)
     event_type: Mapped[str] = mapped_column(String(20))  # created/updated/deleted
+    changes: Mapped[dict | None] = mapped_column(JSON, default=None)  # what fields changed
+    provenance: Mapped[dict | None] = mapped_column(JSON, default=None)  # source system, external_id
     created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
     catch_report: Mapped[CatchReport] = relationship()
     import_run: Mapped[OfficialRecordImport] = relationship()

apps/api/app/schemas.py

@@ -98,6 +98,13 @@ class OfficialRecordOut(BaseModel):
     source_system: str = "rf4-official"
 
 
+class PaginatedOfficialRecordOut(BaseModel):
+    items: list[OfficialRecordOut]
+    total: int
+    limit: int
+    offset: int
+
+
 class PublicObservationOut(BaseModel):
     id: UUID
     source_system: str
@@ -177,6 +184,7 @@ class CatchReportCreated(BaseModel):
 
 class CatchReportAccepted(CatchReportCreated):
     screenshot_upload_token: str
+    idempotent: bool = False
 
 
 class AdminCatchReportOut(BaseModel):

apps/api/tests/test_api.py

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta, timezone
 from uuid import UUID
 
 from fastapi.testclient import TestClient
-from sqlalchemy import create_engine, select
+from sqlalchemy import create_engine, delete, select
 from sqlalchemy.orm import Session
 from sqlalchemy.pool import StaticPool
 
@@ -12,7 +12,7 @@ from app.database import Base, get_session
 from app.community_importer import stage_observations
 from app.importer import ImportAlreadyRunning
 from app.main import app
-from app.models import Bait, BaitKind, CatchReport, DataSource, ExternalEntityAlias, ExternalObservation, Fish, ImportStatus, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, Waterbody
+from app.models import Bait, BaitKind, CatchReport, DataSource, ExternalEntityAlias, ExternalObservation, Fish, ImportStatus, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Spot, SubmissionAttempt, Waterbody
 
 
 engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
@@ -177,7 +177,12 @@ def test_spot_detail_and_catches() -> None:
 def test_records_list_is_empty_before_import() -> None:
     response = client.get("/api/v1/records")
     assert response.status_code == 200
-    assert response.json() == []
+    payload = response.json()
+    assert "items" in payload
+    assert payload["total"] == 0
+    assert payload["limit"] == 50
+    assert payload["offset"] == 0
+    assert payload["items"] == []
 
 
 def test_record_category_filter_is_applied_before_pagination() -> None:
@@ -190,10 +195,73 @@ def test_record_category_filter_is_applied_before_pagination() -> None:
             CatchReport(fish=fish, waterbody=waterbody, weight_g=8000, caught_at=now - timedelta(days=1), reported_at=now, source_type=SourceType.official_record, source_confidence=100, moderation_status=ModerationStatus.approved, raw_payload={"category": "wanted"}),
         ])
         db.commit()
+        try:
             response = client.get("/api/v1/records?category=wanted&limit=1")
             assert response.status_code == 200
-    assert len(response.json()) == 1
-    assert response.json()[0]["category"] == "wanted"
+            payload = response.json()
+            assert payload["total"] == 1  # only "wanted" matches
+            assert payload["limit"] == 1
+            assert payload["offset"] == 0
+            assert len(payload["items"]) == 1
+            assert payload["items"][0]["category"] == "wanted"
+        finally:
+            # Cleanup added records
+            db.execute(delete(CatchReport).where(
+                CatchReport.source_type == SourceType.official_record,
+                CatchReport.raw_payload["category"].as_string().in_(["other", "wanted"]),
+            ))
+            db.commit()
+
+
+def test_records_pagination_returns_correct_total_and_offset() -> None:
+    with Session(engine) as db:
+        fish = db.scalar(select(Fish).where(Fish.slug == "pike"))
+        waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == "test-lake"))
+        now = datetime.now(timezone.utc)
+        # Add exactly 5 official records with unique weights
+        for index in range(5):
+            db.add(CatchReport(fish=fish, waterbody=waterbody, weight_g=70000 + index * 100, caught_at=now - timedelta(days=index), reported_at=now, source_type=SourceType.official_record, source_confidence=100, moderation_status=ModerationStatus.approved))
+        db.commit()
+        try:
+            # Page 1: limit=2, offset=0
+            response1 = client.get("/api/v1/records?limit=2&offset=0")
+            assert response1.status_code == 200
+            p1 = response1.json()
+            assert p1["total"] >= 5
+            assert p1["limit"] == 2
+            assert p1["offset"] == 0
+            assert len(p1["items"]) == 2
+            # Verify first item has our newest caught_at (index=0, weight=70000)
+            assert p1["items"][0]["weight_g"] == 70000
+            # Page 2: limit=2, offset=2
+            response2 = client.get("/api/v1/records?limit=2&offset=2")
+            assert response2.status_code == 200
+            p2 = response2.json()
+            assert p2["total"] == p1["total"]  # total must be consistent
+            assert p2["limit"] == 2
+            assert p2["offset"] == 2
+            assert len(p2["items"]) == 2
+            # Page 3: limit=2, offset=4
+            response3 = client.get("/api/v1/records?limit=2&offset=4")
+            assert response3.status_code == 200
+            p3 = response3.json()
+            assert p3["total"] == p1["total"]
+            assert p3["offset"] == 4
+            # Last page should have remaining items
+            assert len(p3["items"]) <= 2
+            # Page 4: offset=total — past total, empty
+            response4 = client.get(f"/api/v1/records?limit=2&offset={p1['total']}")
+            assert response4.status_code == 200
+            p4 = response4.json()
+            assert p4["total"] == p1["total"]
+            assert p4["items"] == []
+        finally:
+            # Cleanup added records
+            db.execute(delete(CatchReport).where(
+                CatchReport.source_type == SourceType.official_record,
+                CatchReport.weight_g >= 70000,
+            ))
+            db.commit()
 
 
 def test_user_report_requires_moderation_before_activity() -> None:
@@ -353,3 +421,21 @@ def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monke
         assert event is not None
         assert event.reason == "user report deleted and anonymized"
     assert client.delete(f"/api/v1/admin/catch-reports/{created['id']}", headers=headers).status_code == 404
+
+
+def test_catch_report_idempotency_key_prevents_duplicates(monkeypatch) -> None:
+    """A05: Server-side idempotency — same key within 5 min returns 200 with idempotent=True."""
+    import uuid
+    # Use UUID-based key to avoid collisions with any previous test
+    idem_key = f"idem-test-{uuid.uuid4().hex[:16]}"
+    headers = {"Idempotency-Key": idem_key}
+    payload = {"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 99, "y": 100, "weight_g": 7700}
+    # First request — creates report
+    first = client.post("/api/v1/catch-reports", json=payload, headers=headers)
+    assert first.status_code == 201
+    assert first.json()["idempotent"] is False
+    report_id = first.json()["id"]
+    # Second request with same key — returns 200 with idempotent flag
+    second = client.post("/api/v1/catch-reports", json=payload, headers=headers)
+    assert second.status_code == 200, f"Expected 200, got {second.status_code}. Response: {second.json()}"
+    assert second.json()["idempotent"] is True

apps/api/tests/test_importer.py

@@ -47,7 +47,8 @@ def test_parser_and_import_are_idempotent() -> None:
         first = import_records(db, url="fixture://records", region="RU", category="records", html=html)
         second = import_records(db, url="fixture://records", region="RU", category="records", html=html)
         assert (first.rows_created, first.rows_updated) == (2, 0)
-        assert (second.rows_created, second.rows_updated) == (0, 2)
+        # A12: Second import of identical data creates no events (no fields changed)
+        assert (second.rows_created, second.rows_updated) == (0, 0)
         assert db.scalar(select(func.count()).select_from(CatchReport).where(CatchReport.source_type == SourceType.official_record)) == 2
         assert db.scalar(select(func.count()).select_from(OfficialRecordImport)) == 2
 

apps/web/src/lib/api.ts

@@ -17,6 +17,12 @@ export type Spot = { id: string; waterbody_slug: string; waterbody: string; x: n
 export type Catch = { id: string; fish: string; weight_g: number; bait: string | null; player_name: string | null; caught_at: string | null; reported_at: string; retrieve_method: string | null; retrieve_speed: number | null; source_system: string; source_url: string | null };
 export type DictionaryItem = { id: string; slug: string; name_ru: string };
 export type OfficialRecord = { id: string; fish: string; weight_g: number; waterbody: string; bait: string | null; player_name: string | null; record_date: string | null; category: string | null; region: string | null; source_url: string | null; source_system: string };
+export type PaginatedOfficialRecord = {
+  items: OfficialRecord[];
+  total: number;
+  limit: number;
+  offset: number;
+};
 export type PublicObservation = { id: string; source_system: string; source_name: string; source_url: string; fish_name: string; waterbody_name: string; x: number | null; y: number | null; weight_g: number | null; last_seen_at: string; missing_fields: string[]; quality: "incomplete" | "unverified" };
 export type ImportRun = { id: string; started_at: string; finished_at: string | null; status: string; source_url: string; rows_seen: number; rows_created: number; rows_updated: number; error_summary: string | null };
 export type SourceStatus = { source_system: string; name: string; status: "healthy" | "stale" | "temporarily_limited" | "source_changed" | "waiting" | "disabled"; last_started_at: string | null; last_success_at: string | null; observations: number };

apps/web/src/pages/records.astro

@@ -1,18 +1,30 @@
 ---
 import Layout from "../layouts/Layout.astro";
 import SourceBadge from "../components/SourceBadge.astro";
-import { api, kg, type DictionaryItem, type ImportRun, type OfficialRecord } from "../lib/api";
+import { api, kg, type DictionaryItem, type ImportRun, type OfficialRecord, type PaginatedOfficialRecord } from "../lib/api";
 const params = Astro.url.searchParams;
 const fish = params.get("fish") ?? "";
 const waterbody = params.get("waterbody") ?? "";
-let records: OfficialRecord[] = [], runs: ImportRun[] = [], fishes: DictionaryItem[] = [], waterbodies: DictionaryItem[] = [], unavailable = false, showNoIndex = false;
-try { [records, runs, fishes, waterbodies] = await Promise.all([api<OfficialRecord[]>(`/api/v1/records?${new URLSearchParams({ fish, waterbody })}`), api<ImportRun[]>("/api/v1/imports?limit=1"), api<DictionaryItem[]>("/api/v1/fishes"), api<DictionaryItem[]>("/api/v1/waterbodies")]); } catch { unavailable = true; showNoIndex = true; Astro.response.status = 503; Astro.response.headers.set("Retry-After", "60"); }
+const requestedOffset = Number(params.get("offset") ?? 0);
+const offset = Number.isInteger(requestedOffset) && requestedOffset >= 0 ? requestedOffset : 0;
+let items: OfficialRecord[] = [], runs: ImportRun[] = [], fishes: DictionaryItem[] = [], waterbodies: DictionaryItem[] = [];
+let unavailable = false, showNoIndex = false, totalRecords = 0;
+try {
+  const query = new URLSearchParams({ fish, waterbody, limit: "50", offset: String(offset) });
+  const paginated = await api<PaginatedOfficialRecord>(`/api/v1/records?${query}`);
+  items = offset > 0 ? [...items, ...paginated.items] : paginated.items;
+  totalRecords = paginated.total;
+  [runs, fishes, waterbodies] = await Promise.all([api<ImportRun[]>("/api/v1/imports?limit=1"), api<DictionaryItem[]>("/api/v1/fishes"), api<DictionaryItem[]>("/api/v1/waterbodies")]);
+} catch { unavailable = true; showNoIndex = true; Astro.response.status = 503; Astro.response.headers.set("Retry-After", "60"); }
 const last = runs[0];
 ---
 <Layout title="Официальные рекорды Russian Fishing 4 — RF4 Spotter" description="Последние официальные рекорды RF4 по рыбам и водоёмам: вес, приманка, игрок, дата и прямая ссылка на источник." noindex={showNoIndex} errorPage={unavailable}>
   <section class="records-hero"><div><span class="eyebrow">Публичные данные RF4</span><h1>Официальные<br/><em>рекорды</em></h1></div><div class="source-status"><span class:list={["status-dot", last?.status]}></span><strong>{last ? `Импорт: ${last.status}` : "Импорт ещё не запускался"}</strong>{last?.finished_at && <small>{new Date(last.finished_at).toLocaleString("ru-RU")} · {last.rows_seen} строк</small>}</div></section>
   <form class="record-filters" method="get"><label>Рыба<select name="fish"><option value="">Любая рыба</option>{fishes.map(item => <option value={item.slug} selected={fish === item.slug}>{item.name_ru}</option>)}</select></label><label>Водоём<select name="waterbody"><option value="">Все водоёмы</option>{waterbodies.map(item => <option value={item.slug} selected={waterbody === item.slug}>{item.name_ru}</option>)}</select></label><button>Фильтровать</button>{(fish || waterbody) && <a href="/records">Сбросить</a>}</form>
-  <div class="section-heading content-grid"><div><span class="overline">Официальный источник</span><h2>Последние записи</h2></div><span class="result-count">{records.length} показано</span></div>
-  {unavailable ? <div class="state content-grid"><h2>Источник временно недоступен</h2></div> : records.length ? <div class="record-table"><div class="record-row record-head"><span>Рыба</span><span>Вес</span><span>Водоём</span><span>Приманка</span><span>Игрок</span><span>Дата и источник</span></div>{records.map(record => <article class="record-row"><strong data-label="Рыба">{record.fish}</strong><strong data-label="Вес">{kg(record.weight_g)}</strong><span data-label="Водоём">{record.waterbody}</span><span data-label="Приманка">{record.bait ?? "—"}</span><span data-label="Игрок">{record.player_name ?? "—"}</span><span data-label="Дата и источник" class="record-provenance"><time>{record.record_date ? new Date(record.record_date).toLocaleDateString("ru-RU") : "—"}</time><SourceBadge source={record.source_system} href={record.source_url}/></span></article>)}</div> : <div class="state content-grid"><h2>Рекорды ещё не импортированы</h2><p>Для выбранных условий записей пока нет.</p></div>}
+  <div class="section-heading content-grid"><div><span class="overline">Официальный источник</span><h2>Последние записи</h2></div><span class="result-count">{items.length} из {totalRecords} записей</span></div>
+  {unavailable && <div class="state content-grid"><h2>Источник временно недоступен</h2></div>}
+  {!unavailable && items.length && <div class="record-table"><div class="record-row record-head"><span>Рыба</span><span>Вес</span><span>Водоём</span><span>Приманка</span><span>Игрок</span><span>Дата и источник</span></div>{items.map(record => <article class="record-row"><strong data-label="Рыба">{record.fish}</strong><strong data-label="Вес">{kg(record.weight_g)}</strong><span data-label="Водоём">{record.waterbody}</span><span data-label="Приманка">{record.bait ?? "—"}</span><span data-label="Игрок">{record.player_name ?? "—"}</span><span data-label="Дата и источник" class="record-provenance"><time>{record.record_date ? new Date(record.record_date).toLocaleDateString("ru-RU") : "—"}</time><SourceBadge source={record.source_system} href={record.source_url}/></span></article>)}</div>}
+  {!unavailable && items.length && offset + items.length < totalRecords && <a class="load-more" href={`/records?${(() => { const p = new URLSearchParams(params); p.delete("offset"); p.set("offset", String(offset + items.length)); return p.toString(); })()}`}>Показать ещё <span>{offset + items.length} из {totalRecords}</span> ↓</a>}
+  {!unavailable && !items.length && <div class="state content-grid"><h2>Рекорды ещё не импортированы</h2><p>Для выбранных условий записей пока нет.</p></div>}
   <p class="official-note">Источник: <a href="https://rf4game.de/records/region/RU/" rel="noreferrer">официальный сайт Russian Fishing 4</a>. Координаты в официальных таблицах отсутствуют.</p>
 </Layout>

deploy/test-production-bootstrap.sh

@@ -27,6 +27,27 @@ curl -fsS "http://127.0.0.1:$BOOTSTRAP_WEB_PORT/" >/dev/null
 curl -fsS -D - -o /dev/null "http://127.0.0.1:$BOOTSTRAP_API_PORT/health" | grep -qi '^x-frame-options: DENY'
 curl -fsS -D - -o /dev/null "http://127.0.0.1:$BOOTSTRAP_API_PORT/health" | grep -qi '^cross-origin-opener-policy: same-origin'
 
+# A10: Validate Caddy configuration without running the proxy
+echo "Validating Caddy configuration..."
+docker compose --env-file .env.production.example -f compose.production.yaml -f deploy/compose.bootstrap.yaml run --rm --no-deps --entrypoint "caddy adapt --config /etc/caddy/Caddyfile --pretty" proxy >/dev/null 2>&1 || {
+  echo "ERROR: Caddy configuration is invalid" >&2
+  exit 1
+}
+echo "Caddy configuration valid ✓"
+
+# A10: Check scheduler can start without external network
+echo "Validating community scheduler..."
+$compose up --build -d --wait community-scheduler
+# Scheduler runs in a loop, check it's healthy by verifying the process is running
+$compose exec -T community-scheduler python -c "from app.community_scheduler import schedule_interval; print('scheduler module loads OK')" >/dev/null 2>&1 || {
+  echo "ERROR: Community scheduler failed to start" >&2
+  $compose logs --no-color community-scheduler >&2
+  exit 1
+}
+echo "Community scheduler valid ✓"
+# Stop scheduler to free resources
+$compose stop community-scheduler
+
 # A10: Extract Alembic revision ID programmatically, handle multiple heads
 ALEMBIC_HEADS_OUTPUT=$($compose exec -T api alembic heads 2>/dev/null || true)
 # Extract revision IDs (first field before space or '(head)'), handle multiple heads

rf4_research/community_cli.py

@@ -44,12 +44,25 @@ MAX_RESPONSE_BYTES = 5 * 1024 * 1024  # 5 MB
 
 
 def _read_state(state_file: Path) -> dict:
-    """Read state file with shared lock; return empty dict if missing/corrupt."""
+    """Read state file with shared lock; return empty dict if missing/corrupt.
+
+    Validates that the parsed JSON is a flat dict of string->number pairs.
+    Returns {} for missing, corrupt, or structurally invalid files.
+    """
     try:
         with open(state_file, "r") as f:
             fcntl.flock(f, fcntl.LOCK_SH)
             try:
-                return json.loads(f.read())
+                data = json.loads(f.read())
+                if not isinstance(data, dict):
+                    return {}
+                # Validate structure: flat dict of string->number
+                for key, value in data.items():
+                    if not isinstance(key, str):
+                        return {}
+                    if not isinstance(value, (int, float)):
+                        return {}
+                return data
             finally:
                 fcntl.flock(f, fcntl.LOCK_UN)
     except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError):
@@ -76,10 +89,21 @@ def _write_state(state_file: Path, state: dict) -> None:
 
 
 def fetch_site_key(url: str) -> str:
-    """Return a stable cooldown key shared by all endpoints of one site."""
+    """Return a stable cooldown key shared by all endpoints of one site.
+
+    Normalizes hostname to a common key for related domains:
+    - Strips common subdomains (www, download, api, cdn)
+    - Keeps the base domain as the key
+    """
     hostname = (urlsplit(url).hostname or "").lower()
     if hostname.startswith("www."):
         hostname = hostname[4:]
+    if hostname.startswith("download."):
+        hostname = hostname[9:]
+    if hostname.startswith("api."):
+        hostname = hostname[4:]
+    if hostname.startswith("cdn."):
+        hostname = hostname[4:]
     if not hostname:
         raise ValueError("source URL must include a hostname")
     return hostname
@@ -105,6 +129,8 @@ def check_and_reserve(
     Opens state file with exclusive lock, reads state, checks cooldown,
     reserves if allowed — all in one critical section. Uses lockfile
     pattern for cross-process coordination.
+
+    Handles corrupt/missing state files gracefully by treating them as empty.
     """
     if now is None:
         now = time.time()
@@ -115,10 +141,17 @@ def check_and_reserve(
     with open(lock_file, "w") as lf:
         fcntl.flock(lf, fcntl.LOCK_EX)
         try:
-            # Read state under lock
+            # Read state under lock; validate structure
             try:
                 with open(state_file, "r") as sf:
-                    state = json.loads(sf.read()) or {}
+                    raw = sf.read()
+                state = json.loads(raw) or {}
+                if not isinstance(state, dict):
+                    state = {}
+                for key, value in state.items():
+                    if not isinstance(key, str) or not isinstance(value, (int, float)):
+                        state = {}
+                        break
             except (FileNotFoundError, json.JSONDecodeError, ValueError, OSError):
                 state = {}
             # Check cooldown

tests/test_community_cli.py

@@ -28,6 +28,19 @@ def test_fetch_site_key_groups_endpoints_and_normalizes_www() -> None:
     assert fetch_site_key("https://www.rf4-stat.ru/posts/") == "rf4-stat.ru"
 
 
+def test_fetch_site_key_normalizes_common_subdomains() -> None:
+    """A02: download/api/cdn subdomains share the base domain key."""
+    assert fetch_site_key("https://download.rf4db.com/ru/catches") == "rf4db.com"
+    assert fetch_site_key("https://api.rf4db.com/v1/catches") == "rf4db.com"
+    assert fetch_site_key("https://cdn.rf4db.com/assets/img.jpg") == "rf4db.com"
+    assert fetch_site_key("https://www.rf4db.com/") == "rf4db.com"
+    # Base domain stays the same
+    assert fetch_site_key("https://rf4db.com/") == "rf4db.com"
+    # Non-normalized subdomains stay as-is
+    assert fetch_site_key("https://rf4map.ru/point/1") == "rf4map.ru"
+    assert fetch_site_key("https://rf4-posts.com/spots/") == "rf4-posts.com"
+
+
 def test_failed_fetch_still_reserves_site_cooldown(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     state_file = tmp_path / "fetch-state.json"
 
@@ -36,7 +49,8 @@ def test_failed_fetch_still_reserves_site_cooldown(tmp_path: Path, monkeypatch:
 
     monkeypatch.setattr(community_cli, "fetch_html", fail)
     assert community_cli.main(["rf4db", "--state-file", str(state_file)]) == 1
-    assert "download.rf4db.com" in json.loads(state_file.read_text(encoding="utf-8"))
+    # A02: download.rf4db.com normalized to rf4db.com
+    assert "rf4db.com" in json.loads(state_file.read_text(encoding="utf-8"))
 
 
 def test_validate_url_host_rejects_disallowed_hosts() -> None:
@@ -73,15 +87,15 @@ def test_write_state_is_atomic_with_flush(tmp_path: Path) -> None:
     from rf4_research.community_cli import _read_state, _write_state
 
     state_file = tmp_path / "state.json"
-    _write_state(state_file, {"key1": "value1"})
+    _write_state(state_file, {"key1": 1000.0})
     assert state_file.exists()
     assert not (state_file.with_suffix(".tmp")).exists()
     state = _read_state(state_file)
-    assert state == {"key1": "value1"}
+    assert state == {"key1": 1000.0}
 
-    _write_state(state_file, {"key1": "value2", "key2": "value3"})
+    _write_state(state_file, {"key1": 2000.0, "key2": 3000.0})
     state = _read_state(state_file)
-    assert state == {"key1": "value2", "key2": "value3"}
+    assert state == {"key1": 2000.0, "key2": 3000.0}
     assert not (state_file.with_suffix(".tmp")).exists()
 
 
@@ -273,3 +287,40 @@ def test_urljoin_resolves_relative_redirects() -> None:
     assert urljoin("https://rf4-stat.ru/old/path", "new") == "https://rf4-stat.ru/old/new"
     # Absolute URL
     assert urljoin("https://rf4-stat.ru/old", "https://rf4-stat.ru/absolute") == "https://rf4-stat.ru/absolute"
+
+
+# A02: State validation tests
+
+def test_read_state_rejects_corrupt_json(tmp_path: Path) -> None:
+    """A02: Corrupt JSON returns empty dict."""
+    from rf4_research.community_cli import _read_state
+
+    state_file = tmp_path / "corrupt.json"
+    state_file.write_text("not valid json {{{")
+    assert _read_state(state_file) == {}
+
+
+def test_read_state_rejects_invalid_structure(tmp_path: Path) -> None:
+    """A02: Non-dict or non-flat structures return empty dict."""
+    from rf4_research.community_cli import _read_state
+
+    state_file = tmp_path / "invalid.json"
+    # List instead of dict
+    state_file.write_text("[1, 2, 3]")
+    assert _read_state(state_file) == {}
+    # Dict with non-number values
+    state_file.write_text('{"key": "value"}')
+    assert _read_state(state_file) == {}
+    # Dict with nested dict
+    state_file.write_text('{"key": {"nested": true}}')
+    assert _read_state(state_file) == {}
+    # Dict with non-string keys (JSON always has string keys, but validate anyway)
+    state_file.write_text('{"valid": 123}')
+    assert _read_state(state_file) == {"valid": 123}
+
+
+def test_read_state_handles_missing_file() -> None:
+    """A02: Missing state file returns empty dict."""
+    from rf4_research.community_cli import _read_state
+
+    assert _read_state(Path("/nonexistent/state.json")) == {}