feat: add production data retention

apps/api/app/cli.py

@@ -3,10 +3,14 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+from dataclasses import asdict
 
+from .config import settings
 from .database import SessionLocal
 from .importer import import_records
 from .community_importer import stage_observations
+from .retention import RetentionPolicy, apply_retention
+from .storage import delete_screenshot
 
 
 def main() -> int:
@@ -19,12 +23,14 @@ def main() -> int:
     community = sub.add_parser("stage-community-json")
     community.add_argument("--input", default="-", help="JSON array path or - for stdin")
     community.add_argument("--limit", type=int, default=500)
+    cleanup = sub.add_parser("cleanup-retention")
+    cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run")
     args = parser.parse_args()
     with SessionLocal() as session:
         if args.command == "import-records":
             run = import_records(session, url=args.url, region=args.region, category=args.category)
             print(f"import {run.status.value}: seen={run.rows_seen} created={run.rows_created} updated={run.rows_updated}")
-        else:
+        elif args.command == "stage-community-json":
             if not 1 <= args.limit <= 5000:
                 parser.error("--limit must be between 1 and 5000")
             stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
@@ -37,6 +43,17 @@ def main() -> int:
                 parser.error("input must be a JSON array")
             created, updated = stage_observations(session, payload[:args.limit])
             print(f"staged: created={created} updated={updated}")
+        else:
+            policy = RetentionPolicy(
+                submission_days=settings.retention_submission_days,
+                unreviewed_days=settings.retention_unreviewed_days,
+                approved_personal_days=settings.retention_approved_personal_days,
+                staging_days=settings.retention_staging_days,
+                audit_days=settings.retention_audit_days,
+                published_payload_days=settings.retention_published_payload_days,
+            )
+            counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot)
+            print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False))
     return 0
 
 

apps/api/app/config.py

@@ -17,6 +17,12 @@ class Settings(BaseSettings):
     official_records_category: str = "records"
     official_import_required: bool = False
     seed_demo_data: bool = True
+    retention_submission_days: int = Field(default=1, ge=1)
+    retention_unreviewed_days: int = Field(default=30, ge=7)
+    retention_approved_personal_days: int = Field(default=180, ge=30)
+    retention_staging_days: int = Field(default=90, ge=30)
+    retention_audit_days: int = Field(default=365, ge=90)
+    retention_published_payload_days: int = Field(default=365, ge=90)
     import_interval_seconds: int = Field(default=3600, ge=3600)
     rate_limit_secret: str = "change-rate-limit-secret"
     log_level: str = "INFO"

apps/api/app/retention.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Callable
+
+from sqlalchemy import delete, or_, select
+from sqlalchemy.orm import Session
+
+from .models import CatchReport, ExternalObservation, ModerationEvent, ModerationStatus, SourceType, SubmissionAttempt
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+    submission_days: int = 1
+    unreviewed_days: int = 30
+    approved_personal_days: int = 180
+    staging_days: int = 90
+    audit_days: int = 365
+    published_payload_days: int = 365
+
+
+def apply_retention(
+    session: Session, *, policy: RetentionPolicy = RetentionPolicy(),
+    now: datetime | None = None, dry_run: bool = True,
+    delete_object: Callable[[str], None] | None = None,
+) -> dict[str, int]:
+    current = now or datetime.now(timezone.utc)
+    counts = {
+        "submission_attempts": 0,
+        "user_reports_anonymized": 0,
+        "pending_reports_expired": 0,
+        "screenshots_deleted": 0,
+        "staging_observations_deleted": 0,
+        "published_payloads_cleared": 0,
+        "moderation_events_deleted": 0,
+    }
+
+    attempts = list(session.scalars(select(SubmissionAttempt.id).where(
+        SubmissionAttempt.created_at < current - timedelta(days=policy.submission_days),
+    )))
+    counts["submission_attempts"] = len(attempts)
+
+    candidate_cutoff = current - timedelta(days=min(policy.unreviewed_days, policy.approved_personal_days))
+    candidates = list(session.scalars(select(CatchReport).where(
+        CatchReport.source_type == SourceType.user,
+        CatchReport.reported_at < candidate_cutoff,
+        or_(
+            CatchReport.player_name.is_not(None), CatchReport.source_url.is_not(None),
+            CatchReport.raw_payload.is_not(None), CatchReport.screenshot_key.is_not(None),
+            CatchReport.screenshot_upload_token_hash.is_not(None),
+        ),
+    )))
+    reports = [item for item in candidates if item.reported_at.replace(tzinfo=item.reported_at.tzinfo or timezone.utc) < current - timedelta(
+        days=policy.approved_personal_days if item.moderation_status == ModerationStatus.approved else policy.unreviewed_days,
+    )]
+    counts["user_reports_anonymized"] = len(reports)
+    counts["screenshots_deleted"] = sum(bool(item.screenshot_key) for item in reports)
+    counts["pending_reports_expired"] = sum(item.moderation_status == ModerationStatus.pending for item in reports)
+
+    stale = list(session.scalars(select(ExternalObservation).where(
+        ExternalObservation.catch_report_id.is_(None),
+        ExternalObservation.status != "published",
+        ExternalObservation.last_seen_at < current - timedelta(days=policy.staging_days),
+    )))
+    counts["staging_observations_deleted"] = len(stale)
+    published = list(session.scalars(select(ExternalObservation).where(
+        ExternalObservation.status == "published",
+        ExternalObservation.last_seen_at < current - timedelta(days=policy.published_payload_days),
+    )))
+    published = [item for item in published if item.payload]
+    counts["published_payloads_cleared"] = len(published)
+    events = list(session.scalars(select(ModerationEvent.id).where(
+        ModerationEvent.created_at < current - timedelta(days=policy.audit_days),
+    )))
+    counts["moderation_events_deleted"] = len(events)
+
+    if dry_run:
+        return counts
+
+    if counts["screenshots_deleted"] and delete_object is None:
+        raise ValueError("delete_object is required when retained screenshots must be deleted")
+
+    for report in reports:
+        if report.screenshot_key and delete_object:
+            delete_object(report.screenshot_key)
+        if report.moderation_status == ModerationStatus.pending:
+            session.add(ModerationEvent(
+                catch_report=report, created_at=current,
+                previous_status=ModerationStatus.pending, new_status=ModerationStatus.rejected,
+                moderator="retention-policy", reason="pending report retention period expired",
+            ))
+            report.moderation_status = ModerationStatus.rejected
+        report.player_name = None
+        report.source_url = None
+        report.raw_payload = None
+        report.screenshot_key = None
+        report.screenshot_upload_token_hash = None
+    for observation in stale:
+        session.delete(observation)
+    for observation in published:
+        observation.payload = {}
+    if attempts:
+        session.execute(delete(SubmissionAttempt).where(SubmissionAttempt.id.in_(attempts)))
+    if events:
+        session.execute(delete(ModerationEvent).where(ModerationEvent.id.in_(events)))
+    session.commit()
+    return counts