feat: add production data retention

2026-09-06 14:37:58 +07:00
parent 0f075469c8
commit 2200336524
10 changed files with 264 additions and 14 deletions
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import Callable
+
+from sqlalchemy import delete, or_, select
+from sqlalchemy.orm import Session
+
+from .models import CatchReport, ExternalObservation, ModerationEvent, ModerationStatus, SourceType, SubmissionAttempt
+
+
+@dataclass(frozen=True)
+class RetentionPolicy:
+    submission_days: int = 1
+    unreviewed_days: int = 30
+    approved_personal_days: int = 180
+    staging_days: int = 90
+    audit_days: int = 365
+    published_payload_days: int = 365
+
+
+def apply_retention(
+    session: Session, *, policy: RetentionPolicy = RetentionPolicy(),
+    now: datetime | None = None, dry_run: bool = True,
+    delete_object: Callable[[str], None] | None = None,
+) -> dict[str, int]:
+    current = now or datetime.now(timezone.utc)
+    counts = {
+        "submission_attempts": 0,
+        "user_reports_anonymized": 0,
+        "pending_reports_expired": 0,
+        "screenshots_deleted": 0,
+        "staging_observations_deleted": 0,
+        "published_payloads_cleared": 0,
+        "moderation_events_deleted": 0,
+    }
+
+    attempts = list(session.scalars(select(SubmissionAttempt.id).where(
+        SubmissionAttempt.created_at < current - timedelta(days=policy.submission_days),
+    )))
+    counts["submission_attempts"] = len(attempts)
+
+    candidate_cutoff = current - timedelta(days=min(policy.unreviewed_days, policy.approved_personal_days))
+    candidates = list(session.scalars(select(CatchReport).where(
+        CatchReport.source_type == SourceType.user,
+        CatchReport.reported_at < candidate_cutoff,
+        or_(
+            CatchReport.player_name.is_not(None), CatchReport.source_url.is_not(None),
+            CatchReport.raw_payload.is_not(None), CatchReport.screenshot_key.is_not(None),
+            CatchReport.screenshot_upload_token_hash.is_not(None),
+        ),
+    )))
+    reports = [item for item in candidates if item.reported_at.replace(tzinfo=item.reported_at.tzinfo or timezone.utc) < current - timedelta(
+        days=policy.approved_personal_days if item.moderation_status == ModerationStatus.approved else policy.unreviewed_days,
+    )]
+    counts["user_reports_anonymized"] = len(reports)
+    counts["screenshots_deleted"] = sum(bool(item.screenshot_key) for item in reports)
+    counts["pending_reports_expired"] = sum(item.moderation_status == ModerationStatus.pending for item in reports)
+
+    stale = list(session.scalars(select(ExternalObservation).where(
+        ExternalObservation.catch_report_id.is_(None),
+        ExternalObservation.status != "published",
+        ExternalObservation.last_seen_at < current - timedelta(days=policy.staging_days),
+    )))
+    counts["staging_observations_deleted"] = len(stale)
+    published = list(session.scalars(select(ExternalObservation).where(
+        ExternalObservation.status == "published",
+        ExternalObservation.last_seen_at < current - timedelta(days=policy.published_payload_days),
+    )))
+    published = [item for item in published if item.payload]
+    counts["published_payloads_cleared"] = len(published)
+    events = list(session.scalars(select(ModerationEvent.id).where(
+        ModerationEvent.created_at < current - timedelta(days=policy.audit_days),
+    )))
+    counts["moderation_events_deleted"] = len(events)
+
+    if dry_run:
+        return counts
+
+    if counts["screenshots_deleted"] and delete_object is None:
+        raise ValueError("delete_object is required when retained screenshots must be deleted")
+
+    for report in reports:
+        if report.screenshot_key and delete_object:
+            delete_object(report.screenshot_key)
+        if report.moderation_status == ModerationStatus.pending:
+            session.add(ModerationEvent(
+                catch_report=report, created_at=current,
+                previous_status=ModerationStatus.pending, new_status=ModerationStatus.rejected,
+                moderator="retention-policy", reason="pending report retention period expired",
+            ))
+            report.moderation_status = ModerationStatus.rejected
+        report.player_name = None
+        report.source_url = None
+        report.raw_payload = None
+        report.screenshot_key = None
+        report.screenshot_upload_token_hash = None
+    for observation in stale:
+        session.delete(observation)
+    for observation in published:
+        observation.payload = {}
+    if attempts:
+        session.execute(delete(SubmissionAttempt).where(SubmissionAttempt.id.in_(attempts)))
+    if events:
+        session.execute(delete(ModerationEvent).where(ModerationEvent.id.in_(events)))
+    session.commit()
+    return counts