ik/rf4-spotter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add production data retention

Browse Source
This commit is contained in:
ik
2026-09-06 14:37:58 +07:00
parent 0f075469c8
commit 2200336524
10 changed files with 264 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/cli.py
+18 -1
View File
@@ -3,10 +3,14 @@ from __future__ import annotations
import argparse
import json
import sys
from dataclasses import asdict
from .config import settings
from .database import SessionLocal
from .importer import import_records
from .community_importer import stage_observations
from .retention import RetentionPolicy, apply_retention
from .storage import delete_screenshot
def main() -> int:
@@ -19,12 +23,14 @@ def main() -> int:
community = sub.add_parser("stage-community-json")
community.add_argument("--input", default="-", help="JSON array path or - for stdin")
community.add_argument("--limit", type=int, default=500)
cleanup = sub.add_parser("cleanup-retention")
cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run")
args = parser.parse_args()
with SessionLocal() as session:
if args.command == "import-records":
run = import_records(session, url=args.url, region=args.region, category=args.category)
print(f"import {run.status.value}: seen={run.rows_seen} created={run.rows_created} updated={run.rows_updated}")
else:
elif args.command == "stage-community-json":
if not 1 <= args.limit <= 5000:
parser.error("--limit must be between 1 and 5000")
stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
@@ -37,6 +43,17 @@ def main() -> int:
parser.error("input must be a JSON array")
created, updated = stage_observations(session, payload[:args.limit])
print(f"staged: created={created} updated={updated}")
else:
policy = RetentionPolicy(
submission_days=settings.retention_submission_days,
unreviewed_days=settings.retention_unreviewed_days,
approved_personal_days=settings.retention_approved_personal_days,
staging_days=settings.retention_staging_days,
audit_days=settings.retention_audit_days,
published_payload_days=settings.retention_published_payload_days,
)
counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot)
print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False))
return 0
apps/api/app/config.py
+6
View File
@@ -17,6 +17,12 @@ class Settings(BaseSettings):
official_records_category: str = "records"
official_import_required: bool = False
seed_demo_data: bool = True
retention_submission_days: int = Field(default=1, ge=1)
retention_unreviewed_days: int = Field(default=30, ge=7)
retention_approved_personal_days: int = Field(default=180, ge=30)
retention_staging_days: int = Field(default=90, ge=30)
retention_audit_days: int = Field(default=365, ge=90)
retention_published_payload_days: int = Field(default=365, ge=90)
import_interval_seconds: int = Field(default=3600, ge=3600)
rate_limit_secret: str = "change-rate-limit-secret"
log_level: str = "INFO"
apps/api/app/retention.py
+108
View File
@@ -0,0 +1,108 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable
from sqlalchemy import delete, or_, select
from sqlalchemy.orm import Session
from .models import CatchReport, ExternalObservation, ModerationEvent, ModerationStatus, SourceType, SubmissionAttempt
@dataclass(frozen=True)
class RetentionPolicy:
submission_days: int = 1
unreviewed_days: int = 30
approved_personal_days: int = 180
staging_days: int = 90
audit_days: int = 365
published_payload_days: int = 365
def apply_retention(
session: Session, *, policy: RetentionPolicy = RetentionPolicy(),
now: datetime | None = None, dry_run: bool = True,
delete_object: Callable[[str], None] | None = None,
) -> dict[str, int]:
current = now or datetime.now(timezone.utc)
counts = {
"submission_attempts": 0,
"user_reports_anonymized": 0,
"pending_reports_expired": 0,
"screenshots_deleted": 0,
"staging_observations_deleted": 0,
"published_payloads_cleared": 0,
"moderation_events_deleted": 0,
}
attempts = list(session.scalars(select(SubmissionAttempt.id).where(
SubmissionAttempt.created_at < current - timedelta(days=policy.submission_days),
)))
counts["submission_attempts"] = len(attempts)
candidate_cutoff = current - timedelta(days=min(policy.unreviewed_days, policy.approved_personal_days))
candidates = list(session.scalars(select(CatchReport).where(
CatchReport.source_type == SourceType.user,
CatchReport.reported_at < candidate_cutoff,
or_(
CatchReport.player_name.is_not(None), CatchReport.source_url.is_not(None),
CatchReport.raw_payload.is_not(None), CatchReport.screenshot_key.is_not(None),
CatchReport.screenshot_upload_token_hash.is_not(None),
),
)))
reports = [item for item in candidates if item.reported_at.replace(tzinfo=item.reported_at.tzinfo or timezone.utc) < current - timedelta(
days=policy.approved_personal_days if item.moderation_status == ModerationStatus.approved else policy.unreviewed_days,
)]
counts["user_reports_anonymized"] = len(reports)
counts["screenshots_deleted"] = sum(bool(item.screenshot_key) for item in reports)
counts["pending_reports_expired"] = sum(item.moderation_status == ModerationStatus.pending for item in reports)
stale = list(session.scalars(select(ExternalObservation).where(
ExternalObservation.catch_report_id.is_(None),
ExternalObservation.status != "published",
ExternalObservation.last_seen_at < current - timedelta(days=policy.staging_days),
)))
counts["staging_observations_deleted"] = len(stale)
published = list(session.scalars(select(ExternalObservation).where(
ExternalObservation.status == "published",
ExternalObservation.last_seen_at < current - timedelta(days=policy.published_payload_days),
)))
published = [item for item in published if item.payload]
counts["published_payloads_cleared"] = len(published)
events = list(session.scalars(select(ModerationEvent.id).where(
ModerationEvent.created_at < current - timedelta(days=policy.audit_days),
)))
counts["moderation_events_deleted"] = len(events)
if dry_run:
return counts
if counts["screenshots_deleted"] and delete_object is None:
raise ValueError("delete_object is required when retained screenshots must be deleted")
for report in reports:
if report.screenshot_key and delete_object:
delete_object(report.screenshot_key)
if report.moderation_status == ModerationStatus.pending:
session.add(ModerationEvent(
catch_report=report, created_at=current,
previous_status=ModerationStatus.pending, new_status=ModerationStatus.rejected,
moderator="retention-policy", reason="pending report retention period expired",
))
report.moderation_status = ModerationStatus.rejected
report.player_name = None
report.source_url = None
report.raw_payload = None
report.screenshot_key = None
report.screenshot_upload_token_hash = None
for observation in stale:
session.delete(observation)
for observation in published:
observation.payload = {}
if attempts:
session.execute(delete(SubmissionAttempt).where(SubmissionAttempt.id.in_(attempts)))
if events:
session.execute(delete(ModerationEvent).where(ModerationEvent.id.in_(events)))
session.commit()
return counts
apps/api/tests/test_retention.py
+59
View File
@@ -0,0 +1,59 @@
from datetime import datetime, timedelta, timezone
from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session
from app.database import Base
from app.models import CatchReport, DataSource, ExternalObservation, Fish, ModerationEvent, ModerationStatus, SourceType, SubmissionAttempt, Waterbody
from app.retention import apply_retention
NOW = datetime(2026, 9, 6, tzinfo=timezone.utc)
def test_retention_dry_run_then_apply() -> None:
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
with Session(engine) as db:
fish = Fish(slug="pike", name_ru="Щука", trophy_weight_g=10_000)
waterbody = Waterbody(slug="lake", name_ru="Озеро", unlock_level=1)
source = DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=60, enabled=False)
pending = CatchReport(
fish=fish, waterbody=waterbody, weight_g=1000, reported_at=NOW - timedelta(days=31),
player_name="Private", source_type=SourceType.user, source_confidence=60,
moderation_status=ModerationStatus.pending, raw_payload={"comment": "private"},
screenshot_key="reports/old.jpg", screenshot_upload_token_hash="a" * 64,
)
approved = CatchReport(
fish=fish, waterbody=waterbody, weight_g=2000, reported_at=NOW - timedelta(days=181),
player_name="Old winner", source_type=SourceType.user, source_confidence=60,
moderation_status=ModerationStatus.approved, raw_payload={"comment": "old"},
)
fresh = CatchReport(
fish=fish, waterbody=waterbody, weight_g=3000, reported_at=NOW - timedelta(days=10),
player_name="Fresh", source_type=SourceType.user, source_confidence=60,
moderation_status=ModerationStatus.pending, raw_payload={"comment": "fresh"},
)
db.add_all([source, pending, approved, fresh])
db.flush()
db.add(SubmissionAttempt(client_hash="x" * 64, created_at=NOW - timedelta(days=2)))
db.add(ModerationEvent(catch_report=approved, created_at=NOW - timedelta(days=366), previous_status=ModerationStatus.pending, new_status=ModerationStatus.approved, moderator="admin"))
db.add(ExternalObservation(source=source, source_external_id="stale", source_url="https://rf4db.com/1", fish_name="Щука", waterbody_name="Озеро", first_seen_at=NOW - timedelta(days=100), last_seen_at=NOW - timedelta(days=100), status="staged", payload={"raw": True}))
db.add(ExternalObservation(source=source, source_external_id="published", source_url="https://rf4db.com/2", fish_name="Щука", waterbody_name="Озеро", first_seen_at=NOW - timedelta(days=400), last_seen_at=NOW - timedelta(days=400), status="published", payload={"raw": True}, catch_report=approved))
db.commit()
preview = apply_retention(db, now=NOW)
assert preview == {"submission_attempts": 1, "user_reports_anonymized": 2, "pending_reports_expired": 1, "screenshots_deleted": 1, "staging_observations_deleted": 1, "published_payloads_cleared": 1, "moderation_events_deleted": 1}
assert db.get(CatchReport, pending.id).player_name == "Private"
deleted: list[str] = []
assert apply_retention(db, now=NOW, dry_run=False, delete_object=deleted.append) == preview
assert deleted == ["reports/old.jpg"]
assert db.get(CatchReport, pending.id).moderation_status == ModerationStatus.rejected
assert db.get(CatchReport, pending.id).player_name is None
assert db.get(CatchReport, approved.id).raw_payload is None
assert db.get(CatchReport, fresh.id).player_name == "Fresh"
assert db.scalar(select(func.count()).select_from(SubmissionAttempt)) == 0
assert db.scalar(select(func.count()).select_from(ExternalObservation)) == 1
assert db.scalar(select(ExternalObservation)).payload == {}
assert db.scalar(select(func.count()).select_from(ModerationEvent)) == 1