feat: add production data retention

This commit is contained in:
ik
2026-09-06 14:37:58 +07:00
parent 0f075469c8
commit 2200336524
10 changed files with 264 additions and 14 deletions
+18 -1
View File
@@ -3,10 +3,14 @@ from __future__ import annotations
import argparse
import json
import sys
from dataclasses import asdict
from .config import settings
from .database import SessionLocal
from .importer import import_records
from .community_importer import stage_observations
from .retention import RetentionPolicy, apply_retention
from .storage import delete_screenshot
def main() -> int:
@@ -19,12 +23,14 @@ def main() -> int:
community = sub.add_parser("stage-community-json")
community.add_argument("--input", default="-", help="JSON array path or - for stdin")
community.add_argument("--limit", type=int, default=500)
cleanup = sub.add_parser("cleanup-retention")
cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run")
args = parser.parse_args()
with SessionLocal() as session:
if args.command == "import-records":
run = import_records(session, url=args.url, region=args.region, category=args.category)
print(f"import {run.status.value}: seen={run.rows_seen} created={run.rows_created} updated={run.rows_updated}")
else:
elif args.command == "stage-community-json":
if not 1 <= args.limit <= 5000:
parser.error("--limit must be between 1 and 5000")
stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
@@ -37,6 +43,17 @@ def main() -> int:
parser.error("input must be a JSON array")
created, updated = stage_observations(session, payload[:args.limit])
print(f"staged: created={created} updated={updated}")
else:
policy = RetentionPolicy(
submission_days=settings.retention_submission_days,
unreviewed_days=settings.retention_unreviewed_days,
approved_personal_days=settings.retention_approved_personal_days,
staging_days=settings.retention_staging_days,
audit_days=settings.retention_audit_days,
published_payload_days=settings.retention_published_payload_days,
)
counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot)
print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False))
return 0