feat: add production data retention
This commit is contained in:
+18
-1
@@ -3,10 +3,14 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
|
||||
from .config import settings
|
||||
from .database import SessionLocal
|
||||
from .importer import import_records
|
||||
from .community_importer import stage_observations
|
||||
from .retention import RetentionPolicy, apply_retention
|
||||
from .storage import delete_screenshot
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -19,12 +23,14 @@ def main() -> int:
|
||||
community = sub.add_parser("stage-community-json")
|
||||
community.add_argument("--input", default="-", help="JSON array path or - for stdin")
|
||||
community.add_argument("--limit", type=int, default=500)
|
||||
cleanup = sub.add_parser("cleanup-retention")
|
||||
cleanup.add_argument("--apply", action="store_true", help="apply changes; default is dry-run")
|
||||
args = parser.parse_args()
|
||||
with SessionLocal() as session:
|
||||
if args.command == "import-records":
|
||||
run = import_records(session, url=args.url, region=args.region, category=args.category)
|
||||
print(f"import {run.status.value}: seen={run.rows_seen} created={run.rows_created} updated={run.rows_updated}")
|
||||
else:
|
||||
elif args.command == "stage-community-json":
|
||||
if not 1 <= args.limit <= 5000:
|
||||
parser.error("--limit must be between 1 and 5000")
|
||||
stream = sys.stdin if args.input == "-" else open(args.input, encoding="utf-8")
|
||||
@@ -37,6 +43,17 @@ def main() -> int:
|
||||
parser.error("input must be a JSON array")
|
||||
created, updated = stage_observations(session, payload[:args.limit])
|
||||
print(f"staged: created={created} updated={updated}")
|
||||
else:
|
||||
policy = RetentionPolicy(
|
||||
submission_days=settings.retention_submission_days,
|
||||
unreviewed_days=settings.retention_unreviewed_days,
|
||||
approved_personal_days=settings.retention_approved_personal_days,
|
||||
staging_days=settings.retention_staging_days,
|
||||
audit_days=settings.retention_audit_days,
|
||||
published_payload_days=settings.retention_published_payload_days,
|
||||
)
|
||||
counts = apply_retention(session, policy=policy, dry_run=not args.apply, delete_object=delete_screenshot)
|
||||
print(json.dumps({"mode": "apply" if args.apply else "dry-run", "policy": asdict(policy), "counts": counts}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user