rf4-spotter/apps/api/app/readiness.py

from __future__ import annotations

from datetime import datetime, timedelta, timezone
from typing import Any

from sqlalchemy import func, select, text
from sqlalchemy.orm import Session

from .models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport


def readiness_report(
    session: Session, s3: Any, *, import_required: bool,
    import_interval_seconds: int, community_import_interval_seconds: int = 1800,
    now: datetime | None = None,
) -> tuple[bool, dict[str, dict[str, object]]]:
    """A01: Separate infrastructure readiness from import health diagnostics.

    Infrastructure (DB, MinIO) blocks readiness. Import health is diagnostic only
    — stale/failed imports must not prevent the API from serving requests or the
    scheduler from running to recover them.
    """
    current = now or datetime.now(timezone.utc)
    components: dict[str, dict[str, object]] = {}
    ready = True

    try:
        session.execute(text("SELECT 1"))
        components["postgresql"] = {"status": "ready"}
    except Exception:
        components["postgresql"] = {"status": "unavailable"}
        ready = False

    try:
        s3.list_buckets()
        components["minio"] = {"status": "ready"}
    except Exception:
        components["minio"] = {"status": "unavailable"}
        ready = False

    # Official import health — diagnostic only, never blocks readiness (A01)
    try:
        latest = session.scalar(select(OfficialRecordImport).order_by(
            OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc(),
        ).limit(1))
        if not import_required:
            components["official_import"] = {
                "status": "optional",
                "last_run_status": latest.status.value if latest else None,
                "blocking": False,
            }
        elif latest is None:
            components["official_import"] = {
                "status": "not_run",
                "blocking": False,
            }
        else:
            started = latest.started_at if latest.started_at.tzinfo else latest.started_at.replace(tzinfo=timezone.utc)
            stale = started < current - timedelta(seconds=import_interval_seconds * 2)
            healthy = latest.status == ImportStatus.success and not stale
            components["official_import"] = {
                "status": "ready" if healthy else ("stale" if stale else latest.status.value),
                "last_run_status": latest.status.value,
                "last_started_at": started.isoformat(),
                "blocking": False,
            }
    except Exception:
        components["official_import"] = {
            "status": "unknown",
            "blocking": False,
        }

    # Community scheduler health — diagnostic only, never blocks readiness (A01)
    # Track per-source health with rotation, backoff, last success, and stalled attempts
    # Overall status reflects worst-case source health (success of one does not mask failure of another)
    try:
        enabled_sources = list(session.scalars(
            select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.key)
        ))
        source_health: dict[str, dict[str, object]] = {}
        has_any_failure = False
        has_any_success = False
        for source in enabled_sources:
            latest_run = session.scalar(
                select(CommunityImportRun)
                .where(CommunityImportRun.source_system == source.key)
                .order_by(CommunityImportRun.started_at.desc())
                .limit(1)
            )
            if latest_run is None:
                source_health[source.key] = {"status": "not_started", "blocking": False}
                continue
            started = latest_run.started_at
            if started.tzinfo is None:
                started = started.replace(tzinfo=timezone.utc)
            stale = started < current - timedelta(seconds=community_import_interval_seconds * 2)
            healthy = latest_run.status == "success" and not stale
            # Count recent failures for backoff detection
            recent_failures = session.scalar(
                select(func.count()).select_from(CommunityImportRun)
                .where(
                    CommunityImportRun.source_system == source.key,
                    CommunityImportRun.status == "failed",
                    CommunityImportRun.started_at >= current - timedelta(hours=24),
                )
            ) or 0
            source_health[source.key] = {
                "status": "ready" if healthy else ("stale" if stale else latest_run.status),
                "last_started_at": started.isoformat(),
                "recent_failures_24h": recent_failures,
                "backoff_recommended": recent_failures >= 5,
                "blocking": False,
            }
            if healthy:
                has_any_success = True
            elif latest_run.status == "failed":
                has_any_failure = True
        # Overall status: "degraded" if any source failed, "ready" if all healthy, "stale" if no failures but stale
        if has_any_failure:
            scheduler_status = "degraded"
        elif has_any_success:
            scheduler_status = "ready"
        else:
            scheduler_status = "stale" if enabled_sources else "not_started"
        components["community_scheduler"] = {"status": scheduler_status, "sources": source_health}
    except Exception:
        components["community_scheduler"] = {"status": "unknown", "sources": {}}

    return ready, components