140 lines
5.9 KiB
Python
140 lines
5.9 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, select, text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .config import settings
|
|
from .models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport
|
|
|
|
|
|
def readiness_report(
|
|
session: Session, s3: Any, *, import_required: bool,
|
|
import_interval_seconds: int, community_import_interval_seconds: int = 1800,
|
|
now: datetime | None = None,
|
|
) -> tuple[bool, dict[str, dict[str, object]]]:
|
|
"""A01: Separate infrastructure readiness from import health diagnostics.
|
|
|
|
Infrastructure (DB, MinIO) blocks readiness. Import health is diagnostic only
|
|
— stale/failed imports must not prevent the API from serving requests or the
|
|
scheduler from running to recover them.
|
|
"""
|
|
current = now or datetime.now(timezone.utc)
|
|
components: dict[str, dict[str, object]] = {}
|
|
ready = True
|
|
|
|
try:
|
|
session.execute(text("SELECT 1"))
|
|
components["postgresql"] = {"status": "ready"}
|
|
except Exception:
|
|
components["postgresql"] = {"status": "unavailable"}
|
|
ready = False
|
|
|
|
try:
|
|
s3.head_bucket(Bucket=settings.s3_bucket)
|
|
components["minio"] = {"status": "ready"}
|
|
except Exception:
|
|
components["minio"] = {"status": "unavailable"}
|
|
ready = False
|
|
|
|
# Official import health — diagnostic only, never blocks readiness (A01)
|
|
try:
|
|
latest = session.scalar(select(OfficialRecordImport).order_by(
|
|
OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc(),
|
|
).limit(1))
|
|
if not import_required:
|
|
components["official_import"] = {
|
|
"status": "optional",
|
|
"last_run_status": latest.status.value if latest else None,
|
|
"blocking": False,
|
|
}
|
|
elif latest is None:
|
|
components["official_import"] = {
|
|
"status": "not_run",
|
|
"blocking": False,
|
|
}
|
|
else:
|
|
started = latest.started_at if latest.started_at.tzinfo else latest.started_at.replace(tzinfo=timezone.utc)
|
|
stale = started < current - timedelta(seconds=import_interval_seconds * 2)
|
|
healthy = latest.status == ImportStatus.success and not stale
|
|
components["official_import"] = {
|
|
"status": "ready" if healthy else ("stale" if stale else latest.status.value),
|
|
"last_run_status": latest.status.value,
|
|
"last_started_at": started.isoformat(),
|
|
"blocking": False,
|
|
}
|
|
except Exception:
|
|
components["official_import"] = {
|
|
"status": "unknown",
|
|
"blocking": False,
|
|
}
|
|
|
|
# Community scheduler health — diagnostic only, never blocks readiness (A01)
|
|
# Track per-source health with rotation, backoff, last success, and stalled attempts
|
|
# Overall status reflects worst-case source health (success of one does not mask failure of another)
|
|
try:
|
|
enabled_sources = list(session.scalars(
|
|
select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.key)
|
|
))
|
|
source_health: dict[str, dict[str, object]] = {}
|
|
has_any_failure = False
|
|
has_any_success = False
|
|
has_any_stale = False
|
|
has_any_running = False
|
|
for source in enabled_sources:
|
|
latest_run = session.scalar(
|
|
select(CommunityImportRun)
|
|
.where(CommunityImportRun.source_system == source.key)
|
|
.order_by(CommunityImportRun.started_at.desc())
|
|
.limit(1)
|
|
)
|
|
if latest_run is None:
|
|
source_health[source.key] = {"status": "not_started", "blocking": False}
|
|
continue
|
|
started = latest_run.started_at
|
|
if started.tzinfo is None:
|
|
started = started.replace(tzinfo=timezone.utc)
|
|
stale = started < current - timedelta(seconds=community_import_interval_seconds * 2)
|
|
healthy = latest_run.status == "success" and not stale
|
|
# Count recent failures for backoff detection
|
|
recent_failures = session.scalar(
|
|
select(func.count()).select_from(CommunityImportRun)
|
|
.where(
|
|
CommunityImportRun.source_system == source.key,
|
|
CommunityImportRun.status == "failed",
|
|
CommunityImportRun.started_at >= current - timedelta(hours=24),
|
|
)
|
|
) or 0
|
|
source_health[source.key] = {
|
|
"status": "ready" if healthy else ("stale" if stale else latest_run.status),
|
|
"last_started_at": started.isoformat(),
|
|
"recent_failures_24h": recent_failures,
|
|
"backoff_recommended": recent_failures >= 5,
|
|
"blocking": False,
|
|
}
|
|
if healthy:
|
|
has_any_success = True
|
|
elif latest_run.status == "failed":
|
|
has_any_failure = True
|
|
elif stale:
|
|
has_any_stale = True
|
|
elif latest_run.status == "running":
|
|
has_any_running = True
|
|
# Overall status: never mask failures with success of another source
|
|
# "degraded" if any source failed/stale/running
|
|
# "ready" only when ALL enabled sources are healthy
|
|
# "not_started" when no sources are enabled
|
|
if has_any_failure or has_any_stale or has_any_running:
|
|
scheduler_status = "degraded"
|
|
elif has_any_success and len(source_health) > 0:
|
|
scheduler_status = "ready"
|
|
else:
|
|
scheduler_status = "not_started"
|
|
components["community_scheduler"] = {"status": scheduler_status, "sources": source_health}
|
|
except Exception:
|
|
components["community_scheduler"] = {"status": "unknown", "sources": {}}
|
|
|
|
return ready, components
|