ik
779d554057
- Infrastructure (DB/MinIO) blocks readiness; imports are diagnostic only - Per-source community scheduler health with backoff detection - Stale/failed imports never block /ready — scheduler can recover them - Add 'blocking: false' to all import components - 4 new tests: per-source health, backoff detection, stale/failed non-blocking - 108 Python tests pass
116 lines
4.8 KiB
Python
116 lines
4.8 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, select, text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport
|
|
|
|
|
|
def readiness_report(
|
|
session: Session, s3: Any, *, import_required: bool,
|
|
import_interval_seconds: int, community_import_interval_seconds: int = 1800,
|
|
now: datetime | None = None,
|
|
) -> tuple[bool, dict[str, dict[str, object]]]:
|
|
"""A01: Separate infrastructure readiness from import health diagnostics.
|
|
|
|
Infrastructure (DB, MinIO) blocks readiness. Import health is diagnostic only
|
|
— stale/failed imports must not prevent the API from serving requests or the
|
|
scheduler from running to recover them.
|
|
"""
|
|
current = now or datetime.now(timezone.utc)
|
|
components: dict[str, dict[str, object]] = {}
|
|
ready = True
|
|
|
|
try:
|
|
session.execute(text("SELECT 1"))
|
|
components["postgresql"] = {"status": "ready"}
|
|
except Exception:
|
|
components["postgresql"] = {"status": "unavailable"}
|
|
ready = False
|
|
|
|
try:
|
|
s3.list_buckets()
|
|
components["minio"] = {"status": "ready"}
|
|
except Exception:
|
|
components["minio"] = {"status": "unavailable"}
|
|
ready = False
|
|
|
|
# Official import health — diagnostic only, never blocks readiness (A01)
|
|
try:
|
|
latest = session.scalar(select(OfficialRecordImport).order_by(
|
|
OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc(),
|
|
).limit(1))
|
|
if not import_required:
|
|
components["official_import"] = {
|
|
"status": "optional",
|
|
"last_run_status": latest.status.value if latest else None,
|
|
"blocking": False,
|
|
}
|
|
elif latest is None:
|
|
components["official_import"] = {
|
|
"status": "not_run",
|
|
"blocking": False,
|
|
}
|
|
else:
|
|
started = latest.started_at if latest.started_at.tzinfo else latest.started_at.replace(tzinfo=timezone.utc)
|
|
stale = started < current - timedelta(seconds=import_interval_seconds * 2)
|
|
healthy = latest.status == ImportStatus.success and not stale
|
|
components["official_import"] = {
|
|
"status": "ready" if healthy else ("stale" if stale else latest.status.value),
|
|
"last_run_status": latest.status.value,
|
|
"last_started_at": started.isoformat(),
|
|
"blocking": False,
|
|
}
|
|
except Exception:
|
|
components["official_import"] = {
|
|
"status": "unknown",
|
|
"blocking": False,
|
|
}
|
|
|
|
# Community scheduler health — diagnostic only, never blocks readiness (A01)
|
|
# Track per-source health with rotation, backoff, last success, and stalled attempts
|
|
try:
|
|
enabled_sources = list(session.scalars(
|
|
select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.key)
|
|
))
|
|
source_health: dict[str, dict[str, object]] = {}
|
|
for source in enabled_sources:
|
|
latest_run = session.scalar(
|
|
select(CommunityImportRun)
|
|
.where(CommunityImportRun.source_system == source.key)
|
|
.order_by(CommunityImportRun.started_at.desc())
|
|
.limit(1)
|
|
)
|
|
if latest_run is None:
|
|
source_health[source.key] = {"status": "not_started", "blocking": False}
|
|
continue
|
|
started = latest_run.started_at
|
|
if started.tzinfo is None:
|
|
started = started.replace(tzinfo=timezone.utc)
|
|
stale = started < current - timedelta(seconds=community_import_interval_seconds * 2)
|
|
healthy = latest_run.status == "success" and not stale
|
|
# Count recent failures for backoff detection
|
|
recent_failures = session.scalar(
|
|
select(func.count()).select_from(CommunityImportRun)
|
|
.where(
|
|
CommunityImportRun.source_system == source.key,
|
|
CommunityImportRun.status == "failed",
|
|
CommunityImportRun.started_at >= current - timedelta(hours=24),
|
|
)
|
|
) or 0
|
|
source_health[source.key] = {
|
|
"status": "ready" if healthy else ("stale" if stale else latest_run.status),
|
|
"last_started_at": started.isoformat(),
|
|
"recent_failures_24h": recent_failures,
|
|
"backoff_recommended": recent_failures >= 5,
|
|
"blocking": False,
|
|
}
|
|
components["community_scheduler"] = {"status": "ready", "sources": source_health}
|
|
except Exception:
|
|
components["community_scheduler"] = {"status": "unknown", "sources": {}}
|
|
|
|
return ready, components
|