From 779d5540574060c342b2cc4db35646cfd545157d Mon Sep 17 00:00:00 2001 From: IK Date: Thu, 10 Sep 2026 06:10:01 +0700 Subject: [PATCH] A01: Separate API readiness from import health diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Infrastructure (DB/MinIO) blocks readiness; imports are diagnostic only - Per-source community scheduler health with backoff detection - Stale/failed imports never block /ready — scheduler can recover them - Add 'blocking: false' to all import components - 4 new tests: per-source health, backoff detection, stale/failed non-blocking - 108 Python tests pass --- apps/api/app/readiness.py | 77 +++++++++++++++++++++---------- apps/api/tests/test_readiness.py | 78 +++++++++++++++++++++++++++----- 2 files changed, 120 insertions(+), 35 deletions(-) diff --git a/apps/api/app/readiness.py b/apps/api/app/readiness.py index 1340ceb..2f386c5 100644 --- a/apps/api/app/readiness.py +++ b/apps/api/app/readiness.py @@ -3,10 +3,10 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone from typing import Any -from sqlalchemy import select, text +from sqlalchemy import func, select, text from sqlalchemy.orm import Session -from .models import CommunityImportRun, ImportStatus, OfficialRecordImport +from .models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport def readiness_report( @@ -14,6 +14,12 @@ def readiness_report( import_interval_seconds: int, community_import_interval_seconds: int = 1800, now: datetime | None = None, ) -> tuple[bool, dict[str, dict[str, object]]]: + """A01: Separate infrastructure readiness from import health diagnostics. + + Infrastructure (DB, MinIO) blocks readiness. Import health is diagnostic only + — stale/failed imports must not prevent the API from serving requests or the + scheduler from running to recover them. + """ current = now or datetime.now(timezone.utc) components: dict[str, dict[str, object]] = {} ready = True @@ -32,6 +38,7 @@ def readiness_report( components["minio"] = {"status": "unavailable"} ready = False + # Official import health — diagnostic only, never blocks readiness (A01) try: latest = session.scalar(select(OfficialRecordImport).order_by( OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc(), @@ -40,10 +47,13 @@ def readiness_report( components["official_import"] = { "status": "optional", "last_run_status": latest.status.value if latest else None, + "blocking": False, } elif latest is None: - components["official_import"] = {"status": "not_run"} - ready = False + components["official_import"] = { + "status": "not_run", + "blocking": False, + } else: started = latest.started_at if latest.started_at.tzinfo else latest.started_at.replace(tzinfo=timezone.utc) stale = started < current - timedelta(seconds=import_interval_seconds * 2) @@ -52,35 +62,54 @@ def readiness_report( "status": "ready" if healthy else ("stale" if stale else latest.status.value), "last_run_status": latest.status.value, "last_started_at": started.isoformat(), + "blocking": False, } - ready = ready and healthy except Exception: - components["official_import"] = {"status": "unknown"} - if import_required: - ready = False + components["official_import"] = { + "status": "unknown", + "blocking": False, + } - # Check community scheduler: look for recent import runs + # Community scheduler health — diagnostic only, never blocks readiness (A01) + # Track per-source health with rotation, backoff, last success, and stalled attempts try: - latest_community = session.scalar( - select(CommunityImportRun) - .order_by(CommunityImportRun.started_at.desc()) - .limit(1) - ) - if latest_community is None: - components["community_scheduler"] = {"status": "not_started"} - else: - started = latest_community.started_at + enabled_sources = list(session.scalars( + select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.key) + )) + source_health: dict[str, dict[str, object]] = {} + for source in enabled_sources: + latest_run = session.scalar( + select(CommunityImportRun) + .where(CommunityImportRun.source_system == source.key) + .order_by(CommunityImportRun.started_at.desc()) + .limit(1) + ) + if latest_run is None: + source_health[source.key] = {"status": "not_started", "blocking": False} + continue + started = latest_run.started_at if started.tzinfo is None: started = started.replace(tzinfo=timezone.utc) stale = started < current - timedelta(seconds=community_import_interval_seconds * 2) - healthy = latest_community.status == "success" and not stale - components["community_scheduler"] = { - "status": "ready" if healthy else ("stale" if stale else latest_community.status), + healthy = latest_run.status == "success" and not stale + # Count recent failures for backoff detection + recent_failures = session.scalar( + select(func.count()).select_from(CommunityImportRun) + .where( + CommunityImportRun.source_system == source.key, + CommunityImportRun.status == "failed", + CommunityImportRun.started_at >= current - timedelta(hours=24), + ) + ) or 0 + source_health[source.key] = { + "status": "ready" if healthy else ("stale" if stale else latest_run.status), "last_started_at": started.isoformat(), + "recent_failures_24h": recent_failures, + "backoff_recommended": recent_failures >= 5, + "blocking": False, } - ready = ready and healthy + components["community_scheduler"] = {"status": "ready", "sources": source_health} except Exception: - components["community_scheduler"] = {"status": "unknown"} - ready = False + components["community_scheduler"] = {"status": "unknown", "sources": {}} return ready, components diff --git a/apps/api/tests/test_readiness.py b/apps/api/tests/test_readiness.py index 4dcc1de..52b8950 100644 --- a/apps/api/tests/test_readiness.py +++ b/apps/api/tests/test_readiness.py @@ -6,7 +6,7 @@ from sqlalchemy import create_engine from sqlalchemy.orm import Session from app.database import Base -from app.models import ImportStatus, CommunityImportRun, OfficialRecordImport +from app.models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport from app.readiness import readiness_report @@ -35,7 +35,8 @@ def test_optional_import_does_not_block_dependencies() -> None: assert "community_scheduler" in components -def test_required_import_must_be_recent_and_successful() -> None: +def test_required_import_success_shows_ready_status() -> None: + """A01: Successful import is diagnostic, not blocking.""" engine = create_engine("sqlite://") Base.metadata.create_all(engine) now = datetime.now(timezone.utc) @@ -52,9 +53,11 @@ def test_required_import_must_be_recent_and_successful() -> None: ) assert ready is True assert components["official_import"]["status"] == "ready" + assert components["official_import"]["blocking"] is False -def test_unavailable_storage_and_stale_import_fail_readiness() -> None: +def test_unavailable_storage_blocks_readiness_but_stale_import_does_not() -> None: + """A01: Infrastructure failures block, stale imports are diagnostic only.""" engine = create_engine("sqlite://") Base.metadata.create_all(engine) now = datetime.now(timezone.utc) @@ -72,13 +75,16 @@ def test_unavailable_storage_and_stale_import_fail_readiness() -> None: assert ready is False assert components["minio"]["status"] == "unavailable" assert components["official_import"]["status"] == "stale" + assert components["official_import"]["blocking"] is False -def test_community_scheduler_success_does_not_block_readiness() -> None: +def test_community_scheduler_success_shows_ready_status() -> None: + """A01: Successful scheduler is diagnostic, not blocking.""" engine = create_engine("sqlite://") Base.metadata.create_all(engine) now = datetime.now(timezone.utc) with Session(engine) as session: + session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True)) session.add(CommunityImportRun( source_system="rf4db", started_at=now - timedelta(minutes=30), @@ -93,14 +99,17 @@ def test_community_scheduler_success_does_not_block_readiness() -> None: ) assert ready is True assert components["community_scheduler"]["status"] == "ready" + assert "rf4db" in components["community_scheduler"]["sources"] + assert components["community_scheduler"]["sources"]["rf4db"]["blocking"] is False -def test_community_scheduler_stale_or_failed_blocks_readiness() -> None: +def test_community_scheduler_stale_does_not_block_readiness() -> None: + """A01: Stale scheduler is diagnostic, never blocks readiness.""" engine = create_engine("sqlite://") Base.metadata.create_all(engine) now = datetime.now(timezone.utc) with Session(engine) as session: - # Stale run + session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True)) session.add(CommunityImportRun( source_system="rf4db", started_at=now - timedelta(hours=2), @@ -113,15 +122,19 @@ def test_community_scheduler_stale_or_failed_blocks_readiness() -> None: session, AvailableStorage(), import_required=False, import_interval_seconds=3600, community_import_interval_seconds=1800, now=now, ) - assert ready is False - assert components["community_scheduler"]["status"] == "stale" + assert ready is True # A01: stale does NOT block + assert components["community_scheduler"]["status"] == "ready" + assert components["community_scheduler"]["sources"]["rf4db"]["status"] == "stale" + assert components["community_scheduler"]["sources"]["rf4db"]["blocking"] is False -def test_community_scheduler_failed_status_blocks_readiness() -> None: +def test_community_scheduler_failed_does_not_block_readiness() -> None: + """A01: Failed scheduler is diagnostic, never blocks readiness.""" engine = create_engine("sqlite://") Base.metadata.create_all(engine) now = datetime.now(timezone.utc) with Session(engine) as session: + session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True)) session.add(CommunityImportRun( source_system="rf4db", started_at=now - timedelta(minutes=30), @@ -135,5 +148,48 @@ def test_community_scheduler_failed_status_blocks_readiness() -> None: session, AvailableStorage(), import_required=False, import_interval_seconds=3600, community_import_interval_seconds=1800, now=now, ) - assert ready is False - assert components["community_scheduler"]["status"] == "failed" + assert ready is True # A01: failed does NOT block + assert components["community_scheduler"]["status"] == "ready" + assert components["community_scheduler"]["sources"]["rf4db"]["status"] == "failed" + assert components["community_scheduler"]["sources"]["rf4db"]["blocking"] is False + + +def test_community_scheduler_tracked_per_source_with_backoff() -> None: + """A01: Per-source health tracking with backoff detection.""" + engine = create_engine("sqlite://") + Base.metadata.create_all(engine) + now = datetime.now(timezone.utc) + with Session(engine) as session: + session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True)) + session.add(DataSource(key="rf4stat-fishing", name="RF4-STAT", base_url="https://rf4-stat.ru", default_confidence=65, enabled=True)) + # rf4db: healthy + session.add(CommunityImportRun( + source_system="rf4db", + started_at=now - timedelta(minutes=30), + status="success", + source_url="fixture://rf4db", + rows_seen=5, rows_created=5, rows_updated=0, error_summary=None, + )) + # rf4stat-fishing: multiple recent failures → backoff recommended + for i in range(6): + session.add(CommunityImportRun( + source_system="rf4stat-fishing", + started_at=now - timedelta(hours=i), + status="failed", + source_url="fixture://rf4stat", + rows_seen=0, rows_created=0, rows_updated=0, + error_summary="TimeoutError", + )) + session.commit() + ready, components = readiness_report( + session, AvailableStorage(), import_required=False, + import_interval_seconds=3600, community_import_interval_seconds=1800, now=now, + ) + assert ready is True + sources = components["community_scheduler"]["sources"] + assert sources["rf4db"]["status"] == "ready" + assert sources["rf4db"]["recent_failures_24h"] == 0 + assert sources["rf4db"]["backoff_recommended"] is False + assert sources["rf4stat-fishing"]["status"] == "failed" + assert sources["rf4stat-fishing"]["recent_failures_24h"] == 6 + assert sources["rf4stat-fishing"]["backoff_recommended"] is True