ik/rf4-spotter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

A01: Separate API readiness from import health diagnostics

Browse Source 
- Infrastructure (DB/MinIO) blocks readiness; imports are diagnostic only
- Per-source community scheduler health with backoff detection
- Stale/failed imports never block /ready — scheduler can recover them
- Add 'blocking: false' to all import components
- 4 new tests: per-source health, backoff detection, stale/failed non-blocking
- 108 Python tests pass
This commit is contained in:
ik
2026-09-10 06:10:01 +07:00
parent 98e7649f9d
commit 779d554057
2 changed files with 120 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/app/readiness.py
+53 -24
View File
@@ -3,10 +3,10 @@ from __future__ import annotations
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import select, text
from sqlalchemy import func, select, text
from sqlalchemy.orm import Session
from .models import CommunityImportRun, ImportStatus, OfficialRecordImport
from .models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport
def readiness_report(
@@ -14,6 +14,12 @@ def readiness_report(
import_interval_seconds: int, community_import_interval_seconds: int = 1800,
now: datetime | None = None,
) -> tuple[bool, dict[str, dict[str, object]]]:
"""A01: Separate infrastructure readiness from import health diagnostics.
Infrastructure (DB, MinIO) blocks readiness. Import health is diagnostic only
— stale/failed imports must not prevent the API from serving requests or the
scheduler from running to recover them.
"""
current = now or datetime.now(timezone.utc)
components: dict[str, dict[str, object]] = {}
ready = True
@@ -32,6 +38,7 @@ def readiness_report(
components["minio"] = {"status": "unavailable"}
ready = False
# Official import health — diagnostic only, never blocks readiness (A01)
try:
latest = session.scalar(select(OfficialRecordImport).order_by(
OfficialRecordImport.started_at.desc(), OfficialRecordImport.id.desc(),
@@ -40,10 +47,13 @@ def readiness_report(
components["official_import"] = {
"status": "optional",
"last_run_status": latest.status.value if latest else None,
"blocking": False,
}
elif latest is None:
components["official_import"] = {"status": "not_run"}
ready = False
components["official_import"] = {
"status": "not_run",
"blocking": False,
}
else:
started = latest.started_at if latest.started_at.tzinfo else latest.started_at.replace(tzinfo=timezone.utc)
stale = started < current - timedelta(seconds=import_interval_seconds * 2)
@@ -52,35 +62,54 @@ def readiness_report(
"status": "ready" if healthy else ("stale" if stale else latest.status.value),
"last_run_status": latest.status.value,
"last_started_at": started.isoformat(),
"blocking": False,
}
ready = ready and healthy
except Exception:
components["official_import"] = {"status": "unknown"}
if import_required:
ready = False
components["official_import"] = {
"status": "unknown",
"blocking": False,
}
# Check community scheduler: look for recent import runs
# Community scheduler health — diagnostic only, never blocks readiness (A01)
# Track per-source health with rotation, backoff, last success, and stalled attempts
try:
latest_community = session.scalar(
select(CommunityImportRun)
.order_by(CommunityImportRun.started_at.desc())
.limit(1)
)
if latest_community is None:
components["community_scheduler"] = {"status": "not_started"}
else:
started = latest_community.started_at
enabled_sources = list(session.scalars(
select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.key)
))
source_health: dict[str, dict[str, object]] = {}
for source in enabled_sources:
latest_run = session.scalar(
select(CommunityImportRun)
.where(CommunityImportRun.source_system == source.key)
.order_by(CommunityImportRun.started_at.desc())
.limit(1)
)
if latest_run is None:
source_health[source.key] = {"status": "not_started", "blocking": False}
continue
started = latest_run.started_at
if started.tzinfo is None:
started = started.replace(tzinfo=timezone.utc)
stale = started < current - timedelta(seconds=community_import_interval_seconds * 2)
healthy = latest_community.status == "success" and not stale
components["community_scheduler"] = {
"status": "ready" if healthy else ("stale" if stale else latest_community.status),
healthy = latest_run.status == "success" and not stale
# Count recent failures for backoff detection
recent_failures = session.scalar(
select(func.count()).select_from(CommunityImportRun)
.where(
CommunityImportRun.source_system == source.key,
CommunityImportRun.status == "failed",
CommunityImportRun.started_at >= current - timedelta(hours=24),
)
) or 0
source_health[source.key] = {
"status": "ready" if healthy else ("stale" if stale else latest_run.status),
"last_started_at": started.isoformat(),
"recent_failures_24h": recent_failures,
"backoff_recommended": recent_failures >= 5,
"blocking": False,
}
ready = ready and healthy
components["community_scheduler"] = {"status": "ready", "sources": source_health}
except Exception:
components["community_scheduler"] = {"status": "unknown"}
ready = False
components["community_scheduler"] = {"status": "unknown", "sources": {}}
return ready, components
apps/api/tests/test_readiness.py
+67 -11
View File
@@ -6,7 +6,7 @@ from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from app.database import Base
from app.models import ImportStatus, CommunityImportRun, OfficialRecordImport
from app.models import CommunityImportRun, DataSource, ImportStatus, OfficialRecordImport
from app.readiness import readiness_report
@@ -35,7 +35,8 @@ def test_optional_import_does_not_block_dependencies() -> None:
assert "community_scheduler" in components
def test_required_import_must_be_recent_and_successful() -> None:
def test_required_import_success_shows_ready_status() -> None:
"""A01: Successful import is diagnostic, not blocking."""
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
@@ -52,9 +53,11 @@ def test_required_import_must_be_recent_and_successful() -> None:
)
assert ready is True
assert components["official_import"]["status"] == "ready"
assert components["official_import"]["blocking"] is False
def test_unavailable_storage_and_stale_import_fail_readiness() -> None:
def test_unavailable_storage_blocks_readiness_but_stale_import_does_not() -> None:
"""A01: Infrastructure failures block, stale imports are diagnostic only."""
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
@@ -72,13 +75,16 @@ def test_unavailable_storage_and_stale_import_fail_readiness() -> None:
assert ready is False
assert components["minio"]["status"] == "unavailable"
assert components["official_import"]["status"] == "stale"
assert components["official_import"]["blocking"] is False
def test_community_scheduler_success_does_not_block_readiness() -> None:
def test_community_scheduler_success_shows_ready_status() -> None:
"""A01: Successful scheduler is diagnostic, not blocking."""
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
with Session(engine) as session:
session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True))
session.add(CommunityImportRun(
source_system="rf4db",
started_at=now - timedelta(minutes=30),
@@ -93,14 +99,17 @@ def test_community_scheduler_success_does_not_block_readiness() -> None:
)
assert ready is True
assert components["community_scheduler"]["status"] == "ready"
assert "rf4db" in components["community_scheduler"]["sources"]
assert components["community_scheduler"]["sources"]["rf4db"]["blocking"] is False
def test_community_scheduler_stale_or_failed_blocks_readiness() -> None:
def test_community_scheduler_stale_does_not_block_readiness() -> None:
"""A01: Stale scheduler is diagnostic, never blocks readiness."""
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
with Session(engine) as session:
# Stale run
session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True))
session.add(CommunityImportRun(
source_system="rf4db",
started_at=now - timedelta(hours=2),
@@ -113,15 +122,19 @@ def test_community_scheduler_stale_or_failed_blocks_readiness() -> None:
session, AvailableStorage(), import_required=False,
import_interval_seconds=3600, community_import_interval_seconds=1800, now=now,
)
assert ready is False
assert components["community_scheduler"]["status"] == "stale"
assert ready is True # A01: stale does NOT block
assert components["community_scheduler"]["status"] == "ready"
assert components["community_scheduler"]["sources"]["rf4db"]["status"] == "stale"
assert components["community_scheduler"]["sources"]["rf4db"]["blocking"] is False
def test_community_scheduler_failed_status_blocks_readiness() -> None:
def test_community_scheduler_failed_does_not_block_readiness() -> None:
"""A01: Failed scheduler is diagnostic, never blocks readiness."""
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
with Session(engine) as session:
session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True))
session.add(CommunityImportRun(
source_system="rf4db",
started_at=now - timedelta(minutes=30),
@@ -135,5 +148,48 @@ def test_community_scheduler_failed_status_blocks_readiness() -> None:
session, AvailableStorage(), import_required=False,
import_interval_seconds=3600, community_import_interval_seconds=1800, now=now,
)
assert ready is False
assert components["community_scheduler"]["status"] == "failed"
assert ready is True # A01: failed does NOT block
assert components["community_scheduler"]["status"] == "ready"
assert components["community_scheduler"]["sources"]["rf4db"]["status"] == "failed"
assert components["community_scheduler"]["sources"]["rf4db"]["blocking"] is False
def test_community_scheduler_tracked_per_source_with_backoff() -> None:
"""A01: Per-source health tracking with backoff detection."""
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
now = datetime.now(timezone.utc)
with Session(engine) as session:
session.add(DataSource(key="rf4db", name="RF4DB", base_url="https://rf4db.com", default_confidence=70, enabled=True))
session.add(DataSource(key="rf4stat-fishing", name="RF4-STAT", base_url="https://rf4-stat.ru", default_confidence=65, enabled=True))
# rf4db: healthy
session.add(CommunityImportRun(
source_system="rf4db",
started_at=now - timedelta(minutes=30),
status="success",
source_url="fixture://rf4db",
rows_seen=5, rows_created=5, rows_updated=0, error_summary=None,
))
# rf4stat-fishing: multiple recent failures → backoff recommended
for i in range(6):
session.add(CommunityImportRun(
source_system="rf4stat-fishing",
started_at=now - timedelta(hours=i),
status="failed",
source_url="fixture://rf4stat",
rows_seen=0, rows_created=0, rows_updated=0,
error_summary="TimeoutError",
))
session.commit()
ready, components = readiness_report(
session, AvailableStorage(), import_required=False,
import_interval_seconds=3600, community_import_interval_seconds=1800, now=now,
)
assert ready is True
sources = components["community_scheduler"]["sources"]
assert sources["rf4db"]["status"] == "ready"
assert sources["rf4db"]["recent_failures_24h"] == 0
assert sources["rf4db"]["backoff_recommended"] is False
assert sources["rf4stat-fishing"]["status"] == "failed"
assert sources["rf4stat-fishing"]["recent_failures_24h"] == 6
assert sources["rf4stat-fishing"]["backoff_recommended"] is True