from __future__ import annotations from datetime import datetime, timezone from typing import Literal from urllib.error import HTTPError from sqlalchemy import select from sqlalchemy.orm import Session from .models import ExternalObservation, ModerationStatus SourceCheckStatus = Literal["available", "missing", "temporary_error", "blocked"] def classify_source_failure(exc: Exception) -> SourceCheckStatus: """Classify the result of the scheduled request without retrying it.""" if isinstance(exc, HTTPError): if exc.code in {404, 410}: return "missing" if exc.code in {401, 403, 429}: return "blocked" return "temporary_error" def record_source_check( session: Session, observation: ExternalObservation, status: SourceCheckStatus, *, checked_at: datetime | None = None, ) -> ExternalObservation: """Persist a check performed during an already scheduled source request. Only an authoritative 404/410-style ``missing`` result withdraws published data. Transient errors and access blocks remain diagnostic and never remove an observation from activity. """ _apply_source_check(observation, status, checked_at or datetime.now(timezone.utc)) session.commit() return observation def _apply_source_check( observation: ExternalObservation, status: SourceCheckStatus, checked_at: datetime, ) -> None: """Mutate one observation; the caller owns the transaction boundary.""" current = checked_at observation.source_check_status = status observation.source_checked_at = current if status == "missing" and observation.status != "withdrawn": if observation.catch_report is not None: observation.catch_report.moderation_status = ModerationStatus.pending observation.status = "withdrawn" observation.review_note = "Source record missing; withdrawn pending moderator review" observation.reviewed_at = current observation.moderation_version += 1 def record_scheduled_source_check( session: Session, *, source_system: str, source_url: str, status: SourceCheckStatus, checked_at: datetime | None = None, ) -> int: """Apply one scheduled request result only to observations with that exact URL. Aggregate pages cannot prove that an omitted record was deleted, so absence from a parsed listing is deliberately ignored. """ observations = list(session.scalars(select(ExternalObservation).where( ExternalObservation.source_system == source_system, ExternalObservation.source_url == source_url, ))) current = checked_at or datetime.now(timezone.utc) for observation in observations: _apply_source_check(observation, status, current) session.commit() return len(observations)