84 lines
2.8 KiB
Python
84 lines
2.8 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Literal
|
|
from urllib.error import HTTPError
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .models import ExternalObservation, ModerationStatus
|
|
|
|
|
|
SourceCheckStatus = Literal["available", "missing", "temporary_error", "blocked"]
|
|
|
|
|
|
def classify_source_failure(exc: Exception) -> SourceCheckStatus:
|
|
"""Classify the result of the scheduled request without retrying it."""
|
|
if isinstance(exc, HTTPError):
|
|
if exc.code in {404, 410}:
|
|
return "missing"
|
|
if exc.code in {401, 403, 429}:
|
|
return "blocked"
|
|
return "temporary_error"
|
|
|
|
|
|
def record_source_check(
|
|
session: Session,
|
|
observation: ExternalObservation,
|
|
status: SourceCheckStatus,
|
|
*,
|
|
checked_at: datetime | None = None,
|
|
) -> ExternalObservation:
|
|
"""Persist a check performed during an already scheduled source request.
|
|
|
|
Only an authoritative 404/410-style ``missing`` result withdraws published
|
|
data. Transient errors and access blocks remain diagnostic and never remove
|
|
an observation from activity.
|
|
"""
|
|
_apply_source_check(observation, status, checked_at or datetime.now(timezone.utc))
|
|
session.commit()
|
|
return observation
|
|
|
|
|
|
def _apply_source_check(
|
|
observation: ExternalObservation,
|
|
status: SourceCheckStatus,
|
|
checked_at: datetime,
|
|
) -> None:
|
|
"""Mutate one observation; the caller owns the transaction boundary."""
|
|
current = checked_at
|
|
observation.source_check_status = status
|
|
observation.source_checked_at = current
|
|
if status == "missing" and observation.status != "withdrawn":
|
|
if observation.catch_report is not None:
|
|
observation.catch_report.moderation_status = ModerationStatus.pending
|
|
observation.status = "withdrawn"
|
|
observation.review_note = "Source record missing; withdrawn pending moderator review"
|
|
observation.reviewed_at = current
|
|
observation.moderation_version += 1
|
|
|
|
|
|
def record_scheduled_source_check(
|
|
session: Session,
|
|
*,
|
|
source_system: str,
|
|
source_url: str,
|
|
status: SourceCheckStatus,
|
|
checked_at: datetime | None = None,
|
|
) -> int:
|
|
"""Apply one scheduled request result only to observations with that exact URL.
|
|
|
|
Aggregate pages cannot prove that an omitted record was deleted, so absence
|
|
from a parsed listing is deliberately ignored.
|
|
"""
|
|
observations = list(session.scalars(select(ExternalObservation).where(
|
|
ExternalObservation.source_system == source_system,
|
|
ExternalObservation.source_url == source_url,
|
|
)))
|
|
current = checked_at or datetime.now(timezone.utc)
|
|
for observation in observations:
|
|
_apply_source_check(observation, status, current)
|
|
session.commit()
|
|
return len(observations)
|