Compare commits

...
3 Commits
Author SHA1 Message Date
ik b1d6d8d122 fix: harden catch report idempotency
CI / backend-and-migrations (push) Canceled after 0s
CI / astro-build (push) Canceled after 0s
CI / dependency-audit (push) Canceled after 0s
CI / compose-e2e (push) Canceled after 0s
2026-09-22 08:04:11 +07:00
ik 0eb151f5a7 fix: serialize media publishing decisions 2026-09-22 07:58:38 +07:00
ik f3fb5868df fix: persist media decisions across production restores 2026-09-22 07:48:43 +07:00
18 changed files with 602 additions and 121 deletions

No files matched your search

+5
View File
@@ -12,6 +12,11 @@ KNOWN_MEDIA_ROLES = WATERBODY_MEDIA_ROLES | TACKLE_MEDIA_ROLES
MEDIA_ROLES_BY_ENTITY = {"waterbody": WATERBODY_MEDIA_ROLES, "tackle": TACKLE_MEDIA_ROLES} MEDIA_ROLES_BY_ENTITY = {"waterbody": WATERBODY_MEDIA_ROLES, "tackle": TACKLE_MEDIA_ROLES}
def media_manifest_version() -> int:
manifest = json.loads((MEDIA_ROOT / "manifest.json").read_text(encoding="utf-8"))
return max(1, int(manifest.get("version", 1)))
def _public_role_allowed(entity_type: str | None, role: object) -> bool: def _public_role_allowed(entity_type: str | None, role: object) -> bool:
if role is None: if role is None:
return True return True
+26 -6
View File
@@ -15,12 +15,12 @@ from ..community_review import ExternalReviewError, map_observation, publish_obs
from ..config import settings from ..config import settings
from ..dependencies import Db from ..dependencies import Db
from ..importer import ImportAlreadyRunning, ImportSourceError, import_records from ..importer import ImportAlreadyRunning, ImportSourceError, import_records
from rf4_research.media_assets import publish_quality_upgrades, rollback_quality_upgrade from rf4_research.media_assets import preview_quality_upgrades, publish_quality_upgrades, rollback_quality_upgrade
from ..media_catalog import KNOWN_MEDIA_ROLES, MEDIA_ROLES_BY_ENTITY, MEDIA_ROOT, review_assets, review_file from ..media_catalog import KNOWN_MEDIA_ROLES, MEDIA_ROLES_BY_ENTITY, MEDIA_ROOT, media_manifest_version, review_assets, review_file
from ..models import CatchReport, CommunityImportRun, DataSource, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Waterbody from ..models import CatchReport, CommunityImportRun, DataSource, ExternalObservation, Fish, ModerationEvent, ModerationStatus, OfficialRecordImport, SourceType, Waterbody
from ..public_cache import public_cache from ..public_cache import public_cache
from ..schemas import AdminCatchReportOut, AdminMediaDecision, AdminMediaReviewOut, AdminMediaRollback, AdminModerationHistoryOut, AdminSourceStatusOut, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationAction, ExternalObservationBulkAction, ExternalObservationBulkDecision, ExternalObservationBulkMapping, ExternalObservationBulkResult, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, ImportRunOut, ModerationBulkResult, ModerationBulkUpdate, ModerationUpdate from ..schemas import AdminCatchReportOut, AdminMediaDecision, AdminMediaPreview, AdminMediaReviewOut, AdminMediaRollback, AdminModerationHistoryOut, AdminSourceStatusOut, CatchReportCreated, ExternalAliasSuggestionOut, ExternalObservationAction, ExternalObservationBulkAction, ExternalObservationBulkDecision, ExternalObservationBulkMapping, ExternalObservationBulkResult, ExternalObservationDecision, ExternalObservationMapping, ExternalObservationOut, ExternalObservationPublished, ImportRunOut, ModerationBulkResult, ModerationBulkUpdate, ModerationUpdate
from ..storage import delete_screenshot, signed_screenshot_url from ..storage import delete_screenshot, signed_screenshot_url
from ..time_utils import aware from ..time_utils import aware
@@ -35,6 +35,7 @@ def _admin(request: Request, db: Db, authorization: Annotated[str | None, Header
@router.get("/api/v1/admin/media/catalog", response_model=list[AdminMediaReviewOut]) @router.get("/api/v1/admin/media/catalog", response_model=list[AdminMediaReviewOut])
def admin_media_catalog( def admin_media_catalog(
_: Annotated[str, Depends(_admin)], _: Annotated[str, Depends(_admin)],
response: Response,
entity_type: str | None = Query(None, pattern="^(fish|waterbody|tackle|reference)$"), entity_type: str | None = Query(None, pattern="^(fish|waterbody|tackle|reference)$"),
status: str | None = Query(None, pattern="^(approved|upgrade_queued|upgrade_stored)$"), status: str | None = Query(None, pattern="^(approved|upgrade_queued|upgrade_stored)$"),
media_role: str | None = Query(None, pattern="^(waterbody_cover|waterbody_map|waterbody_depth_map|waterbody_screenshot|tackle_card|tackle_detail|rig_diagram|tackle_screenshot)$"), media_role: str | None = Query(None, pattern="^(waterbody_cover|waterbody_map|waterbody_depth_map|waterbody_screenshot|tackle_card|tackle_detail|rig_diagram|tackle_screenshot)$"),
@@ -46,6 +47,7 @@ def admin_media_catalog(
raise HTTPException(status_code=422, detail="unknown media role") raise HTTPException(status_code=422, detail="unknown media role")
if media_role and (not entity_type or media_role not in MEDIA_ROLES_BY_ENTITY.get(entity_type, set())): if media_role and (not entity_type or media_role not in MEDIA_ROLES_BY_ENTITY.get(entity_type, set())):
raise HTTPException(status_code=422, detail="media role is incompatible with entity type") raise HTTPException(status_code=422, detail="media role is incompatible with entity type")
response.headers["X-Media-Manifest-Version"] = str(media_manifest_version())
return review_assets(entity_type, status, media_role, q.strip() if q else None)[offset:offset + limit] return review_assets(entity_type, status, media_role, q.strip() if q else None)[offset:offset + limit]
@@ -58,6 +60,20 @@ def admin_media_asset(digest: str, _: Annotated[str, Depends(_admin)]) -> FileRe
return FileResponse(path, media_type=media_type, headers={"Cache-Control": "private, no-store"}) return FileResponse(path, media_type=media_type, headers={"Cache-Control": "private, no-store"})
@router.post("/api/v1/admin/media/upgrades/preview")
def admin_preview_media_upgrades(
payload: AdminMediaPreview,
_: Annotated[str, Depends(_admin)],
) -> dict:
try:
return preview_quality_upgrades(
MEDIA_ROOT / "manifest.json", asset_ids=payload.asset_ids,
expected_version=payload.expected_version,
)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/api/v1/admin/media/upgrades/publish") @router.post("/api/v1/admin/media/upgrades/publish")
def admin_publish_media_upgrades( def admin_publish_media_upgrades(
payload: AdminMediaDecision, payload: AdminMediaDecision,
@@ -65,7 +81,10 @@ def admin_publish_media_upgrades(
) -> dict[str, int]: ) -> dict[str, int]:
"""Atomically publish all stored quality upgrades after an explicit decision.""" """Atomically publish all stored quality upgrades after an explicit decision."""
try: try:
return publish_quality_upgrades(MEDIA_ROOT / "manifest.json", note=payload.note) return publish_quality_upgrades(
MEDIA_ROOT / "manifest.json", note=payload.note,
asset_ids=payload.asset_ids, expected_version=payload.expected_version,
)
except ValueError as exc: except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc raise HTTPException(status_code=409, detail=str(exc)) from exc
@@ -74,11 +93,12 @@ def admin_publish_media_upgrades(
def admin_rollback_media_upgrade( def admin_rollback_media_upgrade(
payload: AdminMediaRollback, payload: AdminMediaRollback,
_: Annotated[str, Depends(_admin)], _: Annotated[str, Depends(_admin)],
) -> dict[str, str]: ) -> dict:
"""Restore one superseded fallback while retaining the reviewed candidate.""" """Restore one superseded fallback while retaining the reviewed candidate."""
try: try:
return rollback_quality_upgrade( return rollback_quality_upgrade(
MEDIA_ROOT / "manifest.json", asset_url=payload.asset_url, note=payload.note, MEDIA_ROOT / "manifest.json", asset_id=payload.asset_id,
note=payload.note, expected_version=payload.expected_version,
) )
except ValueError as exc: except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc raise HTTPException(status_code=409, detail=str(exc)) from exc
+50 -19
View File
@@ -39,25 +39,14 @@ def create_catch_report(payload: CatchReportCreate, request: Request, db: Db, id
db.expire_all() db.expire_all()
existing = db.scalar(select(SubmissionAttempt).where(SubmissionAttempt.idempotency_key == key_hash, SubmissionAttempt.created_at >= cutoff)) existing = db.scalar(select(SubmissionAttempt).where(SubmissionAttempt.idempotency_key == key_hash, SubmissionAttempt.created_at >= cutoff))
if existing is not None: if existing is not None:
logger.info("idempotent hit", extra={"idempotency_key": idempotency_key[:8]}) return _replay_existing(existing, payload_hash, key_hash, cutoff)
if existing.payload_hash and not hmac.compare_digest(existing.payload_hash, payload_hash):
raise HTTPException(status_code=409, detail="Idempotency-Key was already used with different payload")
if existing.catch_report is None:
raise HTTPException(status_code=409, detail="idempotency record is incomplete; retry with a new key")
replay_token = _replay_token(key_hash)
if not hmac.compare_digest(hashlib.sha256(replay_token.encode()).hexdigest(), existing.catch_report.screenshot_upload_token_hash or ""):
raise HTTPException(status_code=409, detail="idempotency record token mismatch; retry with a new key")
return JSONResponse(status_code=200, content=_accepted(existing.catch_report, replay_token, True))
logger.info("idempotency check miss", extra={"idempotency_key": idempotency_key[:8]}) logger.info("idempotency check miss", extra={"idempotency_key": idempotency_key[:8]})
check_rate_limit(request, db, settings) check_rate_limit(request, db, settings)
fish = db.scalar(select(Fish).where(Fish.slug == payload.fish_slug)) fish = db.scalar(select(Fish).where(Fish.slug == payload.fish_slug))
waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == payload.waterbody_slug)) waterbody = db.scalar(select(Waterbody).where(Waterbody.slug == payload.waterbody_slug))
if fish is None or waterbody is None: if fish is None or waterbody is None:
raise HTTPException(status_code=422, detail="unknown fish or waterbody") raise HTTPException(status_code=422, detail="unknown fish or waterbody")
spot = db.scalar(select(Spot).where(Spot.waterbody_id == waterbody.id, Spot.x == payload.x, Spot.y == payload.y)) spot = _spot(db, waterbody, payload.x, payload.y)
if spot is None:
spot = Spot(waterbody=waterbody, x=payload.x, y=payload.y)
db.add(spot)
bait = _bait(db, payload.bait_name) bait = _bait(db, payload.bait_name)
upload_token = _replay_token(key_hash) if key_hash else secrets.token_urlsafe(32) upload_token = _replay_token(key_hash) if key_hash else secrets.token_urlsafe(32)
report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None, screenshot_upload_token_hash=hashlib.sha256(upload_token.encode()).hexdigest()) report = CatchReport(fish=fish, spot=spot, waterbody=waterbody, bait=bait, weight_g=payload.weight_g, fishing_method=payload.fishing_method, rig_type=payload.rig_type, retrieve_method=payload.retrieve_method, retrieve_speed=payload.retrieve_speed, caught_at=payload.caught_at, reported_at=datetime.now(timezone.utc), player_name=payload.player_name, source_type=SourceType.user, source_url=payload.source_url, source_confidence=60, moderation_status=ModerationStatus.pending, raw_payload={"comment": payload.comment} if payload.comment else None, screenshot_upload_token_hash=hashlib.sha256(upload_token.encode()).hexdigest())
@@ -77,8 +66,8 @@ def create_catch_report(payload: CatchReportCreate, request: Request, db: Db, id
except IntegrityError: except IntegrityError:
db.rollback() db.rollback()
winner = db.scalar(select(SubmissionAttempt).where(SubmissionAttempt.idempotency_key == key_hash)) winner = db.scalar(select(SubmissionAttempt).where(SubmissionAttempt.idempotency_key == key_hash))
if winner and winner.catch_report: if winner is not None:
return JSONResponse(status_code=200, content=_accepted(winner.catch_report, _replay_token(key_hash), True)) return _replay_existing(winner, payload_hash, key_hash, cutoff)
raise raise
logger.info("idempotency key stored", extra={"idempotency_key": idempotency_key[:8]}) logger.info("idempotency key stored", extra={"idempotency_key": idempotency_key[:8]})
else: else:
@@ -114,12 +103,54 @@ def _accepted(report: CatchReport, token: str, idempotent: bool) -> dict[str, ob
return {"id": str(report.id), "moderation_status": report.moderation_status.value, "screenshot_upload_token": token, "idempotent": idempotent} return {"id": str(report.id), "moderation_status": report.moderation_status.value, "screenshot_upload_token": token, "idempotent": idempotent}
def _replay_existing(existing: SubmissionAttempt, payload_hash: str, key_hash: str, cutoff: datetime) -> JSONResponse:
logger.info("idempotent hit", extra={"idempotency_key": key_hash[:8]})
created_at = existing.created_at if existing.created_at.tzinfo else existing.created_at.replace(tzinfo=timezone.utc)
if created_at < cutoff:
raise HTTPException(status_code=409, detail="Idempotency-Key has expired; retry with a new key")
if existing.payload_hash and not hmac.compare_digest(existing.payload_hash, payload_hash):
raise HTTPException(status_code=409, detail="Idempotency-Key was already used with different payload")
if existing.catch_report is None:
raise HTTPException(status_code=409, detail="idempotency record is incomplete; retry with a new key")
replay_token = _replay_token(key_hash)
if existing.catch_report.screenshot_key is None:
if not hmac.compare_digest(hashlib.sha256(replay_token.encode()).hexdigest(), existing.catch_report.screenshot_upload_token_hash or ""):
raise HTTPException(status_code=409, detail="idempotency record token mismatch; retry with a new key")
return JSONResponse(status_code=200, content=_accepted(existing.catch_report, replay_token, True))
def _spot(db: Db, waterbody: Waterbody, x: int, y: int) -> Spot:
spot = db.scalar(select(Spot).where(Spot.waterbody_id == waterbody.id, Spot.x == x, Spot.y == y))
if spot is not None:
return spot
candidate = Spot(waterbody=waterbody, x=x, y=y)
try:
with db.begin_nested():
db.add(candidate)
db.flush()
return candidate
except IntegrityError:
winner = db.scalar(select(Spot).where(Spot.waterbody_id == waterbody.id, Spot.x == x, Spot.y == y))
if winner is None:
raise
return winner
def _bait(db: Db, value: str | None) -> Bait | None: def _bait(db: Db, value: str | None) -> Bait | None:
if not value or not value.strip(): if not value or not value.strip():
return None return None
key = normalize(value) key = normalize(value)
bait = db.scalar(select(Bait).where(Bait.normalized_name == key)) bait = db.scalar(select(Bait).where(Bait.normalized_name == key))
if bait is None: if bait is not None:
bait = Bait(name=value.strip(), normalized_name=key, kind=BaitKind.unknown) return bait
db.add(bait) candidate = Bait(name=value.strip(), normalized_name=key, kind=BaitKind.unknown)
return bait try:
with db.begin_nested():
db.add(candidate)
db.flush()
return candidate
except IntegrityError:
winner = db.scalar(select(Bait).where(Bait.normalized_name == key))
if winner is None:
raise
return winner
+32 -2
View File
@@ -486,7 +486,37 @@ class AdminMediaReviewOut(BaseModel):
class AdminMediaDecision(BaseModel): class AdminMediaDecision(BaseModel):
note: str = Field(min_length=1, max_length=1000) note: str = Field(min_length=1, max_length=1000)
asset_ids: list[str] = Field(min_length=1, max_length=500)
expected_version: int = Field(ge=1)
@field_validator("asset_ids")
@classmethod
def validate_asset_ids(cls, value: list[str]) -> list[str]:
if len(set(value)) != len(value) or any(len(item) != 64 or any(char not in "0123456789abcdef" for char in item) for item in value):
raise ValueError("asset_ids must contain unique SHA-256 ids")
return value
class AdminMediaRollback(AdminMediaDecision): class AdminMediaPreview(BaseModel):
asset_url: str = Field(min_length=1, max_length=2000) asset_ids: list[str] = Field(min_length=1, max_length=500)
expected_version: int | None = Field(default=None, ge=1)
@field_validator("asset_ids")
@classmethod
def validate_asset_ids(cls, value: list[str]) -> list[str]:
if len(set(value)) != len(value) or any(len(item) != 64 or any(char not in "0123456789abcdef" for char in item) for item in value):
raise ValueError("asset_ids must contain unique SHA-256 ids")
return value
class AdminMediaRollback(BaseModel):
note: str = Field(min_length=1, max_length=1000)
asset_id: str = Field(min_length=64, max_length=64)
expected_version: int = Field(ge=1)
@field_validator("asset_id")
@classmethod
def validate_asset_id(cls, value: str) -> str:
if any(char not in "0123456789abcdef" for char in value):
raise ValueError("asset_id must be a SHA-256 id")
return value
+117 -9
View File
@@ -226,6 +226,20 @@
}, },
"AdminMediaDecision": { "AdminMediaDecision": {
"properties": { "properties": {
"asset_ids": {
"items": {
"type": "string"
},
"maxItems": 500,
"minItems": 1,
"title": "Asset Ids",
"type": "array"
},
"expected_version": {
"minimum": 1.0,
"title": "Expected Version",
"type": "integer"
},
"note": { "note": {
"maxLength": 1000, "maxLength": 1000,
"minLength": 1, "minLength": 1,
@@ -234,7 +248,9 @@
} }
}, },
"required": [ "required": [
"note" "note",
"asset_ids",
"expected_version"
], ],
"title": "AdminMediaDecision", "title": "AdminMediaDecision",
"type": "object" "type": "object"
@@ -295,6 +311,36 @@
"title": "AdminMediaDerivativeOut", "title": "AdminMediaDerivativeOut",
"type": "object" "type": "object"
}, },
"AdminMediaPreview": {
"properties": {
"asset_ids": {
"items": {
"type": "string"
},
"maxItems": 500,
"minItems": 1,
"title": "Asset Ids",
"type": "array"
},
"expected_version": {
"anyOf": [
{
"minimum": 1.0,
"type": "integer"
},
{
"type": "null"
}
],
"title": "Expected Version"
}
},
"required": [
"asset_ids"
],
"title": "AdminMediaPreview",
"type": "object"
},
"AdminMediaReviewOut": { "AdminMediaReviewOut": {
"properties": { "properties": {
"asset_url": { "asset_url": {
@@ -439,12 +485,17 @@
}, },
"AdminMediaRollback": { "AdminMediaRollback": {
"properties": { "properties": {
"asset_url": { "asset_id": {
"maxLength": 2000, "maxLength": 64,
"minLength": 1, "minLength": 64,
"title": "Asset Url", "title": "Asset Id",
"type": "string" "type": "string"
}, },
"expected_version": {
"minimum": 1.0,
"title": "Expected Version",
"type": "integer"
},
"note": { "note": {
"maxLength": 1000, "maxLength": 1000,
"minLength": 1, "minLength": 1,
@@ -454,7 +505,8 @@
}, },
"required": [ "required": [
"note", "note",
"asset_url" "asset_id",
"expected_version"
], ],
"title": "AdminMediaRollback", "title": "AdminMediaRollback",
"type": "object" "type": "object"
@@ -4365,6 +4417,64 @@
"summary": "Admin Media Catalog" "summary": "Admin Media Catalog"
} }
}, },
"/api/v1/admin/media/upgrades/preview": {
"post": {
"operationId": "admin_preview_media_upgrades_api_v1_admin_media_upgrades_preview_post",
"parameters": [
{
"in": "header",
"name": "authorization",
"required": false,
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Authorization"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AdminMediaPreview"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"additionalProperties": true,
"title": "Response Admin Preview Media Upgrades Api V1 Admin Media Upgrades Preview Post",
"type": "object"
}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"summary": "Admin Preview Media Upgrades"
}
},
"/api/v1/admin/media/upgrades/publish": { "/api/v1/admin/media/upgrades/publish": {
"post": { "post": {
"description": "Atomically publish all stored quality upgrades after an explicit decision.", "description": "Atomically publish all stored quality upgrades after an explicit decision.",
@@ -4463,9 +4573,7 @@
"content": { "content": {
"application/json": { "application/json": {
"schema": { "schema": {
"additionalProperties": { "additionalProperties": true,
"type": "string"
},
"title": "Response Admin Rollback Media Upgrade Api V1 Admin Media Upgrades Rollback Post", "title": "Response Admin Rollback Media Upgrade Api V1 Admin Media Upgrades Rollback Post",
"type": "object" "type": "object"
} }
+60 -9
View File
@@ -207,29 +207,39 @@ def test_admin_media_review_requires_auth() -> None:
def test_admin_media_decisions_require_auth_and_note(monkeypatch) -> None: def test_admin_media_decisions_require_auth_and_note(monkeypatch) -> None:
assert client.post("/api/v1/admin/media/upgrades/publish", json={"note": "publish"}).status_code == 401 decision = {"asset_ids": ["a" * 64], "expected_version": 1, "note": "publish"}
assert client.post("/api/v1/admin/media/upgrades/rollback", json={"asset_url": "https://example.test/a", "note": "rollback"}).status_code == 401 assert client.post("/api/v1/admin/media/upgrades/publish", json=decision).status_code == 401
assert client.post("/api/v1/admin/media/upgrades/rollback", json={"asset_id": "a" * 64, "expected_version": 1, "note": "rollback"}).status_code == 401
monkeypatch.setattr(admin_router, "publish_quality_upgrades", lambda path, note: {"published": 2, "retained_fallbacks": 2}) monkeypatch.setattr(admin_router, "preview_quality_upgrades", lambda path, asset_ids, expected_version: {"manifest_version": expected_version or 1, "assets": [{"id": asset_ids[0]}]})
preview = client.post(
"/api/v1/admin/media/upgrades/preview",
json={"asset_ids": ["a" * 64], "expected_version": 2},
headers={"Authorization": "Bearer change-me-in-production"},
)
assert preview.status_code == 200
assert preview.json()["manifest_version"] == 2
monkeypatch.setattr(admin_router, "publish_quality_upgrades", lambda path, note, asset_ids, expected_version: {"published": 2, "retained_fallbacks": 2, "manifest_version": expected_version + 1})
publish = client.post( publish = client.post(
"/api/v1/admin/media/upgrades/publish", "/api/v1/admin/media/upgrades/publish",
json={"note": "visual review complete"}, json={"asset_ids": ["a" * 64], "expected_version": 3, "note": "visual review complete"},
headers={"Authorization": "Bearer change-me-in-production"}, headers={"Authorization": "Bearer change-me-in-production"},
) )
assert publish.status_code == 200 assert publish.status_code == 200
assert publish.json() == {"published": 2, "retained_fallbacks": 2} assert publish.json() == {"published": 2, "retained_fallbacks": 2, "manifest_version": 4}
monkeypatch.setattr(admin_router, "rollback_quality_upgrade", lambda path, asset_url, note: {"rolled_back": asset_url, "restored": "https://example.test/fallback"}) monkeypatch.setattr(admin_router, "rollback_quality_upgrade", lambda path, asset_id, note, expected_version: {"rolled_back": asset_id, "restored": "https://example.test/fallback", "manifest_version": expected_version + 1})
rollback = client.post( rollback = client.post(
"/api/v1/admin/media/upgrades/rollback", "/api/v1/admin/media/upgrades/rollback",
json={"asset_url": "https://example.test/a", "note": "fallback is preferred"}, json={"asset_id": "a" * 64, "expected_version": 4, "note": "fallback is preferred"},
headers={"Authorization": "Bearer change-me-in-production"}, headers={"Authorization": "Bearer change-me-in-production"},
) )
assert rollback.status_code == 200 assert rollback.status_code == 200
assert rollback.json()["rolled_back"] == "https://example.test/a" assert rollback.json()["rolled_back"] == "a" * 64
assert client.post( assert client.post(
"/api/v1/admin/media/upgrades/publish", "/api/v1/admin/media/upgrades/publish",
json={"note": ""}, json={"asset_ids": ["a" * 64], "expected_version": 1, "note": ""},
headers={"Authorization": "Bearer change-me-in-production"}, headers={"Authorization": "Bearer change-me-in-production"},
).status_code == 422 ).status_code == 422
@@ -528,6 +538,47 @@ def test_pending_report_accepts_one_validated_screenshot(monkeypatch) -> None:
assert reused.status_code == 401 assert reused.status_code == 401
def test_idempotency_replay_survives_completed_screenshot(monkeypatch) -> None:
import uuid
monkeypatch.setattr("app.routers.submissions.check_rate_limit", lambda *args, **kwargs: None)
key = f"idem-upload-{uuid.uuid4().hex}"
payload = {"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 94, "y": 95, "weight_g": 4300}
created = client.post("/api/v1/catch-reports", json=payload, headers={"Idempotency-Key": key}).json()
monkeypatch.setattr("app.routers.submissions.upload_screenshot", lambda raw, **metadata: "reports/idempotent.jpg")
upload = client.post(
f"/api/v1/catch-reports/{created['id']}/screenshot",
headers={"X-Upload-Token": created["screenshot_upload_token"]},
files={"screenshot": ("catch.jpg", b"image-bytes", "image/jpeg")},
)
assert upload.status_code == 204
replay = client.post("/api/v1/catch-reports", json=payload, headers={"Idempotency-Key": key})
assert replay.status_code == 200
assert replay.json()["id"] == created["id"]
assert replay.json()["screenshot_upload_token"] == created["screenshot_upload_token"]
assert client.post("/api/v1/catch-reports", json=dict(payload, weight_g=4301), headers={"Idempotency-Key": key}).status_code == 409
def test_expired_idempotency_key_integrity_winner_is_not_replayed(monkeypatch) -> None:
import uuid
monkeypatch.setattr("app.routers.submissions.check_rate_limit", lambda *args, **kwargs: None)
key = f"idem-expired-{uuid.uuid4().hex}"
payload = {"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 96, "y": 97, "weight_g": 4400}
created = client.post("/api/v1/catch-reports", json=payload, headers={"Idempotency-Key": key})
assert created.status_code == 201
with Session(engine) as db:
attempt = db.scalar(select(SubmissionAttempt).where(SubmissionAttempt.catch_report_id == UUID(created.json()["id"])))
assert attempt is not None
attempt.created_at = datetime.now(timezone.utc) - timedelta(minutes=6)
db.commit()
expired = client.post("/api/v1/catch-reports", json=payload, headers={"Idempotency-Key": key})
assert expired.status_code == 409
assert "expired" in expired.json()["detail"]
def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None: def test_admin_delete_anonymizes_report_removes_screenshot_and_keeps_audit(monkeypatch) -> None:
created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 93, "y": 94, "weight_g": 4300, "player_name": "Private Player", "source_url": "https://example.test/private", "comment": "private comment"}).json() created = client.post("/api/v1/catch-reports", json={"fish_slug": "pike", "waterbody_slug": "test-lake", "x": 93, "y": 94, "weight_g": 4300, "player_name": "Private Player", "source_url": "https://example.test/private", "comment": "private comment"}).json()
with Session(engine) as db: with Session(engine) as db:
+5 -5
View File
@@ -34,7 +34,7 @@ const apiUrl = import.meta.env.PUBLIC_API_URL || "http://localhost:8000";
const roleFilter = filters?.querySelector<HTMLSelectElement>('[name="media_role"]'); const roleFilter = filters?.querySelector<HTMLSelectElement>('[name="media_role"]');
const rolesByEntity: Record<string, Set<string>> = { waterbody: new Set(["waterbody_cover", "waterbody_map", "waterbody_depth_map", "waterbody_screenshot"]), tackle: new Set(["tackle_card", "tackle_detail", "rig_diagram", "tackle_screenshot"]) }; const rolesByEntity: Record<string, Set<string>> = { waterbody: new Set(["waterbody_cover", "waterbody_map", "waterbody_depth_map", "waterbody_screenshot"]), tackle: new Set(["tackle_card", "tackle_detail", "rig_diagram", "tackle_screenshot"]) };
const syncRoleFilter = () => { const allowed = rolesByEntity[entityFilter?.value ?? ""] ?? new Set<string>(); if (roleFilter) { [...roleFilter.options].forEach(option => { if (!option.value) return; option.hidden = !allowed.has(option.value); option.disabled = !allowed.has(option.value); }); if (roleFilter.value && !allowed.has(roleFilter.value)) roleFilter.value = ""; roleFilter.disabled = allowed.size === 0; } }; const syncRoleFilter = () => { const allowed = rolesByEntity[entityFilter?.value ?? ""] ?? new Set<string>(); if (roleFilter) { [...roleFilter.options].forEach(option => { if (!option.value) return; option.hidden = !allowed.has(option.value); option.disabled = !allowed.has(option.value); }); if (roleFilter.value && !allowed.has(roleFilter.value)) roleFilter.value = ""; roleFilter.disabled = allowed.size === 0; } };
let token = ""; let offset = 0; let timer: ReturnType<typeof setTimeout> | undefined; let token = ""; let offset = 0; let manifestVersion = 1; let timer: ReturnType<typeof setTimeout> | undefined;
const esc = (value: unknown) => String(value ?? "—").replace(/[&<>'"]/g, char => ({"&":"&amp;","<":"&lt;",">":"&gt;","'":"&#39;",'"':"&quot;"}[char] ?? char)); const esc = (value: unknown) => String(value ?? "—").replace(/[&<>'"]/g, char => ({"&":"&amp;","<":"&lt;",">":"&gt;","'":"&#39;",'"':"&quot;"}[char] ?? char));
const url = (value: unknown) => { try { const parsed = new URL(String(value), root?.dataset.apiUrl); return (parsed.protocol === "http:" || parsed.protocol === "https:") && !parsed.username && !parsed.password && parsed.hostname ? esc(parsed.href) : ""; } catch { return ""; } }; const url = (value: unknown) => { try { const parsed = new URL(String(value), root?.dataset.apiUrl); return (parsed.protocol === "http:" || parsed.protocol === "https:") && !parsed.username && !parsed.password && parsed.hostname ? esc(parsed.href) : ""; } catch { return ""; } };
const fail = (message: string) => { if (status) status.hidden = true; if (error) { error.textContent = message; error.hidden = false; } }; const fail = (message: string) => { if (status) status.hidden = true; if (error) { error.textContent = message; error.hidden = false; } };
@@ -48,13 +48,13 @@ const apiUrl = import.meta.env.PUBLIC_API_URL || "http://localhost:8000";
const values = new FormData(filters); const params = new URLSearchParams({limit:"51", offset:String(offset)}); for (const key of ["q", "entity_type", "media_role", "status"]) { const value = String(values.get(key) || "").trim(); if (value) params.set(key, value); } const values = new FormData(filters); const params = new URLSearchParams({limit:"51", offset:String(offset)}); for (const key of ["q", "entity_type", "media_role", "status"]) { const value = String(values.get(key) || "").trim(); if (value) params.set(key, value); }
const response = await fetch(`${root.dataset.apiUrl}/api/v1/admin/media/catalog?${params}`, {headers:{Authorization:`Bearer ${token}`} }); const response = await fetch(`${root.dataset.apiUrl}/api/v1/admin/media/catalog?${params}`, {headers:{Authorization:`Bearer ${token}`} });
if (!response.ok) { if (adminEndsSession(response.status)) endSession(); throw new Error(adminErrorMessage(response.status, "Не удалось загрузить медиатеку.")); } if (!response.ok) { if (adminEndsSession(response.status)) endSession(); throw new Error(adminErrorMessage(response.status, "Не удалось загрузить медиатеку.")); }
const rows: Record<string, unknown>[] = await response.json(); keepSession(); list.removeAttribute("aria-busy"); manifestVersion = Number(response.headers.get("X-Media-Manifest-Version") || 1); const rows: Record<string, unknown>[] = await response.json(); keepSession(); list.removeAttribute("aria-busy");
if (!rows.length && offset > 0) { offset = 0; return load(); } if (!rows.length && offset > 0) { offset = 0; return load(); }
const assets = rows.slice(0, 50); if (pages) pages.hidden = !assets.length; if (previous) previous.disabled = offset === 0; if (next) next.disabled = rows.length <= 50; if (pageNumber) pageNumber.textContent = `Страница ${offset / 50 + 1}`; const assets = rows.slice(0, 50); if (pages) pages.hidden = !assets.length; if (previous) previous.disabled = offset === 0; if (next) next.disabled = rows.length <= 50; if (pageNumber) pageNumber.textContent = `Страница ${offset / 50 + 1}`;
if (!assets.length) { list.innerHTML = '<div class="state"><h2>Кандидатов нет</h2><p>Для выбранных фильтров нет approved или upgrade_queued файлов.</p></div>'; return; } if (!assets.length) { list.innerHTML = '<div class="state"><h2>Кандидатов нет</h2><p>Для выбранных фильтров нет approved или upgrade_queued файлов.</p></div>'; return; }
list.innerHTML = assets.map(asset => { const image = url(asset.image_url); const source = url(asset.source_url); const role = String(asset.media_role || ""); const variants = (asset.derivatives as Record<string, unknown>[] ?? []).map(item => `${esc(item.format)} ${esc(item.width)}×${esc(item.height)}`).join(", "); const rollback = asset.status === "approved" && asset.supersedes ? `<button data-media-rollback="${esc(asset.asset_url)}" type="button" data-action="secondary">Откатить замену</button>` : ""; return `<article class="media-library__card"${role ? ` data-media-role="${esc(role)}"` : ""}><a href="${image}" target="_blank" rel="noreferrer">${image ? `<img src="${image}" alt="${esc(asset.label)}" loading="lazy" />` : ""}</a><h2>${esc(asset.label)}</h2><span>${esc(asset.status)} · ${esc(asset.entity_type)} · ${esc(asset.width)}×${esc(asset.height)}</span>${role ? `<small>${esc(mediaRoleLabel(role))}</small>` : ""}<p>${esc(asset.source_system)}${asset.duplicate_of ? ` · duplicate_of ${esc(asset.duplicate_of)}` : ""}</p>${variants ? `<small>Производные: ${variants}</small>` : "<small>Производных нет</small>"}${source ? `<a href="${source}" target="_blank" rel="noreferrer">Первоисточник →</a>` : ""}${rollback}</article>`; }).join(""); list.innerHTML = assets.map(asset => { const image = url(asset.image_url); const source = url(asset.source_url); const role = String(asset.media_role || ""); const variants = (asset.derivatives as Record<string, unknown>[] ?? []).map(item => `${esc(item.format)} ${esc(item.width)}×${esc(item.height)}`).join(", "); const rollback = asset.status === "approved" && asset.supersedes ? `<button data-media-rollback="${esc(asset.id)}" type="button" data-action="secondary">Откатить замену</button>` : ""; return `<article class="media-library__card"${role ? ` data-media-role="${esc(role)}"` : ""}><a href="${image}" target="_blank" rel="noreferrer">${image ? `<img src="${image}" alt="${esc(asset.label)}" loading="lazy" />` : ""}</a><h2>${esc(asset.label)}</h2><span>${esc(asset.status)} · ${esc(asset.entity_type)} · ${esc(asset.width)}×${esc(asset.height)}</span>${role ? `<small>${esc(mediaRoleLabel(role))}</small>` : ""}<p>${esc(asset.source_system)}${asset.duplicate_of ? ` · duplicate_of ${esc(asset.duplicate_of)}` : ""}</p>${variants ? `<small>Производные: ${variants}</small>` : "<small>Производных нет</small>"}${source ? `<a href="${source}" target="_blank" rel="noreferrer">Первоисточник →</a>` : ""}${rollback}</article>`; }).join("");
list.querySelectorAll<HTMLButtonElement>("[data-media-rollback]").forEach(button => button.addEventListener("click", async () => { const note = window.prompt("Причина отката:"); if (!note?.trim()) return; button.disabled = true; try { const response = await fetch(`${root?.dataset.apiUrl}/api/v1/admin/media/upgrades/rollback`, {method:"POST", headers:{Authorization:`Bearer ${token}`, "Content-Type":"application/json"}, body:JSON.stringify({asset_url:button.dataset.mediaRollback, note:note.trim()})}); if (!response.ok) throw new Error(adminErrorMessage(response.status, "Не удалось выполнить откат.")); succeed("Fallback восстановлен."); await load(); } catch (cause) { fail(cause instanceof Error ? cause.message : "Не удалось выполнить откат."); } finally { button.disabled = false; } })); list.querySelectorAll<HTMLButtonElement>("[data-media-rollback]").forEach(button => button.addEventListener("click", async () => { const note = window.prompt("Причина отката:"); if (!note?.trim()) return; button.disabled = true; try { const response = await fetch(`${root?.dataset.apiUrl}/api/v1/admin/media/upgrades/rollback`, {method:"POST", headers:{Authorization:`Bearer ${token}`, "Content-Type":"application/json"}, body:JSON.stringify({asset_id:button.dataset.mediaRollback, expected_version:manifestVersion, note:note.trim()})}); if (!response.ok) throw new Error(adminErrorMessage(response.status, "Не удалось выполнить откат.")); succeed("Fallback восстановлен."); await load(); } catch (cause) { fail(cause instanceof Error ? cause.message : "Не удалось выполнить откат."); } finally { button.disabled = false; } }));
if (assets.some(asset => asset.status === "upgrade_stored")) { const publish = document.createElement("button"); publish.type = "button"; publish.dataset.action = "primary"; publish.textContent = "Опубликовать сохранённые замены"; publish.addEventListener("click", async () => { const note = window.prompt("Причина публикации замен:"); if (!note?.trim()) return; publish.disabled = true; try { const response = await fetch(`${root?.dataset.apiUrl}/api/v1/admin/media/upgrades/publish`, {method:"POST", headers:{Authorization:`Bearer ${token}`, "Content-Type":"application/json"}, body:JSON.stringify({note:note.trim()})}); if (!response.ok) throw new Error(adminErrorMessage(response.status, "Не удалось опубликовать замены.")); succeed("Замены опубликованы атомарно, fallback сохранены."); await load(); } catch (cause) { fail(cause instanceof Error ? cause.message : "Не удалось опубликовать замены."); } finally { publish.disabled = false; } }); list.prepend(publish); } if (assets.some(asset => asset.status === "upgrade_stored")) { const publish = document.createElement("button"); publish.type = "button"; publish.dataset.action = "primary"; publish.textContent = "Опубликовать выбранные замены"; publish.addEventListener("click", async () => { const note = window.prompt("Причина публикации замен:"); if (!note?.trim()) return; publish.disabled = true; try { const asset_ids = assets.filter(asset => asset.status === "upgrade_stored").map(asset => String(asset.id)); const response = await fetch(`${root?.dataset.apiUrl}/api/v1/admin/media/upgrades/publish`, {method:"POST", headers:{Authorization:`Bearer ${token}`, "Content-Type":"application/json"}, body:JSON.stringify({asset_ids, expected_version:manifestVersion, note:note.trim()})}); if (!response.ok) throw new Error(adminErrorMessage(response.status, "Не удалось опубликовать замены.")); succeed("Выбранные замены опубликованы атомарно, fallback сохранены."); await load(); } catch (cause) { fail(cause instanceof Error ? cause.message : "Не удалось опубликовать замены."); } finally { publish.disabled = false; } }); list.prepend(publish); }
} }
login?.addEventListener("submit", async event => { event.preventDefault(); token = String(new FormData(login).get("token") || ""); offset = 0; try { await load(); login.hidden = true; if (sessionBar) sessionBar.hidden = false; if (filters) filters.hidden = false; } catch (cause) { list && (list.innerHTML = ""); fail(cause instanceof Error ? cause.message : "Ошибка загрузки."); } }); login?.addEventListener("submit", async event => { event.preventDefault(); token = String(new FormData(login).get("token") || ""); offset = 0; try { await load(); login.hidden = true; if (sessionBar) sessionBar.hidden = false; if (filters) filters.hidden = false; } catch (cause) { list && (list.innerHTML = ""); fail(cause instanceof Error ? cause.message : "Ошибка загрузки."); } });
filters?.addEventListener("submit", async event => { event.preventDefault(); offset = 0; try { await load(); } catch (cause) { fail(cause instanceof Error ? cause.message : "Ошибка фильтрации."); } }); filters?.addEventListener("submit", async event => { event.preventDefault(); offset = 0; try { await load(); } catch (cause) { fail(cause instanceof Error ? cause.message : "Ошибка фильтрации."); } });
+18
View File
@@ -123,6 +123,7 @@ services:
S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY}
S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY}
S3_BUCKET: ${S3_BUCKET:-catch-screenshots} S3_BUCKET: ${S3_BUCKET:-catch-screenshots}
MEDIA_ROOT: /app/runtime-media
OFFICIAL_RECORDS_URL: ${OFFICIAL_RECORDS_URL:-https://rf4game.de/records/region/RU/} OFFICIAL_RECORDS_URL: ${OFFICIAL_RECORDS_URL:-https://rf4game.de/records/region/RU/}
OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU} OFFICIAL_RECORDS_REGION: ${OFFICIAL_RECORDS_REGION:-RU}
OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records} OFFICIAL_RECORDS_CATEGORY: ${OFFICIAL_RECORDS_CATEGORY:-records}
@@ -144,6 +145,7 @@ services:
migrate: {condition: service_completed_successfully} migrate: {condition: service_completed_successfully}
minio: {condition: service_healthy} minio: {condition: service_healthy}
minio-init: {condition: service_completed_successfully} minio-init: {condition: service_completed_successfully}
media-init: {condition: service_completed_successfully}
healthcheck: healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/ready')"] test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/ready')"]
interval: 10s interval: 10s
@@ -152,6 +154,8 @@ services:
networks: [backend, edge] networks: [backend, edge]
security_opt: [no-new-privileges:true] security_opt: [no-new-privileges:true]
logging: *default-logging logging: *default-logging
volumes:
- media_data:/app/runtime-media
deploy: deploy:
resources: resources:
limits: {cpus: "1.00", memory: 1G} limits: {cpus: "1.00", memory: 1G}
@@ -173,6 +177,19 @@ services:
resources: resources:
limits: {cpus: "0.50", memory: 256M} limits: {cpus: "0.50", memory: 256M}
media-init:
build:
context: .
dockerfile: apps/api/Dockerfile
restart: "no"
user: "0:0"
command: ["sh", "-c", "python -c \"from pathlib import Path; from rf4_research.media_store import bootstrap_media_store; print(bootstrap_media_store(Path('/app/data/media'), Path('/app/runtime-media')))\" && chown -R 10001:10001 /app/runtime-media"]
volumes:
- media_data:/app/runtime-media
networks: [backend]
security_opt: [no-new-privileges:true]
logging: *default-logging
web: web:
build: build:
context: ./apps/web context: ./apps/web
@@ -249,5 +266,6 @@ networks:
volumes: volumes:
postgres_data: postgres_data:
minio_data: minio_data:
media_data:
caddy_data: caddy_data:
caddy_config: caddy_config:
+3 -3
View File
@@ -116,20 +116,20 @@ curl -fsS https://rf4spotter.ru/ready
## 6. Резервное копирование и восстановление ## 6. Резервное копирование и восстановление
Храните копии вне диска приложения. Скрипт создаёт PostgreSQL dump, архив MinIO и контрольные суммы в новом каталоге с UTC-временем: Храните копии вне диска приложения. Скрипт создаёт PostgreSQL dump, архив MinIO, архив persistent media store и контрольные суммы в новом каталоге с UTC-временем. Media manifest и файлы решений восстанавливаются вместе с provenance:
```bash ```bash
./deploy/backup.sh /srv/rf4-backups ./deploy/backup.sh /srv/rf4-backups
``` ```
Восстановление заменяет содержимое PostgreSQL и MinIO данными из выбранной копии, временно останавливая API и web. Это намеренно защищённая подтверждением операция: Восстановление заменяет содержимое PostgreSQL, MinIO и persistent media store данными из выбранной копии, временно останавливая API и web. Это намеренно защищённая подтверждением операция:
```bash ```bash
CONFIRM_RESTORE=rf4-spotter ./deploy/restore.sh /srv/rf4-backups/20260906T120000Z CONFIRM_RESTORE=rf4-spotter ./deploy/restore.sh /srv/rf4-backups/20260906T120000Z
curl -fsS https://rf4spotter.ru/ready curl -fsS https://rf4spotter.ru/ready
``` ```
Репозиторий содержит изолированный drill, который создаёт отдельные Compose volumes, портит тестовые данные, восстанавливает их и сравнивает PostgreSQL и MinIO: Репозиторий содержит изолированный drill, который создаёт отдельные Compose volumes, портит тестовые данные, восстанавливает их и сравнивает PostgreSQL, MinIO и media store:
```bash ```bash
./deploy/test-backup-restore.sh ./deploy/test-backup-restore.sh
+7 -1
View File
@@ -21,6 +21,9 @@ fi
mkdir -p "$target" mkdir -p "$target"
chmod 700 "$target" chmod 700 "$target"
export BACKUP_DIRECTORY="$backup_root" export BACKUP_DIRECTORY="$backup_root"
media_tmp=$(mktemp -d "$backup_root/.rf4-media-backup.XXXXXX")
cleanup() { rm -rf "$media_tmp"; }
trap cleanup EXIT INT TERM
env_file=${COMPOSE_ENV_FILE:-.env.production} env_file=${COMPOSE_ENV_FILE:-.env.production}
compose="docker compose --env-file $env_file -f compose.production.yaml" compose="docker compose --env-file $env_file -f compose.production.yaml"
@@ -28,6 +31,9 @@ $compose exec -T db sh -c 'pg_dump --format=custom --no-owner --no-acl -U "$POST
$compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null && mc stat "local/$S3_BUCKET" >/dev/null && mc mirror --overwrite "local/$S3_BUCKET" "/backup/'"$stamp"'/minio" && chmod -R a+rwX "/backup/'"$stamp"'/minio"' $compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null && mc stat "local/$S3_BUCKET" >/dev/null && mc mirror --overwrite "local/$S3_BUCKET" "/backup/'"$stamp"'/minio" && chmod -R a+rwX "/backup/'"$stamp"'/minio"'
tar -czf "$target/minio.tar.gz" -C "$target" minio tar -czf "$target/minio.tar.gz" -C "$target" minio
rm -rf "$target/minio" rm -rf "$target/minio"
(cd "$target" && sha256sum postgres.dump minio.tar.gz > SHA256SUMS) $compose cp api:/app/runtime-media/. "$media_tmp/"
tar -czf "$target/media.tar.gz" -C "$media_tmp" .
(cd "$target" && sha256sum postgres.dump minio.tar.gz media.tar.gz > SHA256SUMS)
chmod 600 "$target/postgres.dump" "$target/minio.tar.gz" "$target/SHA256SUMS" chmod 600 "$target/postgres.dump" "$target/minio.tar.gz" "$target/SHA256SUMS"
chmod 600 "$target/media.tar.gz"
echo "Backup created: $target" echo "Backup created: $target"
+4
View File
@@ -14,12 +14,15 @@ esac
source_dir=${1%/} source_dir=${1%/}
test -f "$source_dir/postgres.dump" test -f "$source_dir/postgres.dump"
test -f "$source_dir/minio.tar.gz" test -f "$source_dir/minio.tar.gz"
test -f "$source_dir/media.tar.gz"
(cd "$source_dir" && sha256sum -c SHA256SUMS) (cd "$source_dir" && sha256sum -c SHA256SUMS)
backup_root=$(dirname "$source_dir") backup_root=$(dirname "$source_dir")
export BACKUP_DIRECTORY="$backup_root" export BACKUP_DIRECTORY="$backup_root"
restore_tmp=$(mktemp -d "$backup_root/.rf4-restore.XXXXXX") restore_tmp=$(mktemp -d "$backup_root/.rf4-restore.XXXXXX")
trap 'rm -rf "$restore_tmp"' EXIT INT TERM trap 'rm -rf "$restore_tmp"' EXIT INT TERM
tar -xzf "$source_dir/minio.tar.gz" -C "$restore_tmp" tar -xzf "$source_dir/minio.tar.gz" -C "$restore_tmp"
mkdir -p "$restore_tmp/media"
tar -xzf "$source_dir/media.tar.gz" -C "$restore_tmp/media"
restore_name=$(basename "$restore_tmp") restore_name=$(basename "$restore_tmp")
env_file=${COMPOSE_ENV_FILE:-.env.production} env_file=${COMPOSE_ENV_FILE:-.env.production}
@@ -27,5 +30,6 @@ compose="docker compose --env-file $env_file -f compose.production.yaml"
$compose stop api web $compose stop api web
$compose exec -T db sh -c 'pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$source_dir/postgres.dump" $compose exec -T db sh -c 'pg_restore --clean --if-exists --exit-on-error --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$source_dir/postgres.dump"
$compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null && mc stat "local/$S3_BUCKET" >/dev/null && mc rm --recursive --force "local/$S3_BUCKET" >/dev/null 2>&1 && mc mirror --overwrite "/backup/'"$restore_name"'/minio" "local/$S3_BUCKET"' $compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null && mc stat "local/$S3_BUCKET" >/dev/null && mc rm --recursive --force "local/$S3_BUCKET" >/dev/null 2>&1 && mc mirror --overwrite "/backup/'"$restore_name"'/minio" "local/$S3_BUCKET"'
tar -cf - -C "$restore_tmp/media" . | $compose run --rm --no-deps -T api sh -c 'find /app/runtime-media -mindepth 1 -delete && tar -xf - -C /app/runtime-media'
$compose up -d ${RESTORE_START_SERVICES:-minio api web proxy} $compose up -d ${RESTORE_START_SERVICES:-minio api web proxy}
echo "Restore completed from: $source_dir" echo "Restore completed from: $source_dir"
+8 -1
View File
@@ -8,6 +8,8 @@ project="rf4-restore-drill-$$"
export COMPOSE_PROJECT_NAME="$project" export COMPOSE_PROJECT_NAME="$project"
export COMPOSE_ENV_FILE=.env.production.example export COMPOSE_ENV_FILE=.env.production.example
export BACKUP_DIRECTORY="$drill_root" export BACKUP_DIRECTORY="$drill_root"
export POSTGRES_PASSWORD=restore-drill-postgres-password
export DATABASE_URL=postgresql+psycopg://rf4:restore-drill-postgres-password@db:5432/rf4_spotter
compose="docker compose --env-file $COMPOSE_ENV_FILE -f compose.production.yaml" compose="docker compose --env-file $COMPOSE_ENV_FILE -f compose.production.yaml"
cleanup() { cleanup() {
@@ -18,8 +20,11 @@ trap cleanup EXIT INT TERM
$compose up -d --wait db minio $compose up -d --wait db minio
$compose run --rm minio-init $compose run --rm minio-init
$compose run --rm media-init
$compose up -d --wait api
$compose exec -T db sh -c 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "create table restore_drill (value text primary key); insert into restore_drill values ('"'"'database-ok'"'"');"' >/dev/null $compose exec -T db sh -c 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "create table restore_drill (value text primary key); insert into restore_drill values ('"'"'database-ok'"'"');"' >/dev/null
$compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null; mc stat "local/$S3_BUCKET" >/dev/null; printf object-ok | mc pipe "local/$S3_BUCKET/restore-drill.txt"' >/dev/null $compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null; mc stat "local/$S3_BUCKET" >/dev/null; printf object-ok | mc pipe "local/$S3_BUCKET/restore-drill.txt"' >/dev/null
$compose exec -T api sh -c 'printf media-ok > /app/runtime-media/restore-drill.txt'
./deploy/backup.sh "$drill_root" ./deploy/backup.sh "$drill_root"
backup=$(find "$drill_root" -mindepth 1 -maxdepth 1 -type d ! -name '.rf4-restore.*' | head -n 1) backup=$(find "$drill_root" -mindepth 1 -maxdepth 1 -type d ! -name '.rf4-restore.*' | head -n 1)
@@ -29,6 +34,8 @@ $compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http:
CONFIRM_RESTORE=rf4-spotter RESTORE_START_SERVICES=minio ./deploy/restore.sh "$backup" CONFIRM_RESTORE=rf4-spotter RESTORE_START_SERVICES=minio ./deploy/restore.sh "$backup"
db_value=$($compose exec -T db sh -c 'psql -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select value from restore_drill;"') db_value=$($compose exec -T db sh -c 'psql -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select value from restore_drill;"')
object_value=$($compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null; mc cat "local/$S3_BUCKET/restore-drill.txt"') object_value=$($compose run --rm --entrypoint /bin/sh storage-tool -c 'mc alias set local http://minio:9000 "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null; mc cat "local/$S3_BUCKET/restore-drill.txt"')
media_value=$($compose run --rm --no-deps api sh -c 'cat /app/runtime-media/restore-drill.txt')
test "$db_value" = "database-ok" test "$db_value" = "database-ok"
test "$object_value" = "object-ok" test "$object_value" = "object-ok"
echo "Backup/restore drill passed for PostgreSQL and MinIO" test "$media_value" = "media-ok"
echo "Backup/restore drill passed for PostgreSQL, MinIO, and media decisions"
+4
View File
@@ -24,6 +24,10 @@ trap cleanup EXIT INT TERM
$compose up --build -d --wait db minio api web $compose up --build -d --wait db minio api web
curl -fsS "http://127.0.0.1:$BOOTSTRAP_API_PORT/ready" >/dev/null curl -fsS "http://127.0.0.1:$BOOTSTRAP_API_PORT/ready" >/dev/null
curl -fsS "http://127.0.0.1:$BOOTSTRAP_WEB_PORT/" >/dev/null curl -fsS "http://127.0.0.1:$BOOTSTRAP_WEB_PORT/" >/dev/null
manifest_digest=$($compose exec -T api sha256sum /app/runtime-media/manifest.json | awk '{print $1}')
test -n "$manifest_digest"
$compose up -d --force-recreate --wait api
test "$manifest_digest" = "$($compose exec -T api sha256sum /app/runtime-media/manifest.json | awk '{print $1}')"
curl -fsS -D - -o /dev/null "http://127.0.0.1:$BOOTSTRAP_API_PORT/health" | grep -qi '^x-frame-options: DENY' curl -fsS -D - -o /dev/null "http://127.0.0.1:$BOOTSTRAP_API_PORT/health" | grep -qi '^x-frame-options: DENY'
curl -fsS -D - -o /dev/null "http://127.0.0.1:$BOOTSTRAP_API_PORT/health" | grep -qi '^cross-origin-opener-policy: same-origin' curl -fsS -D - -o /dev/null "http://127.0.0.1:$BOOTSTRAP_API_PORT/health" | grep -qi '^cross-origin-opener-policy: same-origin'
+3 -3
View File
@@ -23,9 +23,9 @@ R-пункты уточняют критерии существующих B/G/U/
их частичным выполнением. Коммит после каждого действительно закрытого пункта. их частичным выполнением. Коммит после каждого действительно закрытого пункта.
Доказательства и ограничения: [аудит](PROJECT_AUDIT_2026-09-22.md). Доказательства и ограничения: [аудит](PROJECT_AUDIT_2026-09-22.md).
- [ ] **R01 · P1 · Долговременное хранение media-решений (A04/B23/A07).** Persistent store для manifest/assets, bootstrap из Git, release merge и backup/restore. Критерий: publish/rollback переживают recreate и восстановление вместе с provenance. - [x] **R01 · P1 · Долговременное хранение media-решений (A04/B23/A07).** Production Compose получил named `media_data` volume и отдельный root-only `media-init`: baseline из Git атомарно merge-ится только для новых assets, существующие решения/provenance не заменяются, volume передаётся API-пользователю. `backup.sh` сохраняет persistent manifest/assets в `media.tar.gz`, `restore.sh` восстанавливает их вместе с PostgreSQL и MinIO. Изолированный bootstrap подтвердил сохранение manifest после `--force-recreate` API; backup/restore drill подтвердил checksum и восстановление контрольного media-решения. Production secrets, внешнее backup-хранилище и серверный A07 acceptance-run остаются открытыми.
- [ ] **R02 · P1 · Публикация выбранных media и защита от гонок (A04/B23).** Явные asset IDs, preview, версия manifest, lock и проверка файлов. Критерий: скрытые кандидаты не публикуются, устаревшее решение — конфликт, параллельные CLI/API не теряют изменения. - [x] **R02 · P1 · Публикация выбранных media и защита от гонок (A04/B23).** Admin API получил preview и явный набор SHA-256 asset IDs; publish больше не выбирает скрытые `upgrade_stored`, проверяет целостность выбранных файлов/вариантов, сверяет monotonic manifest version и возвращает `409` для устаревшей вкладки. Publish/rollback и CLI-совместимый путь сериализованы общим lock-файлом; решения пишутся атомарно и увеличивают версию. Regression покрывает выбор одного кандидата из двух и stale-version conflict; web admin передаёт ID и версию из заголовка manifest. Browser/production acceptance A04/A06/A07 остаётся отдельным gate.
- [ ] **R03 · P1 · Идемпотентная отправка улова.** Сверять payload/срок ключа во всех ветках, включая IntegrityError и завершённый upload; обработать конкурентное создание Spot/Bait. Критерий: одинаковый повтор возвращает тот же результат, другой payload — 409, ожидаемые гонки не дают 500. - [x] **R03 · P1 · Идемпотентная отправка улова.** Единый replay-контракт теперь сверяет payload hash и TTL также после `IntegrityError`; старый уникальный ключ получает `409`, а завершённый screenshot больше не ломает повтор исходной отправки. Spot/Bait создаются через savepoint и повторный select после unique-конфликта, поэтому ожидаемые гонки не дают необработанный `500`. Regression покрывает одинаковый повтор, изменённый payload, завершённый upload и просроченный ключ.
- [ ] **R04 · P1 · Восстановление загрузки скриншота.** Атомарное одноразовое сохранение, повтор после потерянного ответа, очистка idempotency-cookie после восстановления. Критерий: нет лишних объектов, retry завершает форму и следующий новый улов отправляется. - [ ] **R04 · P1 · Восстановление загрузки скриншота.** Атомарное одноразовое сохранение, повтор после потерянного ответа, очистка idempotency-cookie после восстановления. Критерий: нет лишних объектов, retry завершает форму и следующий новый улов отправляется.
- [ ] **R05 · P1 · Полноценный каталог снастей (G06/G09).** Отдельная пагинация сборок/предметов, независимые ошибки, валидация URL-фильтров, empty search отдельно от незаполненного каталога. Критерий: 49+ сборок достижимы при 0 предметов, 422 не маскируется под 503. Убрать вложенные ссылки: карточка оборачивает DataPassport с source-link; проверить итоговый DOM и keyboard наполненных карточек. - [ ] **R05 · P1 · Полноценный каталог снастей (G06/G09).** Отдельная пагинация сборок/предметов, независимые ошибки, валидация URL-фильтров, empty search отдельно от незаполненного каталога. Критерий: 49+ сборок достижимы при 0 предметов, 422 не маскируется под 503. Убрать вложенные ссылки: карточка оборачивает DataPassport с source-link; проверить итоговый DOM и keyboard наполненных карточек.
- [ ] **R06 · P1 · Надёжный перенос плана (U05).** Preview, объединение/замена с восстановлением, однократный импорт, storage errors, вычисляемая свежесть и точное описание передачи share-данных. Критерий: ссылка не стирает план без выбора, reload не возвращает удалённое, старые данные отмечены. - [ ] **R06 · P1 · Надёжный перенос плана (U05).** Preview, объединение/замена с восстановлением, однократный импорт, storage errors, вычисляемая свежесть и точное описание передачи share-данных. Критерий: ссылка не стирает план без выбора, reload не возвращает удалённое, старые данные отмечены.
+135 -60
View File
@@ -5,11 +5,14 @@ import io
import json import json
import mimetypes import mimetypes
import re import re
from contextlib import contextmanager
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from urllib.parse import urljoin, urlsplit from urllib.parse import urljoin, urlsplit
import fcntl
from bs4 import BeautifulSoup, Tag from bs4 import BeautifulSoup, Tag
from PIL import Image, ImageDraw, ImageFont, UnidentifiedImageError from PIL import Image, ImageDraw, ImageFont, UnidentifiedImageError
@@ -27,6 +30,52 @@ MEDIA_ROLES_BY_ENTITY = {
} }
@contextmanager
def _manifest_lock(path: Path):
"""Serialize decisions made by API and CLI against one manifest."""
lock_path = path.with_name(f"{path.name}.lock")
lock_path.parent.mkdir(parents=True, exist_ok=True)
with lock_path.open("w", encoding="utf-8") as lock:
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
def _manifest_version(manifest: dict) -> int:
try:
return max(1, int(manifest.get("version", 1)))
except (TypeError, ValueError) as exc:
raise ValueError("manifest version is invalid") from exc
def _asset_id(item: dict) -> str | None:
value = item.get("sha256")
return value if isinstance(value, str) and len(value) == 64 else None
def _selected_upgrade_candidates(manifest: dict, asset_ids: list[str] | None) -> list[dict]:
candidates = [item for item in manifest.get("assets", []) if item.get("status") == "upgrade_stored"]
if asset_ids is None:
return candidates
selected = set(asset_ids)
by_id = {_asset_id(item): item for item in candidates}
if len(selected) != len(asset_ids):
raise ValueError("asset_ids must be unique")
missing = sorted(selected - set(by_id))
if missing:
raise ValueError(f"selected asset is not an upgrade_stored candidate: {missing[0]}")
return [by_id[asset_id] for asset_id in asset_ids]
def _check_expected_version(manifest: dict, expected_version: int | None) -> int:
version = _manifest_version(manifest)
if expected_version is not None and expected_version != version:
raise ValueError(f"manifest version conflict: expected {expected_version}, current {version}")
return version
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class MediaCandidate: class MediaCandidate:
source_page: str source_page: str
@@ -197,70 +246,96 @@ def approve_stored_assets(path: Path, *, note: str) -> dict:
return {"approved": approved} return {"approved": approved}
def publish_quality_upgrades(path: Path, *, note: str, minimum_dimension: int = 256) -> dict: def preview_quality_upgrades(path: Path, *, asset_ids: list[str], expected_version: int | None = None) -> dict:
"""Atomically promote every reviewed quality candidate and retain its fallback.""" """Return the exact selected set and version used for a later decision."""
manifest = json.loads(path.read_text(encoding="utf-8")) with _manifest_lock(path):
assets = manifest.get("assets", []) manifest = json.loads(path.read_text(encoding="utf-8"))
by_url = {item.get("asset_url"): item for item in assets} version = _check_expected_version(manifest, expected_version)
candidates = [item for item in assets if item.get("status") == "upgrade_stored"] candidates = _selected_upgrade_candidates(manifest, asset_ids)
return {
# Validate the complete set before changing any public mapping. "manifest_version": version,
for candidate in candidates: "assets": [{
fallback = by_url.get(candidate.get("duplicate_of")) "id": _asset_id(item), "asset_url": item.get("asset_url"),
if not fallback or fallback.get("status") != "approved": "duplicate_of": item.get("duplicate_of"), "label": item.get("label"),
raise ValueError(f"{candidate.get('asset_url')}: approved fallback is missing") "width": item.get("width"), "height": item.get("height"),
if not fallback.get("entity_key"): } for item in candidates],
raise ValueError(f"{candidate.get('asset_url')}: fallback has no entity_key") }
if min(int(candidate.get("width") or 0), int(candidate.get("height") or 0)) < minimum_dimension:
raise ValueError(f"{candidate.get('asset_url')}: candidate is below {minimum_dimension}px")
if not candidate.get("sha256") or not candidate.get("local_path"):
raise ValueError(f"{candidate.get('asset_url')}: stored candidate metadata is incomplete")
reviewed_at = datetime.now(timezone.utc).isoformat()
for candidate in candidates:
fallback = by_url[candidate["duplicate_of"]]
candidate.update({
"status": "approved",
"entity_key": fallback["entity_key"],
"supersedes": fallback["asset_url"],
"reviewed_at": reviewed_at,
"review_note": note,
})
fallback.update({
"status": "superseded",
"replaced_by": candidate["asset_url"],
"reviewed_at": reviewed_at,
"review_note": note,
})
manifest["updated_at"] = reviewed_at
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(path)
return {"published": len(candidates), "retained_fallbacks": len(candidates)}
def rollback_quality_upgrade(path: Path, *, asset_url: str, note: str) -> dict: def publish_quality_upgrades(
path: Path, *, note: str, asset_ids: list[str] | None = None,
expected_version: int | None = None, minimum_dimension: int = 256,
) -> dict:
"""Atomically promote an explicit reviewed set and retain its fallbacks."""
with _manifest_lock(path):
manifest = json.loads(path.read_text(encoding="utf-8"))
previous_version = _check_expected_version(manifest, expected_version)
assets = manifest.get("assets", [])
by_url = {item.get("asset_url"): item for item in assets}
candidates = _selected_upgrade_candidates(manifest, asset_ids)
for candidate in candidates:
fallback = by_url.get(candidate.get("duplicate_of"))
if not fallback or fallback.get("status") != "approved":
raise ValueError(f"{candidate.get('asset_url')}: approved fallback is missing")
if not fallback.get("entity_key"):
raise ValueError(f"{candidate.get('asset_url')}: fallback has no entity_key")
if min(int(candidate.get("width") or 0), int(candidate.get("height") or 0)) < minimum_dimension:
raise ValueError(f"{candidate.get('asset_url')}: candidate is below {minimum_dimension}px")
if not candidate.get("sha256") or not candidate.get("local_path"):
raise ValueError(f"{candidate.get('asset_url')}: stored candidate metadata is incomplete")
reviewed_at = datetime.now(timezone.utc).isoformat()
for candidate in candidates:
fallback = by_url[candidate["duplicate_of"]]
candidate.update({
"status": "approved", "entity_key": fallback["entity_key"],
"supersedes": fallback["asset_url"], "reviewed_at": reviewed_at,
"review_note": note,
})
fallback.update({
"status": "superseded", "replaced_by": candidate["asset_url"],
"reviewed_at": reviewed_at, "review_note": note,
})
manifest["version"] = previous_version + 1
manifest["updated_at"] = reviewed_at
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(path)
return {"published": len(candidates), "retained_fallbacks": len(candidates), "manifest_version": manifest["version"]}
def rollback_quality_upgrade(
path: Path, *, asset_url: str | None = None, asset_id: str | None = None,
note: str, expected_version: int | None = None,
) -> dict:
"""Restore the superseded fallback for one explicitly selected upgrade.""" """Restore the superseded fallback for one explicitly selected upgrade."""
manifest = json.loads(path.read_text(encoding="utf-8")) if not asset_id and not asset_url:
assets = manifest.get("assets", []) raise ValueError("rollback requires an asset id")
candidate = next((item for item in assets if item.get("asset_url") == asset_url), None) with _manifest_lock(path):
if not candidate or candidate.get("status") != "approved" or not candidate.get("supersedes"): manifest = json.loads(path.read_text(encoding="utf-8"))
raise ValueError("rollback requires an approved upgrade with a supersedes link") previous_version = _check_expected_version(manifest, expected_version)
fallback = next((item for item in assets if item.get("asset_url") == candidate["supersedes"]), None) assets = manifest.get("assets", [])
if not fallback or fallback.get("status") != "superseded" or fallback.get("replaced_by") != asset_url: candidate = next((item for item in assets if (asset_id and _asset_id(item) == asset_id) or (asset_url and item.get("asset_url") == asset_url)), None)
raise ValueError("rollback requires the matching superseded fallback") if not candidate or candidate.get("status") != "approved" or not candidate.get("supersedes"):
raise ValueError("rollback requires an approved upgrade with a supersedes link")
selected_url = candidate.get("asset_url")
fallback = next((item for item in assets if item.get("asset_url") == candidate["supersedes"]), None)
if not fallback or fallback.get("status") != "superseded" or fallback.get("replaced_by") != selected_url:
raise ValueError("rollback requires the matching superseded fallback")
reviewed_at = datetime.now(timezone.utc).isoformat() reviewed_at = datetime.now(timezone.utc).isoformat()
candidate.update({"status": "upgrade_stored", "reviewed_at": reviewed_at, "review_note": note}) candidate.update({"status": "upgrade_stored", "reviewed_at": reviewed_at, "review_note": note})
candidate.pop("supersedes", None) candidate.pop("supersedes", None)
fallback.update({"status": "approved", "reviewed_at": reviewed_at, "review_note": note}) fallback.update({"status": "approved", "reviewed_at": reviewed_at, "review_note": note})
fallback.pop("replaced_by", None) fallback.pop("replaced_by", None)
manifest["updated_at"] = reviewed_at manifest["version"] = previous_version + 1
temporary = path.with_suffix(path.suffix + ".tmp") manifest["updated_at"] = reviewed_at
temporary.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") temporary = path.with_suffix(path.suffix + ".tmp")
temporary.replace(path) temporary.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return {"rolled_back": asset_url, "restored": fallback["asset_url"]} temporary.replace(path)
return {"rolled_back": selected_url, "restored": fallback["asset_url"], "manifest_version": manifest["version"]}
def reclassify_manifest(path: Path) -> dict: def reclassify_manifest(path: Path) -> dict:
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
import json
import shutil
from datetime import datetime, timezone
from pathlib import Path
def _safe_path(root: Path, relative: str) -> Path | None:
target = (root / relative).resolve()
if not target.is_relative_to(root.resolve()):
return None
return target
def bootstrap_media_store(source_root: Path, target_root: Path) -> dict[str, int]:
"""Merge the immutable image baseline into the persistent media volume.
Existing decisions and files in the volume always win. New baseline assets
are copied in on release, so a recreated container cannot reset decisions
while a newer Git baseline can still add assets.
"""
source_manifest_path = source_root / "manifest.json"
target_manifest_path = target_root / "manifest.json"
source = json.loads(source_manifest_path.read_text(encoding="utf-8"))
target = json.loads(target_manifest_path.read_text(encoding="utf-8")) if target_manifest_path.exists() else {"version": 1, "assets": []}
target_by_url = {item.get("asset_url"): item for item in target.get("assets", [])}
added = 0
copied = 0
target_root.mkdir(parents=True, exist_ok=True)
for source_item in source.get("assets", []):
url = source_item.get("asset_url")
if not url or url in target_by_url:
continue
item = dict(source_item)
target_by_url[url] = item
target.setdefault("assets", []).append(item)
added += 1
for item in target.get("assets", []):
for relative in [item.get("local_path"), *[variant.get("local_path") for variant in item.get("derivatives", [])]]:
if not isinstance(relative, str):
continue
source_path = _safe_path(source_root, relative)
target_path = _safe_path(target_root, relative)
if not source_path or not target_path or not source_path.is_file() or target_path.exists():
continue
target_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source_path, target_path)
copied += 1
target["version"] = max(int(source.get("version", 1)), int(target.get("version", 1)))
if added or copied or not target_manifest_path.exists():
target["updated_at"] = datetime.now(timezone.utc).isoformat()
temporary = target_manifest_path.with_suffix(".json.tmp")
temporary.write_text(json.dumps(target, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
temporary.replace(target_manifest_path)
return {"added_assets": added, "copied_files": copied}
+25 -3
View File
@@ -6,7 +6,7 @@ import io
import pytest import pytest
from PIL import Image from PIL import Image
from rf4_research.media_assets import approve_stored_assets, audit_media_catalog, compare_quality_upgrades, extract_media_candidates, generate_media_derivatives, generate_quality_contact_sheets, inspect_image, media_coverage, media_quality_report, merge_manifest, publish_quality_upgrades, queue_quality_upgrades, reconcile_queued_duplicates, review_asset, rollback_quality_upgrade, store_asset from rf4_research.media_assets import approve_stored_assets, audit_media_catalog, compare_quality_upgrades, extract_media_candidates, generate_media_derivatives, generate_quality_contact_sheets, inspect_image, media_coverage, media_quality_report, merge_manifest, preview_quality_upgrades, publish_quality_upgrades, queue_quality_upgrades, reconcile_queued_duplicates, review_asset, rollback_quality_upgrade, store_asset
def test_extracts_and_classifies_unique_https_media() -> None: def test_extracts_and_classifies_unique_https_media() -> None:
@@ -357,7 +357,7 @@ def test_publish_quality_upgrades_switches_mapping_and_retains_fallback(tmp_path
report = publish_quality_upgrades(path, note="owner approved quality upgrades") report = publish_quality_upgrades(path, note="owner approved quality upgrades")
assets = json.loads(path.read_text(encoding="utf-8"))["assets"] assets = json.loads(path.read_text(encoding="utf-8"))["assets"]
assert report == {"published": 1, "retained_fallbacks": 1} assert report == {"published": 1, "retained_fallbacks": 1, "manifest_version": 2}
assert assets[0]["status"] == "superseded" assert assets[0]["status"] == "superseded"
assert assets[0]["replaced_by"] == assets[1]["asset_url"] assert assets[0]["replaced_by"] == assets[1]["asset_url"]
assert assets[1]["status"] == "approved" assert assets[1]["status"] == "approved"
@@ -375,8 +375,30 @@ def test_rollback_quality_upgrade_restores_fallback_atomically(tmp_path: Path) -
report = rollback_quality_upgrade(path, asset_url="https://large/pike.webp", note="owner rolled back after visual review") report = rollback_quality_upgrade(path, asset_url="https://large/pike.webp", note="owner rolled back after visual review")
assets = json.loads(path.read_text(encoding="utf-8"))["assets"] assets = json.loads(path.read_text(encoding="utf-8"))["assets"]
assert report == {"rolled_back": "https://large/pike.webp", "restored": "https://small/pike.png"} assert report == {"rolled_back": "https://large/pike.webp", "restored": "https://small/pike.png", "manifest_version": 2}
assert assets[0]["status"] == "approved" assert assets[0]["status"] == "approved"
assert "replaced_by" not in assets[0] assert "replaced_by" not in assets[0]
assert assets[1]["status"] == "upgrade_stored" assert assets[1]["status"] == "upgrade_stored"
assert "supersedes" not in assets[1] assert "supersedes" not in assets[1]
def test_publish_selects_ids_and_rejects_stale_manifest_version(tmp_path: Path) -> None:
path = tmp_path / "manifest.json"
path.write_text(json.dumps({"version": 7, "assets": [
{"entity_type": "fish", "entity_key": "fish:pike", "status": "approved", "asset_url": "https://small/pike.png", "local_path": "files/small.png", "sha256": "a" * 64},
{"entity_type": "fish", "status": "upgrade_stored", "asset_url": "https://large/pike.webp", "duplicate_of": "https://small/pike.png", "local_path": "files/large.webp", "sha256": "b" * 64, "width": 512, "height": 512},
{"entity_type": "fish", "entity_key": "fish:perch", "status": "approved", "asset_url": "https://small/perch.png", "local_path": "files/small-perch.png", "sha256": "c" * 64},
{"entity_type": "fish", "status": "upgrade_stored", "asset_url": "https://large/perch.webp", "duplicate_of": "https://small/perch.png", "local_path": "files/large-perch.webp", "sha256": "d" * 64, "width": 512, "height": 512},
]}), encoding="utf-8")
assert preview_quality_upgrades(path, asset_ids=["b" * 64]) == {
"manifest_version": 7,
"assets": [{"id": "b" * 64, "asset_url": "https://large/pike.webp", "duplicate_of": "https://small/pike.png", "label": None, "width": 512, "height": 512}],
}
report = publish_quality_upgrades(path, asset_ids=["b" * 64], expected_version=7, note="one reviewed replacement")
assert report["manifest_version"] == 8
saved = json.loads(path.read_text(encoding="utf-8"))
assert [item["status"] for item in saved["assets"]] == ["superseded", "approved", "approved", "upgrade_stored"]
with pytest.raises(ValueError, match="manifest version conflict"):
publish_quality_upgrades(path, asset_ids=["d" * 64], expected_version=7, note="stale tab")
+41
View File
@@ -0,0 +1,41 @@
import json
from pathlib import Path
from rf4_research.media_store import bootstrap_media_store
def test_bootstrap_merges_new_baseline_without_replacing_decisions(tmp_path: Path) -> None:
source = tmp_path / "source"
target = tmp_path / "target"
(source / "files").mkdir(parents=True)
(source / "files/new.bin").write_bytes(b"new")
(source / "manifest.json").write_text(json.dumps({"version": 2, "assets": [{
"asset_url": "https://new.example/asset", "status": "approved",
"local_path": "files/new.bin",
}]}), encoding="utf-8")
target.mkdir()
(target / "manifest.json").write_text(json.dumps({"version": 1, "assets": [{
"asset_url": "https://new.example/asset", "status": "superseded",
"local_path": "files/new.bin", "review_note": "owner decision",
}]}), encoding="utf-8")
result = bootstrap_media_store(source, target)
assert result == {"added_assets": 0, "copied_files": 1}
saved = json.loads((target / "manifest.json").read_text(encoding="utf-8"))
assert saved["assets"][0]["status"] == "superseded"
assert saved["assets"][0]["review_note"] == "owner decision"
assert (target / "files/new.bin").read_bytes() == b"new"
def test_bootstrap_adds_assets_from_new_release(tmp_path: Path) -> None:
source = tmp_path / "source"
target = tmp_path / "target"
(source / "files").mkdir(parents=True)
(source / "files/new.bin").write_bytes(b"new")
(source / "manifest.json").write_text(json.dumps({"version": 1, "assets": [{
"asset_url": "https://new.example/asset", "local_path": "files/new.bin",
}]}), encoding="utf-8")
assert bootstrap_media_store(source, target) == {"added_assets": 1, "copied_files": 1}
assert (target / "files/new.bin").exists()