104 lines
5.3 KiB
Python
104 lines
5.3 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
from rf4_research import media_cli
|
|
from rf4_research.media_cli import _download_batch, media_queue_plan
|
|
|
|
|
|
def test_queue_plan_prioritizes_entity_gaps_and_respects_domain_cooldown(tmp_path: Path) -> None:
|
|
root = tmp_path / "media"
|
|
root.mkdir()
|
|
(root / "catalog-baseline.json").write_text(json.dumps({
|
|
"verified_at": "2026-09-12",
|
|
"entities": {
|
|
"fish": {"count": 2}, "waterbody": {"count": 1}, "tackle": {"count": None},
|
|
},
|
|
}), encoding="utf-8")
|
|
(root / "manifest.json").write_text(json.dumps({"version": 1, "assets": [
|
|
{"status": "queued", "asset_url": "https://rf4map.ru/tackle.png", "source_page": "https://rf4map.ru/fishes", "entity_type": "tackle", "label": "Катушка", "external_id": "3"},
|
|
{"status": "queued", "asset_url": "https://rf4map.ru/fish.png", "source_page": "https://rf4map.ru/fishes", "entity_type": "fish", "label": "Щука", "external_id": "2"},
|
|
{"status": "queued", "asset_url": "https://rf4map.ru/water.png", "source_page": "https://rf4map.ru/maps", "entity_type": "waterbody", "label": "Ладога", "external_id": "1"},
|
|
{"status": "queued", "asset_url": "https://rf4-posts.com/fish.png", "source_page": "https://rf4-posts.com/fishes", "entity_type": "fish", "label": "Ёрш", "external_id": "4"},
|
|
{"status": "approved", "asset_url": "https://rf4map.ru/approved.png", "source_page": "https://rf4map.ru/fishes", "entity_type": "fish", "label": "Окунь", "entity_key": "perch"},
|
|
]}), encoding="utf-8")
|
|
state = tmp_path / "state.json"
|
|
state.write_text(json.dumps({"rf4map.ru": 1000}), encoding="utf-8")
|
|
|
|
plan = media_queue_plan(root, state, now=2000)
|
|
|
|
assert plan["network_requests"] == 0
|
|
assert plan["batch_limit_per_domain"] == 40
|
|
assert plan["queued_total"] == 4
|
|
assert plan["domains"]["rf4map.ru"]["ready"] is False
|
|
assert plan["domains"]["rf4map.ru"]["retry_in_seconds"] == 800
|
|
assert plan["domains"]["rf4map.ru"]["queued_by_type"] == {"fish": 1, "tackle": 1, "waterbody": 1}
|
|
assert plan["domains"]["rf4map.ru"]["next_asset"]["entity_type"] == "waterbody"
|
|
assert plan["domains"]["rf4-posts.com"]["ready"] is True
|
|
assert plan["exact_catalog_gaps_known"] is False
|
|
|
|
|
|
def test_batch_reserves_once_per_domain_and_caps_each_window(tmp_path: Path, monkeypatch) -> None:
|
|
root = tmp_path / "media"
|
|
root.mkdir()
|
|
assets = [
|
|
{"status": "queued", "asset_url": f"https://rf4map.ru/fish-{index}.png", "source_page": "https://rf4map.ru/fishes", "entity_type": "fish", "label": f"Рыба {index}"}
|
|
for index in range(45)
|
|
] + [
|
|
{"status": "queued", "asset_url": f"https://rf4-posts.com/fish-{index}.png", "source_page": "https://rf4-posts.com/fishes", "entity_type": "fish", "label": f"Рыба {index}"}
|
|
for index in range(2)
|
|
]
|
|
(root / "manifest.json").write_text(json.dumps({"version": 1, "assets": assets}), encoding="utf-8")
|
|
state = tmp_path / "state.json"
|
|
|
|
monkeypatch.setattr(media_cli, "_attempt_asset", lambda *_: ("stored", None))
|
|
report = _download_batch(root, state, limit=40)
|
|
|
|
assert report["attempted_total"] == 42
|
|
assert report["domains"]["rf4map.ru"]["attempted"] == 40
|
|
assert report["domains"]["rf4-posts.com"]["attempted"] == 2
|
|
assert set(json.loads(state.read_text())) == {"rf4map.ru", "rf4-posts.com"}
|
|
|
|
cooling = _download_batch(root, state, limit=40)
|
|
assert cooling["attempted_total"] == 0
|
|
assert all("cooldown is active" in item["skipped"] for item in cooling["domains"].values())
|
|
|
|
|
|
def test_batch_stops_domain_after_three_consecutive_invalid_assets(tmp_path: Path, monkeypatch) -> None:
|
|
root = tmp_path / "media"
|
|
root.mkdir()
|
|
assets = [
|
|
{"status": "queued", "asset_url": f"https://rf4map.ru/bad-{index}.png", "source_page": "https://rf4map.ru/fishes", "entity_type": "fish", "label": f"Рыба {index}"}
|
|
for index in range(10)
|
|
]
|
|
(root / "manifest.json").write_text(json.dumps({"version": 1, "assets": assets}), encoding="utf-8")
|
|
|
|
def invalid(_root, _path, _manifest, item):
|
|
item["status"] = "invalid"
|
|
return "invalid", ValueError("not an image")
|
|
|
|
monkeypatch.setattr(media_cli, "_attempt_asset", invalid)
|
|
report = _download_batch(root, tmp_path / "state.json", limit=10)
|
|
|
|
assert report["attempted_total"] == 3
|
|
assert report["failed_total"] == 3
|
|
assert "3 consecutive failures" in report["domains"]["rf4map.ru"]["stopped_reason"]
|
|
|
|
|
|
def test_batch_downloads_quality_upgrade_without_unpublishing_fallback(tmp_path: Path, monkeypatch) -> None:
|
|
root = tmp_path / "media"
|
|
root.mkdir()
|
|
candidate = {"status": "upgrade_queued", "asset_url": "https://rf4db.com/pike.webp", "entity_type": "fish", "duplicate_of": "https://rf4map.ru/pike.png"}
|
|
(root / "manifest.json").write_text(json.dumps({"version": 1, "assets": [candidate]}), encoding="utf-8")
|
|
|
|
def store_upgrade(_root, path, manifest, item):
|
|
item["status"] = "upgrade_stored"
|
|
media_cli._save_manifest(path, manifest)
|
|
return "stored", None
|
|
|
|
monkeypatch.setattr(media_cli, "_attempt_asset", store_upgrade)
|
|
report = _download_batch(root, tmp_path / "state.json", limit=1)
|
|
|
|
assert report["stored_total"] == 1
|
|
assets = json.loads((root / "manifest.json").read_text(encoding="utf-8"))["assets"]
|
|
assert assets[0]["status"] == "upgrade_stored"
|