diff --git a/README.md b/README.md index cc26a5f..caac2da 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ RF4DB/RF4-STAT/RF4MAP/RF4 Posts сначала принимаются в изо Медиасборщик индексирует разрешённые изображения отдельно от публичного каталога: manifest хранит исходную страницу, URL, предполагаемый тип сущности и время обнаружения, а оригиналы сохраняются по SHA-256 без hotlink. Индексация страницы и загрузка каждого файла используют общий 30-минутный cooldown домена; непроверенный asset не публикуется автоматически. Локальный `media_cli --audit` без сетевых запросов проверяет хэши, файлы, MIME, размеры, approved-сопоставления и отсутствие бесхозных оригиналов. +Текущее media-покрытие каталога — 0 approved соответствий для 2 рыб, 2 водоёмов и 3 приманок локальной альфы. Userguide дал 69 кандидатов, преимущественно справочные скриншоты; обычный HTML официальной fish gallery отдаёт только элементы оформления, поскольку содержимое галереи динамическое. Поэтому сайт продолжает использовать честные фирменные силуэты/отпечатки/глифы, пока конкретное изображение не сохранено и не сопоставлено вручную. + Для измерений на собственном сервере подготовлен read-only `deploy/load-smoke.py`: он считает p50/p95/max и HTTP-коды для activity/records, а при наличии `ADMIN_TOKEN` — staging/moderation. Методика и безопасные ступени нагрузки описаны в [docs/load-testing.md](docs/load-testing.md); локальные цифры не выдаются за production baseline. В production MinIO root credentials доступны только одноразовому init-контейнеру. API использует отдельного пользователя с доступом исключительно к `S3_BUCKET`: просмотр bucket, чтение, запись и удаление его объектов без глобального списка bucket и без права создавать новые. diff --git a/data/media/manifest.json b/data/media/manifest.json index 56e8c4b..235e96e 100644 --- a/data/media/manifest.json +++ b/data/media/manifest.json @@ -1,6 +1,6 @@ { "version": 1, - "updated_at": "2026-09-12T08:50:41.276898+00:00", + "updated_at": "2026-09-12T14:55:17.283768+00:00", "assets": [ { "source_page": "https://rf4game.de/userguide/", @@ -17,7 +17,10 @@ "bytes": 53123, "width": 602, "height": 208, - "fetched_at": "2026-09-12T14:24:59.865233+00:00" + "fetched_at": "2026-09-12T14:24:59.865233+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -27,7 +30,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -37,7 +43,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -47,7 +56,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -57,7 +69,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -67,7 +82,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -77,7 +95,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -87,7 +108,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -97,7 +121,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -107,7 +134,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -117,7 +147,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -127,7 +160,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -137,7 +173,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -147,7 +186,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -157,7 +199,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -167,7 +212,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -177,7 +225,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -187,7 +238,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -197,7 +251,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -207,7 +264,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -217,7 +277,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -227,7 +290,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -237,7 +303,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -247,7 +316,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -257,7 +329,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -267,7 +342,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -277,7 +355,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -287,7 +368,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -297,7 +381,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -307,7 +394,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -317,7 +407,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -327,7 +420,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -337,7 +433,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -347,7 +446,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -357,7 +459,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -367,7 +472,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -377,7 +485,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -387,7 +498,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -397,7 +511,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -407,7 +524,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -417,7 +537,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -427,7 +550,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -437,7 +563,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -447,7 +576,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -457,7 +589,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -467,7 +602,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -477,7 +615,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -487,7 +628,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -497,7 +641,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -507,7 +654,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -517,7 +667,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -527,7 +680,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -537,7 +693,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -547,7 +706,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -557,7 +719,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -567,7 +732,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -577,7 +745,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -587,7 +758,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -597,7 +771,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -607,7 +784,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -617,7 +797,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -627,7 +810,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -637,7 +823,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -647,7 +836,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -657,7 +849,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -667,7 +862,10 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { "source_page": "https://rf4game.de/userguide/", @@ -677,27 +875,38 @@ "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T08:50:41.276898+00:00", + "source_pages": [ + "https://rf4game.de/userguide/" + ] }, { - "source_page": "https://rf4game.de/userguide/", + "source_page": "https://rf4game.de/media/fische/", "asset_url": "https://rf4game.de/wp-content/plugins/qtranslate-xt-master/flags/de.png", "entity_type": "reference", "label": null, "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T14:55:17.283768+00:00", + "source_pages": [ + "https://rf4game.de/media/fische/", + "https://rf4game.de/userguide/" + ] }, { - "source_page": "https://rf4game.de/userguide/", + "source_page": "https://rf4game.de/media/fische/", "asset_url": "https://rf4game.de/wp-content/themes/rf4/img/banner_de.png", "entity_type": "reference", "label": null, "external_id": null, "status": "queued", "first_seen_at": "2026-09-12T08:50:41.276898+00:00", - "last_seen_at": "2026-09-12T08:50:41.276898+00:00" + "last_seen_at": "2026-09-12T14:55:17.283768+00:00", + "source_pages": [ + "https://rf4game.de/media/fische/", + "https://rf4game.de/userguide/" + ] } ] } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index cbf1634..0df0662 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,7 +30,7 @@ - [x] **B08 · Семейства силуэтов рыб.** Три случайных hash-варианта заменены классификатором и отдельными формами `pike`, `salmonid`, `cyprinid`, `perch`, `catfish`, `eel`, `flatfish`, `marine`, плюс честный `generic`. Компонент допускает ручное переопределение family; название остаётся главным идентификатором. - [x] **B09 · Глифы снастей и приманок.** Добавлены SVG-глифы `spinner`, `wobbler`, `soft`, `boilie`, `worm`, `rig`, `unknown` и стабильная палитра по normalized name. Классификация срабатывает только по явным словам; глиф сопровождает текст в activity, лидере, уловах, рекордах и списке лучших приманок, не выдавая категорию за точную модель. - [x] **B10 · Визуальные отпечатки водоёмов.** Для каждого slug воспроизводимо выбираются один из восьми береговых контуров, число волн, положение точки и двухсимвольный индекс. Знак используется в каталоге и detail-hero; это явно абстрактный отпечаток, а не карта или игровая география. -- [ ] **B11 · Разрешённый media pipeline — в работе.** Получено явное разрешение на локальный сбор. Добавлены allowlist, индекс кандидатов, provenance-manifest, очередь и content-addressed хранение без hotlink; первый официальный обход нашёл 69 кандидатов. Downloader проверяет реальный формат/MIME, лимит 40 Мп, сохраняет dimensions и не блокирует очередь постоянными `missing`/`blocked`/`invalid`. Review допускает `approved` только для сохранённого файла с типом и каноническим ключом сущности; доступен явный `rejected`. Осталось постепенно скачать очередь с общим cooldown 30 минут и проверить соответствия перед публикацией. +- [ ] **B11 · Разрешённый media pipeline — в работе.** Получено явное разрешение на локальный сбор. Первый обход userguide нашёл 69 кандидатов, из которых 1 сохранён, 68 ожидают. Аудит покрытия выявил отсутствие approved-соответствий для всех 2 рыб, 2 водоёмов и 3 приманок локальной альфы. Прямой HTML `/media/fische/` содержит только флаг и banner: галерея динамическая, рыб через текущий HTML-parser не отдаёт. Классификатор учитывает provenance страницы, но исключает theme/flag/logo chrome; manifest сохраняет все страницы обнаружения дубликата. Далее исследовать официальный gallery endpoint без обхода ограничений, отдельно индексировать `/media/levels/` и проверить `.bait_icon` records DOM в следующие 30-минутные окна. Публикация возможна только после ручного canonical review. - [x] **B12 · Эмблема сочетания.** Страница «водоём + рыба» получила составной атласный seal: собственный отпечаток водоёма пересекается со смысловым силуэтом рыбы. Так визуальная идентичность сопровождает всю иерархию каталога и не требует внешних изображений. - [x] **B13 · Навигационная леска атласа.** Разрозненные ссылки назад на detail-страницах заменены доступной breadcrumb-цепочкой с мотивом лески и узлов. Страница точки связывает главную, водоём и координаты; сочетание — каталог, водоём и рыбу. Текущий узел всегда подписан текстом и отмечен `aria-current`. - [x] **B14 · Атласные переходы сущностей.** Боковые списки рыб и водоёмов на detail-страницах получили компактные силуэты и отпечатки рядом с полным текстовым названием. Знаки продолжают систему каталога в рабочей навигации, а стрелка явно показывает переход к странице сочетания. diff --git a/docs/data-sources.md b/docs/data-sources.md index 661cdb4..4b81293 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -50,6 +50,10 @@ Проверка 12 сентября 2026 года не обнаружила стабильного публичного каталога вида `канонический slug → изображение` для рыб, водоёмов или снастей. Официальная [галерея водоёмов](https://rf4game.de/media/levels/) и категория [рыб](https://rf4game.de/media/fische/) содержат тематические публикации, а [руководство](https://rf4game.de/userguide/) — иллюстрации интерфейса, оснасток и карт. Это медиа-галереи, а не полный справочник с устойчивыми entity ID. +Повторная сверка локальной альфы показала 2 рыбы, 2 водоёма и 3 приманки при нулевом числе approved media mappings. В userguide-manifest находится 69 кандидатов: 67 общих reference и 2 изображения карт. Поэтому наличие картинки в очереди не считается покрытием каталога; coverage появляется только после сохранения, визуальной проверки и назначения canonical `entity_key`. + +Прямой индекс `/media/fische/` вернул только общие изображения оформления (`flags/de.png`, `banner_de.png`): содержимое галереи загружается динамически. Эти файлы остаются `reference`, а не ошибочно маркируются как рыбы. Исследование допустимого gallery endpoint должно использовать обычный публичный контракт сайта и тот же общий cooldown, без обхода защит. + Страница рекордов визуально использует `.bait_icon`, но подтверждённый parser-контракт извлекает только `.bait_icon[title]`. Ни parser, ни `Fish`/`Waterbody`/`Bait` модели и API-схемы сейчас не сохраняют image URL. Даже если URL удастся извлечь из CSS или обновлённого DOM, он может быть presentation asset, меняться независимо от названия и покрывать только приманки, попавшие в рекорды. До отдельного разрешения на медиапубликацию изображения официального сайта не скачиваются, не хотлинкаются и не отображаются как собственные. Разрешение использовать фактические данные не трактуется как разрешение републиковать графические произведения. diff --git a/rf4_research/media_assets.py b/rf4_research/media_assets.py index 071f148..3c0bcf6 100644 --- a/rf4_research/media_assets.py +++ b/rf4_research/media_assets.py @@ -39,9 +39,11 @@ def _label(node: Tag) -> str | None: def _kind(url: str, label: str | None, context: str) -> str: path = urlsplit(url).path.casefold() value = f"{label or ''} {context}".casefold() - if re.search(r"/(?:fish|fishes|fische|species)/", path) or any(word in value for word in ("рыба ", "fish: ", "fisch: ")): + if "/flags/" in path or "/themes/" in path or path.endswith(("/logo.png", "/logo.svg")): + return "reference" + if re.search(r"/(?:fish|fishes|fische|species)/", path) or "/media/fische/" in value or any(word in value for word in ("рыба ", "fish: ", "fisch: ")): return "fish" - if re.search(r"/(?:maps?|levels?|lakes?|waterbodies)/", path) or any(word in value for word in ("карта ", "map of ", "gewässerkarte")): + if re.search(r"/(?:maps?|levels?|lakes?|waterbodies)/", path) or "/media/levels/" in value or any(word in value for word in ("карта ", "map of ", "gewässerkarte")): return "waterbody" if re.search(r"(?:^|[/_-])(?:bait|lure|rig|rod|reel|hook|tackle|köder|rute|rolle)(?:[/_.-]|$)", path): return "tackle" @@ -63,10 +65,10 @@ def extract_media_candidates(html: str, *, source_page: str) -> list[MediaCandid context = " ".join(node.parent.get_text(" ", strip=True).split())[:500] if node.parent else "" link = node.find_parent("a", href=True) external_id = str(link.get("href")).rstrip("/").rsplit("/", 1)[-1] if link else None - result.setdefault(url, MediaCandidate(source_page, url, _kind(url, label, context), label, external_id)) + result.setdefault(url, MediaCandidate(source_page, url, _kind(url, label, f"{source_page} {context}"), label, external_id)) for match in re.finditer(r"url\((['\"]?)(https://[^)'\"]+)\1\)", html, re.I): url = match.group(2) - result.setdefault(url, MediaCandidate(source_page, url, _kind(url, None, ""), None, None)) + result.setdefault(url, MediaCandidate(source_page, url, _kind(url, None, source_page), None, None)) return list(result.values()) @@ -76,7 +78,12 @@ def merge_manifest(path: Path, candidates: list[MediaCandidate]) -> dict: seen_at = datetime.now(timezone.utc).isoformat() for candidate in candidates: item = assets.get(candidate.asset_url, {}) + source_pages = set(item.get("source_pages", [])) + if item.get("source_page"): + source_pages.add(item["source_page"]) + source_pages.add(candidate.source_page) item.update(asdict(candidate)) + item["source_pages"] = sorted(source_pages) item.setdefault("status", "queued") item.setdefault("first_seen_at", seen_at) item["last_seen_at"] = seen_at @@ -90,7 +97,8 @@ def merge_manifest(path: Path, candidates: list[MediaCandidate]) -> dict: def reclassify_manifest(path: Path) -> dict: manifest = json.loads(path.read_text(encoding="utf-8")) for item in manifest.get("assets", []): - item["entity_type"] = _kind(item["asset_url"], item.get("label"), "") + item.setdefault("source_pages", [item["source_page"]]) + item["entity_type"] = _kind(item["asset_url"], item.get("label"), item.get("source_page", "")) path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return manifest diff --git a/tests/test_media_assets.py b/tests/test_media_assets.py index 5cd7776..255620e 100644 --- a/tests/test_media_assets.py +++ b/tests/test_media_assets.py @@ -25,11 +25,21 @@ def test_does_not_mistake_seafishing_screenshot_for_fish_entity() -> None: assert items[0].entity_type == "reference" +def test_gallery_page_provenance_classifies_generic_asset_urls() -> None: + fish = extract_media_candidates('Hecht', source_page="https://rf4game.de/media/fische/") + water = extract_media_candidates('Kuori', source_page="https://rf4game.de/media/levels/") + assert fish[0].entity_type == "fish" + assert water[0].entity_type == "waterbody" + chrome = extract_media_candidates('', source_page="https://rf4game.de/media/fische/") + assert chrome[0].entity_type == "reference" + + def test_manifest_merges_and_binary_store_is_content_addressed(tmp_path: Path) -> None: item = extract_media_candidates('Bait worm', source_page="https://example.test")[0] first = merge_manifest(tmp_path / "manifest.json", [item]) second = merge_manifest(tmp_path / "manifest.json", [item]) assert len(first["assets"]) == len(second["assets"]) == 1 + assert second["assets"][0]["source_pages"] == ["https://example.test"] image = io.BytesIO() Image.new("RGB", (3, 2), "green").save(image, format="PNG") digest, relative, width, height, mime = store_asset(tmp_path, image.getvalue(), content_type="image/png", source_url=item.asset_url)