feat: add production health monitoring

2026-09-06 14:44:38 +07:00
parent 2200336524
commit e6653d4c77
8 changed files with 167 additions and 3 deletions
@@ -107,9 +107,19 @@ docker compose --env-file .env.production -f compose.production.yaml exec -T api

 Сроки и состав данных описаны в [`docs/data-retention.md`](../docs/data-retention.md). Автоматизацию включайте только после проверки dry-run на рабочем наборе.

-## 8. Что ещё блокирует приглашение альфа-пользователей
+## 8. Мониторинг

- базовый мониторинг `/ready`, диска и срока TLS-сертификата;
+После настройки DNS, TLS и первого backup выполните:
+
+```bash
+./deploy/monitor.sh
+```
+
+Проверка контролирует контейнеры, `/ready`, диск, свежесть backup и срок TLS. Установка systemd timer, пороги и порядок реакции описаны в [`docs/production-monitoring.md`](../docs/production-monitoring.md). До подключения реального канала уведомлений одного журнала systemd недостаточно.
+
+## 9. Что ещё блокирует приглашение альфа-пользователей
+
+- подключение уведомлений о сбоях мониторинга;
 - проверка DNS/TLS и полного запуска на целевом сервере;
 - внешнее зашифрованное хранилище резервных копий.

@@ -0,0 +1,80 @@
+#!/bin/sh
+set -u
+
+repo=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)
+cd "$repo" || exit 2
+env_file=${COMPOSE_ENV_FILE:-.env.production}
+if [ ! -r "$env_file" ]; then
+  printf 'CRITICAL rf4spotter failures="env-file-unreadable"\n'
+  exit 2
+fi
+
+read_env() {
+  key=$1
+  value=$(sed -n "s/^${key}=//p" "$env_file" | tail -n 1)
+  printf '%s' "$value"
+}
+
+site_domain=${SITE_DOMAIN:-$(read_env SITE_DOMAIN)}
+backup_root=${BACKUP_ROOT:-$(read_env BACKUP_ROOT)}
+disk_max=${MONITOR_DISK_MAX_PERCENT:-$(read_env MONITOR_DISK_MAX_PERCENT)}
+tls_days=${MONITOR_TLS_MIN_DAYS:-$(read_env MONITOR_TLS_MIN_DAYS)}
+backup_hours=${MONITOR_BACKUP_MAX_HOURS:-$(read_env MONITOR_BACKUP_MAX_HOURS)}
+disk_max=${disk_max:-85}
+tls_days=${tls_days:-14}
+backup_hours=${backup_hours:-26}
+for value in "$disk_max" "$tls_days" "$backup_hours"; do
+  case "$value" in ''|*[!0-9]*) printf 'CRITICAL rf4spotter failures="monitor-threshold-invalid"\n'; exit 2 ;; esac
+done
+base_url=${MONITOR_BASE_URL:-https://$site_domain}
+compose="docker compose --env-file $env_file -f compose.production.yaml"
+failures=""
+
+fail() {
+  failures="${failures}${failures:+; }$1"
+}
+
+running=$($compose ps --status running --services 2>/dev/null) || running=""
+for service in proxy db minio api web; do
+  printf '%s\n' "$running" | grep -qx "$service" || fail "service:$service"
+done
+
+ready_payload=$(curl -fsS --max-time 10 "$base_url/ready" 2>/dev/null) || ready_payload=""
+printf '%s' "$ready_payload" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"ready"' || fail "readiness"
+
+disk_used=$(df -Pk "$repo" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+case "$disk_used" in
+  ''|*[!0-9]*) fail "disk:unknown" ;;
+  *) [ "$disk_used" -lt "$disk_max" ] || fail "disk:${disk_used}%" ;;
+esac
+
+if [ -z "$backup_root" ] || [ ! -d "$backup_root" ]; then
+  fail "backup:directory"
+else
+  recent_backup=$(find "$backup_root" -mindepth 2 -maxdepth 2 -name SHA256SUMS -mmin "-$((backup_hours * 60))" -print -quit 2>/dev/null)
+  if [ -z "$recent_backup" ]; then
+    fail "backup:stale"
+  else
+    (cd "$(dirname "$recent_backup")" && sha256sum -c --quiet SHA256SUMS) >/dev/null 2>&1 || fail "backup:checksum"
+  fi
+fi
+
+if [ -z "$site_domain" ]; then
+  fail "tls:domain"
+else
+  tls_seconds=$((tls_days * 86400))
+  certificate=$(openssl s_client -servername "$site_domain" -connect "$site_domain:443" </dev/null 2>/dev/null | openssl x509 -outform PEM 2>/dev/null) || certificate=""
+  if [ -z "$certificate" ]; then
+    fail "tls:unavailable"
+  else
+    printf '%s\n' "$certificate" | openssl x509 -checkend "$tls_seconds" -noout >/dev/null 2>&1 || fail "tls:expires-soon"
+  fi
+fi
+
+timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+if [ -n "$failures" ]; then
+  printf 'CRITICAL rf4spotter timestamp=%s failures="%s"\n' "$timestamp" "$failures"
+  exit 1
+fi
+
+printf 'OK rf4spotter timestamp=%s disk_used=%s%% backup_max_age=%sh tls_min=%sd\n' "$timestamp" "$disk_used" "$backup_hours" "$tls_days"
@@ -0,0 +1,9 @@
+[Unit]
+Description=RF4 Spotter production health check
+After=docker.service network-online.target
+
+[Service]
+Type=oneshot
+User=rf4spotter
+WorkingDirectory=/opt/rf4-spotter
+ExecStart=/opt/rf4-spotter/deploy/monitor.sh
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run RF4 Spotter health check every five minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+RandomizedDelaySec=30s
+Persistent=true
+
+[Install]
+WantedBy=timers.target