diff --git a/auto-healer/auto-heal-agent.sh b/auto-healer/auto-heal-agent.sh index 0dde812..2d2aa73 100644 --- a/auto-healer/auto-heal-agent.sh +++ b/auto-healer/auto-heal-agent.sh @@ -523,6 +523,22 @@ while true; do # ── Periodic maintenance (every 10 cycles = 5 min) ───── CYCLE=$((CYCLE + 1)) [ "$CYCLE" -ge 10 ] && { CYCLE=0; periodic_maintenance; } + + # ── Cross-check: DNS healer alive? ──────────────────── + if ! docker ps --filter 'name=falah-dns-healer' --format '{{.Status}}' 2>/dev/null | grep -q '^Up'; then + log "⚠️ Sibling DNS healer DOWN — restarting..." + docker restart falah-dns-healer 2>/dev/null || \ + docker run -d --name falah-dns-healer --restart unless-stopped \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /var/log/falah:/var/log/falah \ + -e CHECK_INTERVAL=60 \ + falah-dns-healer:latest >> "$AGENT_LOG" 2>&1 + sleep 3 + if docker ps --filter 'name=falah-dns-healer' --format '{{.Status}}' | grep -q '^Up'; then + alert "INFO" "Cross-heal: restarted DNS healer" + fi + fi sleep "$CHECK_INTERVAL" done diff --git a/dns-router-healer/dns-router-heal.sh b/dns-router-healer/dns-router-heal.sh index 1b68324..ac994d0 100644 --- a/dns-router-healer/dns-router-heal.sh +++ b/dns-router-healer/dns-router-heal.sh @@ -350,5 +350,24 @@ while true; do fi log "── Cycle $CYCLE complete ─────────────────────────────" + + # ── Cross-check: App healer alive? ───────────────────── + if ! docker ps --filter 'name=falah-auto-healer' --format '{{.Status}}' 2>/dev/null | grep -q '^Up'; then + log "⚠️ Sibling app healer DOWN — restarting..." + docker restart falah-auto-healer 2>/dev/null || \ + docker run -d --name falah-auto-healer --restart unless-stopped \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /var/log/falah:/var/log/falah \ + -v /var/state/falah:/var/state/falah \ + -e CONTAINER_NAME=falah-mobile-staging \ + -e CHECK_INTERVAL=30 \ + falah-auto-healer:v6 >> "$AGENT_LOG" 2>&1 + sleep 3 + if docker ps --filter 'name=falah-auto-healer' --format '{{.Status}}' | grep -q '^Up'; then + alert "INFO" "Cross-heal: restarted app healer" + fi + fi + sleep "$CHECK_INTERVAL" done diff --git a/scripts/watchdog.sh b/scripts/watchdog.sh new file mode 100644 index 0000000..a973046 --- /dev/null +++ b/scripts/watchdog.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# ============================================================ +# Ultimate Watchdog — "Who watches the watchmen?" +# ============================================================ +# Runs as a cron job every 5 minutes. +# If BOTH healers are down, brings them back. +# This is the last line of defense. +# ============================================================ + +LOG="/var/log/falah/watchdog.log" +DOCKER=/var/packages/Docker/target/usr/bin/docker + +mkdir -p "$(dirname "$LOG")" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [watchdog] $*" | tee -a "$LOG" +} + +check_and_restart() { + local name="$1" + local image="${2:-$name:latest}" + + local status + status=$($DOCKER ps -a --filter "name=$name" --format '{{.Status}}' 2>/dev/null) + + if echo "$status" | grep -q "^Up"; then + return 0 # running + fi + + log "⚠️ $name is DOWN (status=$status) — restarting..." + + # Remove if exists (stopped/crashed) + $DOCKER rm -f "$name" 2>/dev/null || true + + # Determine run args based on container + case "$name" in + falah-auto-healer) + $DOCKER run -d --name "$name" --restart unless-stopped \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /var/log/falah:/var/log/falah \ + -v /var/state/falah:/var/state/falah \ + -e CONTAINER_NAME=falah-mobile-staging \ + -e HEALTH_URL=http://localhost:4014/mobile/api/health \ + -e GATEWAY_URL=http://localhost:7878/mobile/api/health \ + -e CHECK_INTERVAL=30 \ + -e MAX_RESTARTS_PER_HOUR=3 \ + -e ROLLBACK_IMAGE=falah-mobile:rollback \ + "$image" >> "$LOG" 2>&1 + ;; + falah-dns-healer) + $DOCKER run -d --name "$name" --restart unless-stopped \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /var/log/falah:/var/log/falah \ + -e DOMAINS="falahos.my api.falahos.my git.falahos.my app.falahos.my" \ + -e DIRECT_HEALTH="http://localhost:4014/mobile/api/health" \ + -e GATEWAY_URL="http://localhost:7878/mobile/api/health" \ + -e TUNNEL_PRIMARY="cloudflare-YT01" \ + -e TUNNEL_FALLBACK="falah-tunnel" \ + -e CHECK_INTERVAL=60 \ + "$image" >> "$LOG" 2>&1 + ;; + *) + log "❌ Unknown container: $name" + return 1 + ;; + esac + + sleep 3 + if $DOCKER ps --filter "name=$name" --format '{{.Status}}' | grep -q "^Up"; then + log "✅ $name restarted successfully" + return 0 + else + log "❌ $name failed to start" + return 1 + fi +} + +# ── Main ───────────────────────────────────────────────────── + +log "══════════════════════════════════════════════════════" +log "🔍 Watchdog check starting..." + +APP_OK=false +DNS_OK=false + +if check_and_restart "falah-auto-healer" "falah-auto-healer:v6"; then + APP_OK=true +fi + +if check_and_restart "falah-dns-healer" "falah-dns-healer:latest"; then + DNS_OK=true +fi + +if $APP_OK && $DNS_OK; then + log "✅ Both healers running" +elif $APP_OK || $DNS_OK; then + log "⚠️ One healer restored, other still down" +else + log "❌ CRITICAL: Both healers down and could not be restored!" +fi + +# Also verify the app itself is running +APP_STATUS=$($DOCKER ps --filter name=falah-mobile-staging --format '{{.Status}}' 2>/dev/null) +if ! echo "$APP_STATUS" | grep -q "^Up"; then + log "⚠️ App container not running — cannot auto-fix from watchdog" + log " (app healer should handle this once restarted)" +fi + +log "══════════════════════════════════════════════════════"