fix: who watches the watchmen — cross-heal + cron watchdog
- App healer (v7): checks DNS healer every 30s, restarts if down - DNS healer (v2): checks app healer every 60s, restarts if down - scripts/watchdog.sh: ultimate fallback cron job (every 5 min) revives both healers if both are down simultaneously - Installed to /usr/local/bin/falah-watchdog + /etc/cron.d/falah-watchdog Three-layer defense: Layer 1: Docker --restart unless-stopped (instant) Layer 2: Cross-heal between the two healers (30-60s) Layer 3: Cron watchdog (5 min, last resort)
This commit is contained in:
@@ -523,6 +523,22 @@ while true; do
|
||||
# ── Periodic maintenance (every 10 cycles = 5 min) ─────
|
||||
CYCLE=$((CYCLE + 1))
|
||||
[ "$CYCLE" -ge 10 ] && { CYCLE=0; periodic_maintenance; }
|
||||
|
||||
# ── Cross-check: DNS healer alive? ────────────────────
|
||||
if ! docker ps --filter 'name=falah-dns-healer' --format '{{.Status}}' 2>/dev/null | grep -q '^Up'; then
|
||||
log "⚠️ Sibling DNS healer DOWN — restarting..."
|
||||
docker restart falah-dns-healer 2>/dev/null || \
|
||||
docker run -d --name falah-dns-healer --restart unless-stopped \
|
||||
--network host \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock:ro \
|
||||
-v /var/log/falah:/var/log/falah \
|
||||
-e CHECK_INTERVAL=60 \
|
||||
falah-dns-healer:latest >> "$AGENT_LOG" 2>&1
|
||||
sleep 3
|
||||
if docker ps --filter 'name=falah-dns-healer' --format '{{.Status}}' | grep -q '^Up'; then
|
||||
alert "INFO" "Cross-heal: restarted DNS healer"
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$CHECK_INTERVAL"
|
||||
done
|
||||
|
||||
@@ -350,5 +350,24 @@ while true; do
|
||||
fi
|
||||
|
||||
log "── Cycle $CYCLE complete ─────────────────────────────"
|
||||
|
||||
# ── Cross-check: App healer alive? ─────────────────────
|
||||
if ! docker ps --filter 'name=falah-auto-healer' --format '{{.Status}}' 2>/dev/null | grep -q '^Up'; then
|
||||
log "⚠️ Sibling app healer DOWN — restarting..."
|
||||
docker restart falah-auto-healer 2>/dev/null || \
|
||||
docker run -d --name falah-auto-healer --restart unless-stopped \
|
||||
--network host \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock:ro \
|
||||
-v /var/log/falah:/var/log/falah \
|
||||
-v /var/state/falah:/var/state/falah \
|
||||
-e CONTAINER_NAME=falah-mobile-staging \
|
||||
-e CHECK_INTERVAL=30 \
|
||||
falah-auto-healer:v6 >> "$AGENT_LOG" 2>&1
|
||||
sleep 3
|
||||
if docker ps --filter 'name=falah-auto-healer' --format '{{.Status}}' | grep -q '^Up'; then
|
||||
alert "INFO" "Cross-heal: restarted app healer"
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$CHECK_INTERVAL"
|
||||
done
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# Ultimate Watchdog — "Who watches the watchmen?"
|
||||
# ============================================================
|
||||
# Runs as a cron job every 5 minutes.
|
||||
# If BOTH healers are down, brings them back.
|
||||
# This is the last line of defense.
|
||||
# ============================================================
|
||||
|
||||
LOG="/var/log/falah/watchdog.log"
|
||||
DOCKER=/var/packages/Docker/target/usr/bin/docker
|
||||
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [watchdog] $*" | tee -a "$LOG"
|
||||
}
|
||||
|
||||
check_and_restart() {
|
||||
local name="$1"
|
||||
local image="${2:-$name:latest}"
|
||||
|
||||
local status
|
||||
status=$($DOCKER ps -a --filter "name=$name" --format '{{.Status}}' 2>/dev/null)
|
||||
|
||||
if echo "$status" | grep -q "^Up"; then
|
||||
return 0 # running
|
||||
fi
|
||||
|
||||
log "⚠️ $name is DOWN (status=$status) — restarting..."
|
||||
|
||||
# Remove if exists (stopped/crashed)
|
||||
$DOCKER rm -f "$name" 2>/dev/null || true
|
||||
|
||||
# Determine run args based on container
|
||||
case "$name" in
|
||||
falah-auto-healer)
|
||||
$DOCKER run -d --name "$name" --restart unless-stopped \
|
||||
--network host \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock:ro \
|
||||
-v /var/log/falah:/var/log/falah \
|
||||
-v /var/state/falah:/var/state/falah \
|
||||
-e CONTAINER_NAME=falah-mobile-staging \
|
||||
-e HEALTH_URL=http://localhost:4014/mobile/api/health \
|
||||
-e GATEWAY_URL=http://localhost:7878/mobile/api/health \
|
||||
-e CHECK_INTERVAL=30 \
|
||||
-e MAX_RESTARTS_PER_HOUR=3 \
|
||||
-e ROLLBACK_IMAGE=falah-mobile:rollback \
|
||||
"$image" >> "$LOG" 2>&1
|
||||
;;
|
||||
falah-dns-healer)
|
||||
$DOCKER run -d --name "$name" --restart unless-stopped \
|
||||
--network host \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock:ro \
|
||||
-v /var/log/falah:/var/log/falah \
|
||||
-e DOMAINS="falahos.my api.falahos.my git.falahos.my app.falahos.my" \
|
||||
-e DIRECT_HEALTH="http://localhost:4014/mobile/api/health" \
|
||||
-e GATEWAY_URL="http://localhost:7878/mobile/api/health" \
|
||||
-e TUNNEL_PRIMARY="cloudflare-YT01" \
|
||||
-e TUNNEL_FALLBACK="falah-tunnel" \
|
||||
-e CHECK_INTERVAL=60 \
|
||||
"$image" >> "$LOG" 2>&1
|
||||
;;
|
||||
*)
|
||||
log "❌ Unknown container: $name"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
sleep 3
|
||||
if $DOCKER ps --filter "name=$name" --format '{{.Status}}' | grep -q "^Up"; then
|
||||
log "✅ $name restarted successfully"
|
||||
return 0
|
||||
else
|
||||
log "❌ $name failed to start"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────
|
||||
|
||||
log "══════════════════════════════════════════════════════"
|
||||
log "🔍 Watchdog check starting..."
|
||||
|
||||
APP_OK=false
|
||||
DNS_OK=false
|
||||
|
||||
if check_and_restart "falah-auto-healer" "falah-auto-healer:v6"; then
|
||||
APP_OK=true
|
||||
fi
|
||||
|
||||
if check_and_restart "falah-dns-healer" "falah-dns-healer:latest"; then
|
||||
DNS_OK=true
|
||||
fi
|
||||
|
||||
if $APP_OK && $DNS_OK; then
|
||||
log "✅ Both healers running"
|
||||
elif $APP_OK || $DNS_OK; then
|
||||
log "⚠️ One healer restored, other still down"
|
||||
else
|
||||
log "❌ CRITICAL: Both healers down and could not be restored!"
|
||||
fi
|
||||
|
||||
# Also verify the app itself is running
|
||||
APP_STATUS=$($DOCKER ps --filter name=falah-mobile-staging --format '{{.Status}}' 2>/dev/null)
|
||||
if ! echo "$APP_STATUS" | grep -q "^Up"; then
|
||||
log "⚠️ App container not running — cannot auto-fix from watchdog"
|
||||
log " (app healer should handle this once restarted)"
|
||||
fi
|
||||
|
||||
log "══════════════════════════════════════════════════════"
|
||||
Reference in New Issue
Block a user