diff --git a/auto-healer/auto-heal-agent.sh b/auto-healer/auto-heal-agent.sh index 2d2aa73..f9bc170 100644 --- a/auto-healer/auto-heal-agent.sh +++ b/auto-healer/auto-heal-agent.sh @@ -33,7 +33,7 @@ mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")" "$INCIDENTS_DIR" # ── Logging ────────────────────────────────────────────────── log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$AGENT_LOG" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" >> "$AGENT_LOG" } alert() { @@ -77,10 +77,48 @@ check_disk() { # ── Phase 1: Detect Fault ──────────────────────────────────── detect_fault() { + # Cooldown: skip detection if restore happened within 60s + local cooldown=120 + if [ -f /var/state/falah/last_restore_epoch ]; then + local last_restore + last_restore=$(cat /var/state/falah/last_restore_epoch 2>/dev/null || echo 0) + local now + now=$(date +%s) + local elapsed=$((now - last_restore)) + if [ $elapsed -lt $cooldown ]; then + log " (cooldown active — last restore ${elapsed}s ago, skipping detection)" + echo "healthy" + return + fi + fi + # Returns: diagnosis string (or "healthy") - # 1. Container exists? - if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER_NAME}$"; then + # 0a. Check if CONTAINER_NAME is a Swarm service + local is_swarm_service=false + if docker service ls 2>/dev/null | awk '{print $2}' | grep -x "${CONTAINER_NAME}"; then + is_swarm_service=true + fi + + if [ "$is_swarm_service" = true ]; then + # Swarm service mode — check via docker service ps + local unhealthy + unhealthy=$(docker service ps "$CONTAINER_NAME" --filter 'desired-state=running' --format '{{.CurrentState}}' 2>/dev/null | grep -v "^Running" | head -1) + if [ -n "$unhealthy" ]; then + echo "swarm_service_unhealthy" + return + fi + # Service is running, now check HTTP health + local health_body + health_body=$(check_http "$HEALTH_URL") || { echo "health_down"; return; } + echo "$health_body" | grep -q '"db":false' && { echo "db_down"; return; } + echo "$health_body" | grep -q '"status":"ok"' || { echo "health_invalid"; return; } + echo "healthy" + return + fi + + # 1. Container exists? (standalone mode) + if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "${CONTAINER_NAME}"; then echo "container_missing" return fi @@ -286,11 +324,12 @@ restore_protocol() { echo "$logs" echo "" echo "=== Container Inspect ===" - docker inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED" + docker inspect "$CONTAINER_NAME" 2>/dev/null || docker service inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED" echo "" echo "=== Docker Events (last 5 min) ===" timeout 5 docker events --since "${CHECK_INTERVAL}s" --until "$(date +%s)" \ --filter "container=$CONTAINER_NAME" \ + --filter "service=$CONTAINER_NAME" \ --format '{{.Time}} {{.Type}} {{.Action}} {{.Status}}' 2>/dev/null || echo "EVENTS_FAILED" echo "" echo "=== System Resources ===" @@ -301,8 +340,21 @@ restore_protocol() { log "📝 Incident saved to $INCIDENTS_DIR/$incident_id.log" alert "INFO" "Incident $incident_id logged — root cause: $root_cause" + + # Always save cooldown timestamp to prevent detection loops + date +%s > /var/state/falah/last_restore_epoch - # ── Step 1: Apply action ─────────────────────────────────── + # ── Step 1: Check if this is a Swarm service (don't manage containers directly) ── + if docker service ls 2>/dev/null | awk '{print $2}' | grep -x "${CONTAINER_NAME}"; then + log "ℹ️ Swarm service detected — skipping container management, Swarm handles recovery" + date +%s > /var/state/falah/last_restore_epoch + log "🔄 RESTORE: forcing service update to trigger redeploy..." + docker service update --force "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1 + alert "INFO" "Swarm service $CONTAINER_NAME force-updated (incident: $incident_id)" + return 0 + fi + + # ── Step 2: Apply action ─────────────────────────────────── case "$action" in rollback) log "🔄 RESTORE: rolling back to $ROLLBACK_IMAGE..." @@ -323,7 +375,8 @@ restore_protocol() { sleep 15 if check_http "$HEALTH_URL" > /dev/null 2>&1; then - log "✅ RESTORE: rollback successful — service restored" + date +%s > /var/state/falah/last_restore_epoch + log "✅ RESTORE: rollback successful — service restored" alert "INFO" "Restore protocol: rolled back to $ROLLBACK_IMAGE (incident: $incident_id)" record_restart return 0 @@ -337,7 +390,8 @@ restore_protocol() { docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1 sleep 10 if check_http "$HEALTH_URL" > /dev/null 2>&1; then - log "✅ RESTORE: restart successful (fallback)" + date +%s > /var/state/falah/last_restore_epoch + log "✅ RESTORE: restart successful (fallback)" record_restart return 0 fi