diff --git a/auto-healer/auto-heal-agent.sh b/auto-healer/auto-heal-agent.sh index 4d9968f..e9da3d4 100644 --- a/auto-healer/auto-heal-agent.sh +++ b/auto-healer/auto-heal-agent.sh @@ -13,7 +13,8 @@ # Restart, rollback, nginx reload, prune, escalate # ============================================================ -set -e +# NO set -e — we handle errors explicitly to avoid the entire agent crashing +# on a single failed remediation step. Each function manages its own errors. CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}" HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}" @@ -197,10 +198,16 @@ remediate_rollback() { if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then log "🔄 Rolling back to $ROLLBACK_IMAGE..." - docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true - docker rm "$CONTAINER_NAME" >> "$LOG" 2>&1 || true - # Extract original run command from inspect + # Stop and remove old container if it exists (may have been auto-restarted by Docker) + if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true + sleep 2 + docker rm -f "$CONTAINER_NAME" >> "$LOG" 2>&1 || true + sleep 2 + fi + + # Extract original run config from inspect (if available) or use defaults local ports volumes network env_vars ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000") volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null) @@ -282,7 +289,15 @@ while true; do # ── Phase 2: Act ── case "$DIAGNOSIS" in healthy) - # All good — log silently (only every 10th cycle) + # All good — reset restart count if we've had consecutive healthy cycles + if [ -z "$HEALTHY_CYCLES" ]; then HEALTHY_CYCLES=0; fi + HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1)) + # After 10 healthy cycles (5 min), reset restart counter + if [ "$HEALTHY_CYCLES" -ge 10 ]; then + HEALTHY_CYCLES=0 + # Reset state file — system is stable now + echo 0 > "$STATE_FILE" 2>/dev/null || true + fi ;; container_missing) @@ -290,8 +305,16 @@ while true; do ;; oom_killed|sigkill_oom) - alert "CRITICAL" "OOM killed — increasing memory or rolling back" - remediate_rollback + alert "WARN" "Container killed ($DIAGNOSIS) — restarting" + # If we've had too many restarts, escalate to rollback + local restarts + restarts=$(count_restarts) + if [ "$restarts" -ge "$MAX_RESTARTS" ]; then + alert "CRITICAL" "OOM repeated ($restarts restarts/hour) — rolling back" + remediate_rollback + else + remediate_container_restart "$DIAGNOSIS" + fi ;; crashed_*|not_running)