From 4fcfa4375d6f6f6409b839ba3b0c57a5cae134cd Mon Sep 17 00:00:00 2001
From: Pi Agent <pi-agent@falahos.my>
Date: Sun, 28 Jun 2026 03:03:30 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20auto-healer=20v2=20=E2=80=94=20fix=20set?=
 =?UTF-8?q?=20-e=20crash,=20proper=20restart=20vs=20rollback=20escalation,?=
 =?UTF-8?q?=20restart=20count=20reset=20on=20stability?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Removed set -e — agent no longer crashes when a remediation step fails
- SIGKILL/OOM → restart first, only rollback if exceeding max restarts/hour
- Added HEALTHY_CYCLES counter: after 10 consecutive healthy cycles (5 min),
  reset restart tracking state (system considered stable)
- Rollback now handles the case where Docker restart policy already revived
  the container (rm -f before re-creating)
---
 auto-healer/auto-heal-agent.sh | 37 +++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/auto-healer/auto-heal-agent.sh b/auto-healer/auto-heal-agent.sh
index 4d9968f..e9da3d4 100644
--- a/auto-healer/auto-heal-agent.sh
+++ b/auto-healer/auto-heal-agent.sh
@@ -13,7 +13,8 @@
 #   Restart, rollback, nginx reload, prune, escalate
 # ============================================================
 
-set -e
+# NO set -e — we handle errors explicitly to avoid the entire agent crashing
+# on a single failed remediation step. Each function manages its own errors.
 
 CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
 HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
@@ -197,10 +198,16 @@ remediate_rollback() {
   
   if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
     log "🔄 Rolling back to $ROLLBACK_IMAGE..."
-    docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
-    docker rm "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
     
-    # Extract original run command from inspect
+    # Stop and remove old container if it exists (may have been auto-restarted by Docker)
+    if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+      docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
+      sleep 2
+      docker rm -f "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
+      sleep 2
+    fi
+    
+    # Extract original run config from inspect (if available) or use defaults
     local ports volumes network env_vars
     ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000")
     volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null)
@@ -282,7 +289,15 @@ while true; do
   # ── Phase 2: Act ──
   case "$DIAGNOSIS" in
     healthy)
-      # All good — log silently (only every 10th cycle)
+      # All good — reset restart count if we've had consecutive healthy cycles
+      if [ -z "$HEALTHY_CYCLES" ]; then HEALTHY_CYCLES=0; fi
+      HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1))
+      # After 10 healthy cycles (5 min), reset restart counter
+      if [ "$HEALTHY_CYCLES" -ge 10 ]; then
+        HEALTHY_CYCLES=0
+        # Reset state file — system is stable now
+        echo 0 > "$STATE_FILE" 2>/dev/null || true
+      fi
       ;;
       
     container_missing)
@@ -290,8 +305,16 @@ while true; do
       ;;
       
     oom_killed|sigkill_oom)
-      alert "CRITICAL" "OOM killed — increasing memory or rolling back"
-      remediate_rollback
+      alert "WARN" "Container killed ($DIAGNOSIS) — restarting"
+      # If we've had too many restarts, escalate to rollback
+      local restarts
+      restarts=$(count_restarts)
+      if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
+        alert "CRITICAL" "OOM repeated ($restarts restarts/hour) — rolling back"
+        remediate_rollback
+      else
+        remediate_container_restart "$DIAGNOSIS"
+      fi
       ;;
       
     crashed_*|not_running)