fix: healers v15 — cooldown + fix log() stdout corruption + swarm service detection

- FIXED core bug: log() used tee -a, writing to both log file AND stdout,
  corrupting DIAGNOSIS=$(detect_fault) and causing infinite fault loops
- ADDED 120s cooldown after restore to prevent detection loops during startup
- ADDED swarm service detection to auto-healer (handles both standalone
  containers and Docker Swarm services)
- FIXED swarm service detection: awk now uses $2 (service name) not $1 (ID)
- FIXED exact match for service names (grep -x)
- FIXED restore protocol to use docker service update --force for swarm services
- DEPLOYED auto-healers to both Synology (v15) and Contabo (v15)
- DNS healer also deployed to Contabo (v3)
- DNS healer has swarm health monitoring (checks every 2 cycles)
- All 3 nodes in Docker Swarm: Synology (manager), colima (worker), Contabo (leader)

auto-healer/auto-heal-agent.sh

@@ -33,7 +33,7 @@ mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")" "$INCIDENTS_DIR"
 # ── Logging ──────────────────────────────────────────────────
 
 log() {
-  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$AGENT_LOG"
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" >> "$AGENT_LOG"
 }
 
 alert() {
@@ -77,10 +77,48 @@ check_disk() {
 # ── Phase 1: Detect Fault ────────────────────────────────────
 
 detect_fault() {
+  # Cooldown: skip detection if restore happened within 60s
+  local cooldown=120
+  if [ -f /var/state/falah/last_restore_epoch ]; then
+    local last_restore
+    last_restore=$(cat /var/state/falah/last_restore_epoch 2>/dev/null || echo 0)
+    local now
+    now=$(date +%s)
+    local elapsed=$((now - last_restore))
+    if [ $elapsed -lt $cooldown ]; then
+      log "   (cooldown active — last restore ${elapsed}s ago, skipping detection)"
+      echo "healthy"
+      return
+    fi
+  fi
+
   # Returns: diagnosis string (or "healthy")
   
-  # 1. Container exists?
+  # 0a. Check if CONTAINER_NAME is a Swarm service
-  if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER_NAME}$"; then
+  local is_swarm_service=false
+  if docker service ls 2>/dev/null | awk '{print $2}' | grep -x "${CONTAINER_NAME}"; then
+    is_swarm_service=true
+  fi
+  
+  if [ "$is_swarm_service" = true ]; then
+    # Swarm service mode — check via docker service ps
+    local unhealthy
+    unhealthy=$(docker service ps "$CONTAINER_NAME" --filter 'desired-state=running' --format '{{.CurrentState}}' 2>/dev/null | grep -v "^Running" | head -1)
+    if [ -n "$unhealthy" ]; then
+      echo "swarm_service_unhealthy"
+      return
+    fi
+    # Service is running, now check HTTP health
+    local health_body
+    health_body=$(check_http "$HEALTH_URL") || { echo "health_down"; return; }
+    echo "$health_body" | grep -q '"db":false' && { echo "db_down"; return; }
+    echo "$health_body" | grep -q '"status":"ok"' || { echo "health_invalid"; return; }
+    echo "healthy"
+    return
+  fi
+  
+  # 1. Container exists? (standalone mode)
+  if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "${CONTAINER_NAME}"; then
     echo "container_missing"
     return
   fi
@@ -286,11 +324,12 @@ restore_protocol() {
     echo "$logs"
     echo ""
     echo "=== Container Inspect ==="
-    docker inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED"
+    docker inspect "$CONTAINER_NAME" 2>/dev/null || docker service inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED"
     echo ""
     echo "=== Docker Events (last 5 min) ==="
     timeout 5 docker events --since "${CHECK_INTERVAL}s" --until "$(date +%s)" \
       --filter "container=$CONTAINER_NAME" \
+    --filter "service=$CONTAINER_NAME" \
       --format '{{.Time}} {{.Type}} {{.Action}} {{.Status}}' 2>/dev/null || echo "EVENTS_FAILED"
     echo ""
     echo "=== System Resources ==="
@@ -302,7 +341,20 @@ restore_protocol() {
   log "📝 Incident saved to $INCIDENTS_DIR/$incident_id.log"
   alert "INFO" "Incident $incident_id logged — root cause: $root_cause"
   
-  # ── Step 1: Apply action ───────────────────────────────────
+  # Always save cooldown timestamp to prevent detection loops
+  date +%s > /var/state/falah/last_restore_epoch
+
+  # ── Step 1: Check if this is a Swarm service (don't manage containers directly) ──
+  if docker service ls 2>/dev/null | awk '{print $2}' | grep -x "${CONTAINER_NAME}"; then
+    log "ℹ️  Swarm service detected — skipping container management, Swarm handles recovery"
+    date +%s > /var/state/falah/last_restore_epoch
+    log "🔄 RESTORE: forcing service update to trigger redeploy..."
+    docker service update --force "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
+    alert "INFO" "Swarm service $CONTAINER_NAME force-updated (incident: $incident_id)"
+    return 0
+  fi
+
+  # ── Step 2: Apply action ───────────────────────────────────
   case "$action" in
     rollback)
       log "🔄 RESTORE: rolling back to $ROLLBACK_IMAGE..."
@@ -323,7 +375,8 @@ restore_protocol() {
 
         sleep 15
         if check_http "$HEALTH_URL" > /dev/null 2>&1; then
-          log "✅ RESTORE: rollback successful — service restored"
+          date +%s > /var/state/falah/last_restore_epoch
+    log "✅ RESTORE: rollback successful — service restored"
           alert "INFO" "Restore protocol: rolled back to $ROLLBACK_IMAGE (incident: $incident_id)"
           record_restart
           return 0
@@ -337,7 +390,8 @@ restore_protocol() {
         docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
         sleep 10
         if check_http "$HEALTH_URL" > /dev/null 2>&1; then
-          log "✅ RESTORE: restart successful (fallback)"
+          date +%s > /var/state/falah/last_restore_epoch
+        log "✅ RESTORE: restart successful (fallback)"
           record_restart
           return 0
         fi