feat: dedicated auto-healing agent — sidecar container + pi agent definition

- auto-healer/Dockerfile: lightweight Alpine container with Docker CLI - auto-healer/auto-heal-agent.sh: real-time fault detection + remediation engine - Detects: crash, OOM, DB disconnect, gateway down, disk pressure - Fixes: restart, rollback, nginx reload, prune, escalate - Sliding window restart tracking (3/hr max) - auto-healer/docker-compose.healer.yml: sidecar deployment config - Mounts Docker socket for event monitoring - Host network mode for health checks - Persistent state in /var/state/falah - .pi/agents/auto-healer.md: pi agent definition for manual invocation
2026-06-28 02:53:52 +08:00
parent 0098d9ca4a
commit 456295d73e
4 changed files with 465 additions and 0 deletions
@@ -0,0 +1,73 @@
+---
+name: auto-healer
+description: Dedicated auto-healing agent for Falah Mobile — detects faults, diagnoses root cause, executes remediation, escalates if needed
+tools: read, bash, grep, find, ls
+model: 
+---
+
+# Auto-Healer Agent
+
+You are the dedicated auto-healing agent for the Falah Mobile deployment. Your job is to watch for faults, diagnose them, and fix them autonomously.
+
+## Fault Detection
+
+Monitor these signals:
+1. **Container status** — `docker ps`, `docker inspect`
+2. **Health endpoint** — `curl http://localhost:4014/mobile/api/health`
+3. **Gateway health** — `curl http://localhost:7878/mobile/api/health`
+4. **Docker events** — `docker events --filter ...`
+5. **Logs** — `docker logs falah-mobile-staging --tail 50`
+6. **Resource usage** — `df`, `docker stats --no-stream`
+
+## Diagnosis Matrix
+
+| Symptom | Diagnosis | Action |
+|---|---|---|
+| Container not running | crash/oom | Check exit code + OOMKilled flag |
+| Health 000/timeout | container hung | Restart container |
+| Health 200 + db:false | DB disconnected | Restart container |
+| Health 5xx | app error | Check logs → rollback if code error |
+| Gateway 502/000 | nginx down | Reload nginx config |
+| Disk >90% | full disk | `docker system prune -f` |
+| Rate limiter stale | memory leak | Already auto-cleaned (5min interval) |
+
+## Remediation Actions
+
+```bash
+# Restart container
+docker restart falah-mobile-staging
+
+# Reload nginx gateway
+docker exec $(docker ps -q -f name=gateway) nginx -s reload
+
+# Rollback to previous image
+docker stop falah-mobile-staging
+docker rm falah-mobile-staging
+docker run -d --name falah-mobile-staging --restart unless-stopped \
+  -p 4014:3000 \
+  -v /var/services/homes/admin/falah-mobile/data:/app/data \
+  -v /var/services/homes/admin/falah-mobile/.env:/app/.env:ro \
+  --network falah-umbrel_falah-system \
+  falah-mobile:rollback
+
+# Prune disk
+docker system prune -f --volumes
+
+# Run CAB QA after fix
+bash tests/cab-qa.sh http://192.168.0.11:4014/mobile
+```
+
+## Escalation
+
+If 3+ restarts happen within an hour, or if rollback fails:
+1. Log the failure with full diagnostics
+2. Alert via available channels
+3. Do NOT restart again — leave the system in its current state for manual debugging
+
+## Reporting
+
+After each remediation, report:
+- What was detected
+- What action was taken
+- Whether the fix succeeded
+- Current health status
@@ -0,0 +1,25 @@
+# ── Falah Mobile — Auto-Healer Agent ──────────────────────────
+# Dedicated container that monitors the app stack and auto-fixes faults.
+# Runs alongside the main app container with access to Docker socket.
+# ──────────────────────────────────────────────────────────────
+
+FROM alpine:3.19
+
+RUN apk add --no-cache curl docker-cli bash jq
+
+WORKDIR /app
+
+COPY auto-heal-agent.sh /app/auto-heal-agent.sh
+RUN chmod +x /app/auto-heal-agent.sh
+
+# Runtime state
+RUN mkdir -p /var/log/falah /var/state/falah
+
+# Monitor interval in seconds (default 30)
+ENV CHECK_INTERVAL=30
+ENV CONTAINER_NAME=falah-mobile-staging
+ENV HEALTH_URL=http://localhost:4014/mobile/api/health
+ENV GATEWAY_URL=http://localhost:7878/mobile/api/health
+ENV MAX_RESTARTS_PER_HOUR=3
+
+CMD ["/app/auto-heal-agent.sh"]
@@ -0,0 +1,341 @@
+#!/bin/bash
+# ============================================================
+# Falah Mobile — Auto-Healing Agent (Dedicated Container)
+# ============================================================
+# Real-time fault detection + autonomous remediation.
+# Runs as a sidecar container alongside falah-mobile-staging.
+#
+# Detects:
+#   Container crash, health 5xx, DB disconnect, OOM,
+#   Gateway 502, 5xx spikes, disk pressure
+#
+# Fixes:
+#   Restart, rollback, nginx reload, prune, escalate
+# ============================================================
+
+set -e
+
+CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
+HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
+GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
+CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
+MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
+LOG="/var/log/falah/agent.log"
+STATE_DIR="/var/state/falah"
+STATE_FILE="$STATE_DIR/restart-count"
+ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"
+
+mkdir -p "$STATE_DIR" "$(dirname "$LOG")"
+
+# ── Logging ──────────────────────────────────────────────────
+
+log() {
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$LOG"
+}
+
+alert() {
+  local severity="$1" msg="$2"
+  log "[$severity] $msg"
+  # Future: emit to webhook, ntfy, Slack, etc.
+  # curl -sf -X POST -H "Content-Type: application/json" \
+  #   -d "{\"severity\":\"$severity\",\"message\":\"$msg\",\"service\":\"$CONTAINER_NAME\"}" \
+  #   "$ALERT_WEBHOOK" &
+}
+
+# ── Track restarts (sliding window per hour) ─────────────────
+
+count_restarts() {
+  [ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
+  local now window_start count
+  now=$(date +%s)
+  window_start=$((now - 3600))
+  
+  # Filter entries within the last hour
+  awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
+}
+
+record_restart() {
+  date +%s >> "$STATE_FILE"
+  # Trim old entries
+  local now window_start
+  now=$(date +%s)
+  window_start=$((now - 3600))
+  awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
+}
+
+# ── Fault Detection ──────────────────────────────────────────
+
+check_http() {
+  local url="$1" label="$2"
+  local code body
+  body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
+  code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
+  
+  if [ "$code" = "200" ]; then
+    echo "$body"
+    return 0
+  fi
+  return 1
+}
+
+check_db() {
+  local body
+  body=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null) || return 1
+  echo "$body" | grep -q '"db":true' && return 0 || return 1
+}
+
+check_docker_events() {
+  # Check if the container recently crashed
+  local since_seconds="${1:-120}"
+  local since_epoch
+  since_epoch=$(date -d "-${since_seconds} seconds" +%s 2>/dev/null || echo $(( $(date +%s) - since_seconds )))
+  
+  # Use docker events --since to check for recent die/oom events
+  docker events --since "${since_epoch}s" --filter "container=$CONTAINER_NAME" \
+    --filter "event=die" --format '{{.Status}}' 2>/dev/null | head -1
+}
+
+check_disk() {
+  df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
+}
+
+# ── Fault Diagnosis ──────────────────────────────────────────
+
+diagnose() {
+  local fault_reason=""
+  local health_body
+  
+  # 1. Check if container exists
+  if ! docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    echo "container_missing"
+    return
+  fi
+  
+  # 2. Check if container is running
+  local status
+  status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "missing")
+  if [ "$status" != "running" ]; then
+    local exit_code
+    exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
+    local oom
+    oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
+    
+    if [ "$oom" = "true" ]; then
+      echo "oom_killed"
+    elif [ "$exit_code" = "137" ]; then
+      echo "sigkill_oom"
+    elif [ "$exit_code" != "0" ]; then
+      echo "crashed_exit_${exit_code}"
+    else
+      echo "not_running"
+    fi
+    return
+  fi
+  
+  # 3. Check health endpoint
+  health_body=$(check_http "$HEALTH_URL" "direct" 2>/dev/null) || true
+  if [ -z "$health_body" ]; then
+    echo "health_unreachable"
+    return
+  fi
+  
+  # 4. Check DB connectivity
+  local db_status
+  db_status=$(echo "$health_body" | grep -o '"db":false\|"db":true' || echo "unknown")
+  if [ "$db_status" = '"db":false' ]; then
+    echo "db_disconnected"
+    return
+  fi
+  
+  # 5. Check response content
+  if ! echo "$health_body" | grep -q '"status":"ok"'; then
+    echo "health_invalid"
+    return
+  fi
+  
+  # 6. Check gateway
+  if ! check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
+    echo "gateway_down"
+    return
+  fi
+  
+  echo "healthy"
+}
+
+# ── Remediations ─────────────────────────────────────────────
+
+remediate_container_restart() {
+  local diagnosis="$1"
+  local restarts
+  restarts=$(count_restarts)
+  
+  log "🔄 Remediation: restart container (diagnosis=$diagnosis, restarts_last_hour=$restarts)"
+  
+  if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
+    alert "CRITICAL" "Excessive restarts ($restarts/h) — escalating to rollback"
+    remediate_rollback
+    return
+  fi
+  
+  docker restart "$CONTAINER_NAME" >> "$LOG" 2>&1
+  record_restart
+  sleep 10
+  
+  # Verify
+  if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
+    log "✅ Restart successful — container healthy"
+    alert "INFO" "Auto-healed: restart $CONTAINER_NAME (diagnosis=$diagnosis)"
+    return 0
+  else
+    log "⚠️  Restart didn't fix — escalating"
+    remediate_escalate
+  fi
+}
+
+remediate_rollback() {
+  log "🔄 Remediation: rollback to previous image"
+  
+  if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
+    log "🔄 Rolling back to $ROLLBACK_IMAGE..."
+    docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
+    docker rm "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
+    
+    # Extract original run command from inspect
+    local ports volumes network env_vars
+    ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000")
+    volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null)
+    network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge")
+    
+    # Run rollback image with same config
+    docker run -d \
+      --name "$CONTAINER_NAME" \
+      --restart unless-stopped \
+      -p 4014:3000 \
+      $volumes \
+      --network "$network" \
+      "$ROLLBACK_IMAGE" >> "$LOG" 2>&1
+    
+    sleep 15
+    
+    if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
+      log "✅ Rollback successful — previous image restored"
+      alert "INFO" "Auto-healed: rolled back to $ROLLBACK_IMAGE"
+      return 0
+    else
+      alert "CRITICAL" "Rollback also failed — manual intervention required"
+      return 1
+    fi
+  else
+    log "⚠️  No rollback image found — cannot rollback"
+    alert "WARN" "No rollback image available"
+    return 1
+  fi
+}
+
+remediate_gateway() {
+  log "🔄 Remediation: reload nginx gateway"
+  
+  local gw_container
+  gw_container=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
+  
+  if [ -n "$gw_container" ]; then
+    docker exec "$gw_container" nginx -s reload >> "$LOG" 2>&1
+    sleep 3
+    if check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
+      log "✅ Gateway reload successful"
+      alert "INFO" "Auto-healed: reloaded nginx in $gw_container"
+      return 0
+    fi
+  fi
+  log "⚠️  Could not reload gateway"
+  return 1
+}
+
+remediate_prune() {
+  log "🔄 Remediation: docker system prune (disk pressure)"
+  docker system prune -f >> "$LOG" 2>&1
+  log "✅ Prune completed"
+  alert "INFO" "Auto-healed: pruned Docker resources"
+}
+
+remediate_escalate() {
+  alert "CRITICAL" "Auto-heal unable to fix — escalation needed"
+  # Future: send to PagerDuty, OpsGenie, Slack, etc.
+  log "🚨 ESCALATION: Manual intervention required for $CONTAINER_NAME"
+}
+
+# ── Main Loop ────────────────────────────────────────────────
+
+log "══════════════════════════════════════════════════════"
+log "🚀 Auto-Healing Agent starting"
+log "   Container:     $CONTAINER_NAME"
+log "   Health URL:    $HEALTH_URL"
+log "   Gateway URL:   $GATEWAY_URL"
+log "   Interval:      ${CHECK_INTERVAL}s"
+log "   Max restarts:  $MAX_RESTARTS/hour"
+log "══════════════════════════════════════════════════════"
+
+while true; do
+  # ── Phase 1: Detect ──
+  DIAGNOSIS=$(diagnose)
+  
+  # ── Phase 2: Act ──
+  case "$DIAGNOSIS" in
+    healthy)
+      # All good — log silently (only every 10th cycle)
+      ;;
+      
+    container_missing)
+      alert "CRITICAL" "Container $CONTAINER_NAME does not exist"
+      ;;
+      
+    oom_killed|sigkill_oom)
+      alert "CRITICAL" "OOM killed — increasing memory or rolling back"
+      remediate_rollback
+      ;;
+      
+    crashed_*|not_running)
+      alert "WARN" "Container crashed ($DIAGNOSIS)"
+      remediate_container_restart "$DIAGNOSIS"
+      ;;
+      
+    health_unreachable)
+      alert "WARN" "Health endpoint unreachable"
+      remediate_container_restart "$DIAGNOSIS"
+      ;;
+      
+    db_disconnected)
+      alert "WARN" "Database disconnected"
+      remediate_container_restart "$DIAGNOSIS"
+      ;;
+      
+    health_invalid)
+      alert "WARN" "Health response invalid"
+      remediate_container_restart "$DIAGNOSIS"
+      ;;
+      
+    gateway_down)
+      alert "WARN" "Gateway down"
+      remediate_gateway
+      ;;
+      
+    *)
+      alert "WARN" "Unknown diagnosis: $DIAGNOSIS"
+      remediate_container_restart "$DIAGNOSIS"
+      ;;
+  esac
+  
+  # ── Periodic maintenance ──
+  # Check disk every 10 cycles (5 min at 30s interval)
+  CYCLE_COUNTER=$((CYCLE_COUNTER + 1))
+  if [ "$CYCLE_COUNTER" -ge 10 ]; then
+    CYCLE_COUNTER=0
+    DISK_PCT=$(check_disk)
+    if [ "$DISK_PCT" -gt 90 ]; then
+      alert "WARN" "Disk at ${DISK_PCT}% — pruning"
+      remediate_prune
+    fi
+  fi
+  
+  sleep "$CHECK_INTERVAL"
+done
@@ -0,0 +1,26 @@
+version: "3.8"
+services:
+  falah-auto-healer:
+    build:
+      context: ./auto-healer
+      dockerfile: Dockerfile
+    image: falah-auto-healer:latest
+    container_name: falah-auto-healer
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /var/log/falah:/var/log/falah
+      - /var/state/falah:/var/state/falah
+    environment:
+      - CONTAINER_NAME=falah-mobile-staging
+      - HEALTH_URL=http://192.168.0.11:4014/mobile/api/health
+      - GATEWAY_URL=http://192.168.0.11:7878/mobile/api/health
+      - CHECK_INTERVAL=30
+      - MAX_RESTARTS_PER_HOUR=3
+      - ROLLBACK_IMAGE=falah-mobile:rollback
+    network_mode: "host"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"