diff --git a/.pi/agents/auto-healer.md b/.pi/agents/auto-healer.md new file mode 100644 index 0000000..7de328d --- /dev/null +++ b/.pi/agents/auto-healer.md @@ -0,0 +1,73 @@ +--- +name: auto-healer +description: Dedicated auto-healing agent for Falah Mobile — detects faults, diagnoses root cause, executes remediation, escalates if needed +tools: read, bash, grep, find, ls +model: +--- + +# Auto-Healer Agent + +You are the dedicated auto-healing agent for the Falah Mobile deployment. Your job is to watch for faults, diagnose them, and fix them autonomously. + +## Fault Detection + +Monitor these signals: +1. **Container status** — `docker ps`, `docker inspect` +2. **Health endpoint** — `curl http://localhost:4014/mobile/api/health` +3. **Gateway health** — `curl http://localhost:7878/mobile/api/health` +4. **Docker events** — `docker events --filter ...` +5. **Logs** — `docker logs falah-mobile-staging --tail 50` +6. **Resource usage** — `df`, `docker stats --no-stream` + +## Diagnosis Matrix + +| Symptom | Diagnosis | Action | +|---|---|---| +| Container not running | crash/oom | Check exit code + OOMKilled flag | +| Health 000/timeout | container hung | Restart container | +| Health 200 + db:false | DB disconnected | Restart container | +| Health 5xx | app error | Check logs → rollback if code error | +| Gateway 502/000 | nginx down | Reload nginx config | +| Disk >90% | full disk | `docker system prune -f` | +| Rate limiter stale | memory leak | Already auto-cleaned (5min interval) | + +## Remediation Actions + +```bash +# Restart container +docker restart falah-mobile-staging + +# Reload nginx gateway +docker exec $(docker ps -q -f name=gateway) nginx -s reload + +# Rollback to previous image +docker stop falah-mobile-staging +docker rm falah-mobile-staging +docker run -d --name falah-mobile-staging --restart unless-stopped \ + -p 4014:3000 \ + -v /var/services/homes/admin/falah-mobile/data:/app/data \ + -v /var/services/homes/admin/falah-mobile/.env:/app/.env:ro \ + --network falah-umbrel_falah-system \ + falah-mobile:rollback + +# Prune disk +docker system prune -f --volumes + +# Run CAB QA after fix +bash tests/cab-qa.sh http://192.168.0.11:4014/mobile +``` + +## Escalation + +If 3+ restarts happen within an hour, or if rollback fails: +1. Log the failure with full diagnostics +2. Alert via available channels +3. Do NOT restart again — leave the system in its current state for manual debugging + +## Reporting + +After each remediation, report: +- What was detected +- What action was taken +- Whether the fix succeeded +- Current health status diff --git a/auto-healer/Dockerfile b/auto-healer/Dockerfile new file mode 100644 index 0000000..2eaa20f --- /dev/null +++ b/auto-healer/Dockerfile @@ -0,0 +1,25 @@ +# ── Falah Mobile — Auto-Healer Agent ────────────────────────── +# Dedicated container that monitors the app stack and auto-fixes faults. +# Runs alongside the main app container with access to Docker socket. +# ────────────────────────────────────────────────────────────── + +FROM alpine:3.19 + +RUN apk add --no-cache curl docker-cli bash jq + +WORKDIR /app + +COPY auto-heal-agent.sh /app/auto-heal-agent.sh +RUN chmod +x /app/auto-heal-agent.sh + +# Runtime state +RUN mkdir -p /var/log/falah /var/state/falah + +# Monitor interval in seconds (default 30) +ENV CHECK_INTERVAL=30 +ENV CONTAINER_NAME=falah-mobile-staging +ENV HEALTH_URL=http://localhost:4014/mobile/api/health +ENV GATEWAY_URL=http://localhost:7878/mobile/api/health +ENV MAX_RESTARTS_PER_HOUR=3 + +CMD ["/app/auto-heal-agent.sh"] diff --git a/auto-healer/auto-heal-agent.sh b/auto-healer/auto-heal-agent.sh new file mode 100644 index 0000000..4d9968f --- /dev/null +++ b/auto-healer/auto-heal-agent.sh @@ -0,0 +1,341 @@ +#!/bin/bash +# ============================================================ +# Falah Mobile — Auto-Healing Agent (Dedicated Container) +# ============================================================ +# Real-time fault detection + autonomous remediation. +# Runs as a sidecar container alongside falah-mobile-staging. +# +# Detects: +# Container crash, health 5xx, DB disconnect, OOM, +# Gateway 502, 5xx spikes, disk pressure +# +# Fixes: +# Restart, rollback, nginx reload, prune, escalate +# ============================================================ + +set -e + +CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}" +HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}" +GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}" +CHECK_INTERVAL="${CHECK_INTERVAL:-30}" +MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}" +LOG="/var/log/falah/agent.log" +STATE_DIR="/var/state/falah" +STATE_FILE="$STATE_DIR/restart-count" +ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}" + +mkdir -p "$STATE_DIR" "$(dirname "$LOG")" + +# ── Logging ────────────────────────────────────────────────── + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$LOG" +} + +alert() { + local severity="$1" msg="$2" + log "[$severity] $msg" + # Future: emit to webhook, ntfy, Slack, etc. + # curl -sf -X POST -H "Content-Type: application/json" \ + # -d "{\"severity\":\"$severity\",\"message\":\"$msg\",\"service\":\"$CONTAINER_NAME\"}" \ + # "$ALERT_WEBHOOK" & +} + +# ── Track restarts (sliding window per hour) ───────────────── + +count_restarts() { + [ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE" + local now window_start count + now=$(date +%s) + window_start=$((now - 3600)) + + # Filter entries within the last hour + awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l +} + +record_restart() { + date +%s >> "$STATE_FILE" + # Trim old entries + local now window_start + now=$(date +%s) + window_start=$((now - 3600)) + awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE" +} + +# ── Fault Detection ────────────────────────────────────────── + +check_http() { + local url="$1" label="$2" + local code body + body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true + code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000") + + if [ "$code" = "200" ]; then + echo "$body" + return 0 + fi + return 1 +} + +check_db() { + local body + body=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null) || return 1 + echo "$body" | grep -q '"db":true' && return 0 || return 1 +} + +check_docker_events() { + # Check if the container recently crashed + local since_seconds="${1:-120}" + local since_epoch + since_epoch=$(date -d "-${since_seconds} seconds" +%s 2>/dev/null || echo $(( $(date +%s) - since_seconds ))) + + # Use docker events --since to check for recent die/oom events + docker events --since "${since_epoch}s" --filter "container=$CONTAINER_NAME" \ + --filter "event=die" --format '{{.Status}}' 2>/dev/null | head -1 +} + +check_disk() { + df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}' +} + +# ── Fault Diagnosis ────────────────────────────────────────── + +diagnose() { + local fault_reason="" + local health_body + + # 1. Check if container exists + if ! docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + echo "container_missing" + return + fi + + # 2. Check if container is running + local status + status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "missing") + if [ "$status" != "running" ]; then + local exit_code + exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0") + local oom + oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false") + + if [ "$oom" = "true" ]; then + echo "oom_killed" + elif [ "$exit_code" = "137" ]; then + echo "sigkill_oom" + elif [ "$exit_code" != "0" ]; then + echo "crashed_exit_${exit_code}" + else + echo "not_running" + fi + return + fi + + # 3. Check health endpoint + health_body=$(check_http "$HEALTH_URL" "direct" 2>/dev/null) || true + if [ -z "$health_body" ]; then + echo "health_unreachable" + return + fi + + # 4. Check DB connectivity + local db_status + db_status=$(echo "$health_body" | grep -o '"db":false\|"db":true' || echo "unknown") + if [ "$db_status" = '"db":false' ]; then + echo "db_disconnected" + return + fi + + # 5. Check response content + if ! echo "$health_body" | grep -q '"status":"ok"'; then + echo "health_invalid" + return + fi + + # 6. Check gateway + if ! check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then + echo "gateway_down" + return + fi + + echo "healthy" +} + +# ── Remediations ───────────────────────────────────────────── + +remediate_container_restart() { + local diagnosis="$1" + local restarts + restarts=$(count_restarts) + + log "🔄 Remediation: restart container (diagnosis=$diagnosis, restarts_last_hour=$restarts)" + + if [ "$restarts" -ge "$MAX_RESTARTS" ]; then + alert "CRITICAL" "Excessive restarts ($restarts/h) — escalating to rollback" + remediate_rollback + return + fi + + docker restart "$CONTAINER_NAME" >> "$LOG" 2>&1 + record_restart + sleep 10 + + # Verify + if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then + log "✅ Restart successful — container healthy" + alert "INFO" "Auto-healed: restart $CONTAINER_NAME (diagnosis=$diagnosis)" + return 0 + else + log "⚠️ Restart didn't fix — escalating" + remediate_escalate + fi +} + +remediate_rollback() { + log "🔄 Remediation: rollback to previous image" + + if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then + log "🔄 Rolling back to $ROLLBACK_IMAGE..." + docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true + docker rm "$CONTAINER_NAME" >> "$LOG" 2>&1 || true + + # Extract original run command from inspect + local ports volumes network env_vars + ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000") + volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null) + network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge") + + # Run rollback image with same config + docker run -d \ + --name "$CONTAINER_NAME" \ + --restart unless-stopped \ + -p 4014:3000 \ + $volumes \ + --network "$network" \ + "$ROLLBACK_IMAGE" >> "$LOG" 2>&1 + + sleep 15 + + if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then + log "✅ Rollback successful — previous image restored" + alert "INFO" "Auto-healed: rolled back to $ROLLBACK_IMAGE" + return 0 + else + alert "CRITICAL" "Rollback also failed — manual intervention required" + return 1 + fi + else + log "⚠️ No rollback image found — cannot rollback" + alert "WARN" "No rollback image available" + return 1 + fi +} + +remediate_gateway() { + log "🔄 Remediation: reload nginx gateway" + + local gw_container + gw_container=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1) + + if [ -n "$gw_container" ]; then + docker exec "$gw_container" nginx -s reload >> "$LOG" 2>&1 + sleep 3 + if check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then + log "✅ Gateway reload successful" + alert "INFO" "Auto-healed: reloaded nginx in $gw_container" + return 0 + fi + fi + log "⚠️ Could not reload gateway" + return 1 +} + +remediate_prune() { + log "🔄 Remediation: docker system prune (disk pressure)" + docker system prune -f >> "$LOG" 2>&1 + log "✅ Prune completed" + alert "INFO" "Auto-healed: pruned Docker resources" +} + +remediate_escalate() { + alert "CRITICAL" "Auto-heal unable to fix — escalation needed" + # Future: send to PagerDuty, OpsGenie, Slack, etc. + log "🚨 ESCALATION: Manual intervention required for $CONTAINER_NAME" +} + +# ── Main Loop ──────────────────────────────────────────────── + +log "══════════════════════════════════════════════════════" +log "🚀 Auto-Healing Agent starting" +log " Container: $CONTAINER_NAME" +log " Health URL: $HEALTH_URL" +log " Gateway URL: $GATEWAY_URL" +log " Interval: ${CHECK_INTERVAL}s" +log " Max restarts: $MAX_RESTARTS/hour" +log "══════════════════════════════════════════════════════" + +while true; do + # ── Phase 1: Detect ── + DIAGNOSIS=$(diagnose) + + # ── Phase 2: Act ── + case "$DIAGNOSIS" in + healthy) + # All good — log silently (only every 10th cycle) + ;; + + container_missing) + alert "CRITICAL" "Container $CONTAINER_NAME does not exist" + ;; + + oom_killed|sigkill_oom) + alert "CRITICAL" "OOM killed — increasing memory or rolling back" + remediate_rollback + ;; + + crashed_*|not_running) + alert "WARN" "Container crashed ($DIAGNOSIS)" + remediate_container_restart "$DIAGNOSIS" + ;; + + health_unreachable) + alert "WARN" "Health endpoint unreachable" + remediate_container_restart "$DIAGNOSIS" + ;; + + db_disconnected) + alert "WARN" "Database disconnected" + remediate_container_restart "$DIAGNOSIS" + ;; + + health_invalid) + alert "WARN" "Health response invalid" + remediate_container_restart "$DIAGNOSIS" + ;; + + gateway_down) + alert "WARN" "Gateway down" + remediate_gateway + ;; + + *) + alert "WARN" "Unknown diagnosis: $DIAGNOSIS" + remediate_container_restart "$DIAGNOSIS" + ;; + esac + + # ── Periodic maintenance ── + # Check disk every 10 cycles (5 min at 30s interval) + CYCLE_COUNTER=$((CYCLE_COUNTER + 1)) + if [ "$CYCLE_COUNTER" -ge 10 ]; then + CYCLE_COUNTER=0 + DISK_PCT=$(check_disk) + if [ "$DISK_PCT" -gt 90 ]; then + alert "WARN" "Disk at ${DISK_PCT}% — pruning" + remediate_prune + fi + fi + + sleep "$CHECK_INTERVAL" +done diff --git a/auto-healer/docker-compose.healer.yml b/auto-healer/docker-compose.healer.yml new file mode 100644 index 0000000..25b17b1 --- /dev/null +++ b/auto-healer/docker-compose.healer.yml @@ -0,0 +1,26 @@ +version: "3.8" +services: + falah-auto-healer: + build: + context: ./auto-healer + dockerfile: Dockerfile + image: falah-auto-healer:latest + container_name: falah-auto-healer + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log/falah:/var/log/falah + - /var/state/falah:/var/state/falah + environment: + - CONTAINER_NAME=falah-mobile-staging + - HEALTH_URL=http://192.168.0.11:4014/mobile/api/health + - GATEWAY_URL=http://192.168.0.11:7878/mobile/api/health + - CHECK_INTERVAL=30 + - MAX_RESTARTS_PER_HOUR=3 + - ROLLBACK_IMAGE=falah-mobile:rollback + network_mode: "host" + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3"