feat: dedicated auto-healing agent — sidecar container + pi agent definition
- auto-healer/Dockerfile: lightweight Alpine container with Docker CLI - auto-healer/auto-heal-agent.sh: real-time fault detection + remediation engine - Detects: crash, OOM, DB disconnect, gateway down, disk pressure - Fixes: restart, rollback, nginx reload, prune, escalate - Sliding window restart tracking (3/hr max) - auto-healer/docker-compose.healer.yml: sidecar deployment config - Mounts Docker socket for event monitoring - Host network mode for health checks - Persistent state in /var/state/falah - .pi/agents/auto-healer.md: pi agent definition for manual invocation
This commit is contained in:
@@ -0,0 +1,341 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# Falah Mobile — Auto-Healing Agent (Dedicated Container)
|
||||
# ============================================================
|
||||
# Real-time fault detection + autonomous remediation.
|
||||
# Runs as a sidecar container alongside falah-mobile-staging.
|
||||
#
|
||||
# Detects:
|
||||
# Container crash, health 5xx, DB disconnect, OOM,
|
||||
# Gateway 502, 5xx spikes, disk pressure
|
||||
#
|
||||
# Fixes:
|
||||
# Restart, rollback, nginx reload, prune, escalate
|
||||
# ============================================================
|
||||
|
||||
set -e
|
||||
|
||||
CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
|
||||
HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
|
||||
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
|
||||
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
|
||||
MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
|
||||
LOG="/var/log/falah/agent.log"
|
||||
STATE_DIR="/var/state/falah"
|
||||
STATE_FILE="$STATE_DIR/restart-count"
|
||||
ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"
|
||||
|
||||
mkdir -p "$STATE_DIR" "$(dirname "$LOG")"
|
||||
|
||||
# ── Logging ──────────────────────────────────────────────────
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$LOG"
|
||||
}
|
||||
|
||||
alert() {
|
||||
local severity="$1" msg="$2"
|
||||
log "[$severity] $msg"
|
||||
# Future: emit to webhook, ntfy, Slack, etc.
|
||||
# curl -sf -X POST -H "Content-Type: application/json" \
|
||||
# -d "{\"severity\":\"$severity\",\"message\":\"$msg\",\"service\":\"$CONTAINER_NAME\"}" \
|
||||
# "$ALERT_WEBHOOK" &
|
||||
}
|
||||
|
||||
# ── Track restarts (sliding window per hour) ─────────────────
|
||||
|
||||
count_restarts() {
|
||||
[ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
|
||||
local now window_start count
|
||||
now=$(date +%s)
|
||||
window_start=$((now - 3600))
|
||||
|
||||
# Filter entries within the last hour
|
||||
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
|
||||
}
|
||||
|
||||
record_restart() {
|
||||
date +%s >> "$STATE_FILE"
|
||||
# Trim old entries
|
||||
local now window_start
|
||||
now=$(date +%s)
|
||||
window_start=$((now - 3600))
|
||||
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||||
}
|
||||
|
||||
# ── Fault Detection ──────────────────────────────────────────
|
||||
|
||||
check_http() {
|
||||
local url="$1" label="$2"
|
||||
local code body
|
||||
body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
|
||||
|
||||
if [ "$code" = "200" ]; then
|
||||
echo "$body"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
check_db() {
|
||||
local body
|
||||
body=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null) || return 1
|
||||
echo "$body" | grep -q '"db":true' && return 0 || return 1
|
||||
}
|
||||
|
||||
check_docker_events() {
|
||||
# Check if the container recently crashed
|
||||
local since_seconds="${1:-120}"
|
||||
local since_epoch
|
||||
since_epoch=$(date -d "-${since_seconds} seconds" +%s 2>/dev/null || echo $(( $(date +%s) - since_seconds )))
|
||||
|
||||
# Use docker events --since to check for recent die/oom events
|
||||
docker events --since "${since_epoch}s" --filter "container=$CONTAINER_NAME" \
|
||||
--filter "event=die" --format '{{.Status}}' 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
check_disk() {
|
||||
df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
|
||||
}
|
||||
|
||||
# ── Fault Diagnosis ──────────────────────────────────────────
|
||||
|
||||
diagnose() {
|
||||
local fault_reason=""
|
||||
local health_body
|
||||
|
||||
# 1. Check if container exists
|
||||
if ! docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
|
||||
echo "container_missing"
|
||||
return
|
||||
fi
|
||||
|
||||
# 2. Check if container is running
|
||||
local status
|
||||
status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "missing")
|
||||
if [ "$status" != "running" ]; then
|
||||
local exit_code
|
||||
exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
|
||||
local oom
|
||||
oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
|
||||
|
||||
if [ "$oom" = "true" ]; then
|
||||
echo "oom_killed"
|
||||
elif [ "$exit_code" = "137" ]; then
|
||||
echo "sigkill_oom"
|
||||
elif [ "$exit_code" != "0" ]; then
|
||||
echo "crashed_exit_${exit_code}"
|
||||
else
|
||||
echo "not_running"
|
||||
fi
|
||||
return
|
||||
fi
|
||||
|
||||
# 3. Check health endpoint
|
||||
health_body=$(check_http "$HEALTH_URL" "direct" 2>/dev/null) || true
|
||||
if [ -z "$health_body" ]; then
|
||||
echo "health_unreachable"
|
||||
return
|
||||
fi
|
||||
|
||||
# 4. Check DB connectivity
|
||||
local db_status
|
||||
db_status=$(echo "$health_body" | grep -o '"db":false\|"db":true' || echo "unknown")
|
||||
if [ "$db_status" = '"db":false' ]; then
|
||||
echo "db_disconnected"
|
||||
return
|
||||
fi
|
||||
|
||||
# 5. Check response content
|
||||
if ! echo "$health_body" | grep -q '"status":"ok"'; then
|
||||
echo "health_invalid"
|
||||
return
|
||||
fi
|
||||
|
||||
# 6. Check gateway
|
||||
if ! check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
|
||||
echo "gateway_down"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "healthy"
|
||||
}
|
||||
|
||||
# ── Remediations ─────────────────────────────────────────────
|
||||
|
||||
remediate_container_restart() {
|
||||
local diagnosis="$1"
|
||||
local restarts
|
||||
restarts=$(count_restarts)
|
||||
|
||||
log "🔄 Remediation: restart container (diagnosis=$diagnosis, restarts_last_hour=$restarts)"
|
||||
|
||||
if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
|
||||
alert "CRITICAL" "Excessive restarts ($restarts/h) — escalating to rollback"
|
||||
remediate_rollback
|
||||
return
|
||||
fi
|
||||
|
||||
docker restart "$CONTAINER_NAME" >> "$LOG" 2>&1
|
||||
record_restart
|
||||
sleep 10
|
||||
|
||||
# Verify
|
||||
if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
|
||||
log "✅ Restart successful — container healthy"
|
||||
alert "INFO" "Auto-healed: restart $CONTAINER_NAME (diagnosis=$diagnosis)"
|
||||
return 0
|
||||
else
|
||||
log "⚠️ Restart didn't fix — escalating"
|
||||
remediate_escalate
|
||||
fi
|
||||
}
|
||||
|
||||
remediate_rollback() {
|
||||
log "🔄 Remediation: rollback to previous image"
|
||||
|
||||
if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
|
||||
log "🔄 Rolling back to $ROLLBACK_IMAGE..."
|
||||
docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
|
||||
docker rm "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
|
||||
|
||||
# Extract original run command from inspect
|
||||
local ports volumes network env_vars
|
||||
ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000")
|
||||
volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null)
|
||||
network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge")
|
||||
|
||||
# Run rollback image with same config
|
||||
docker run -d \
|
||||
--name "$CONTAINER_NAME" \
|
||||
--restart unless-stopped \
|
||||
-p 4014:3000 \
|
||||
$volumes \
|
||||
--network "$network" \
|
||||
"$ROLLBACK_IMAGE" >> "$LOG" 2>&1
|
||||
|
||||
sleep 15
|
||||
|
||||
if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
|
||||
log "✅ Rollback successful — previous image restored"
|
||||
alert "INFO" "Auto-healed: rolled back to $ROLLBACK_IMAGE"
|
||||
return 0
|
||||
else
|
||||
alert "CRITICAL" "Rollback also failed — manual intervention required"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "⚠️ No rollback image found — cannot rollback"
|
||||
alert "WARN" "No rollback image available"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
remediate_gateway() {
|
||||
log "🔄 Remediation: reload nginx gateway"
|
||||
|
||||
local gw_container
|
||||
gw_container=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
|
||||
|
||||
if [ -n "$gw_container" ]; then
|
||||
docker exec "$gw_container" nginx -s reload >> "$LOG" 2>&1
|
||||
sleep 3
|
||||
if check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
|
||||
log "✅ Gateway reload successful"
|
||||
alert "INFO" "Auto-healed: reloaded nginx in $gw_container"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
log "⚠️ Could not reload gateway"
|
||||
return 1
|
||||
}
|
||||
|
||||
remediate_prune() {
|
||||
log "🔄 Remediation: docker system prune (disk pressure)"
|
||||
docker system prune -f >> "$LOG" 2>&1
|
||||
log "✅ Prune completed"
|
||||
alert "INFO" "Auto-healed: pruned Docker resources"
|
||||
}
|
||||
|
||||
remediate_escalate() {
|
||||
alert "CRITICAL" "Auto-heal unable to fix — escalation needed"
|
||||
# Future: send to PagerDuty, OpsGenie, Slack, etc.
|
||||
log "🚨 ESCALATION: Manual intervention required for $CONTAINER_NAME"
|
||||
}
|
||||
|
||||
# ── Main Loop ────────────────────────────────────────────────
|
||||
|
||||
log "══════════════════════════════════════════════════════"
|
||||
log "🚀 Auto-Healing Agent starting"
|
||||
log " Container: $CONTAINER_NAME"
|
||||
log " Health URL: $HEALTH_URL"
|
||||
log " Gateway URL: $GATEWAY_URL"
|
||||
log " Interval: ${CHECK_INTERVAL}s"
|
||||
log " Max restarts: $MAX_RESTARTS/hour"
|
||||
log "══════════════════════════════════════════════════════"
|
||||
|
||||
while true; do
|
||||
# ── Phase 1: Detect ──
|
||||
DIAGNOSIS=$(diagnose)
|
||||
|
||||
# ── Phase 2: Act ──
|
||||
case "$DIAGNOSIS" in
|
||||
healthy)
|
||||
# All good — log silently (only every 10th cycle)
|
||||
;;
|
||||
|
||||
container_missing)
|
||||
alert "CRITICAL" "Container $CONTAINER_NAME does not exist"
|
||||
;;
|
||||
|
||||
oom_killed|sigkill_oom)
|
||||
alert "CRITICAL" "OOM killed — increasing memory or rolling back"
|
||||
remediate_rollback
|
||||
;;
|
||||
|
||||
crashed_*|not_running)
|
||||
alert "WARN" "Container crashed ($DIAGNOSIS)"
|
||||
remediate_container_restart "$DIAGNOSIS"
|
||||
;;
|
||||
|
||||
health_unreachable)
|
||||
alert "WARN" "Health endpoint unreachable"
|
||||
remediate_container_restart "$DIAGNOSIS"
|
||||
;;
|
||||
|
||||
db_disconnected)
|
||||
alert "WARN" "Database disconnected"
|
||||
remediate_container_restart "$DIAGNOSIS"
|
||||
;;
|
||||
|
||||
health_invalid)
|
||||
alert "WARN" "Health response invalid"
|
||||
remediate_container_restart "$DIAGNOSIS"
|
||||
;;
|
||||
|
||||
gateway_down)
|
||||
alert "WARN" "Gateway down"
|
||||
remediate_gateway
|
||||
;;
|
||||
|
||||
*)
|
||||
alert "WARN" "Unknown diagnosis: $DIAGNOSIS"
|
||||
remediate_container_restart "$DIAGNOSIS"
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Periodic maintenance ──
|
||||
# Check disk every 10 cycles (5 min at 30s interval)
|
||||
CYCLE_COUNTER=$((CYCLE_COUNTER + 1))
|
||||
if [ "$CYCLE_COUNTER" -ge 10 ]; then
|
||||
CYCLE_COUNTER=0
|
||||
DISK_PCT=$(check_disk)
|
||||
if [ "$DISK_PCT" -gt 90 ]; then
|
||||
alert "WARN" "Disk at ${DISK_PCT}% — pruning"
|
||||
remediate_prune
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$CHECK_INTERVAL"
|
||||
done
|
||||
Reference in New Issue
Block a user