pi-agent
4fcfa4375d
Deploy Staging / build (push) Failing after 13m37s
- Removed set -e — agent no longer crashes when a remediation step fails - SIGKILL/OOM → restart first, only rollback if exceeding max restarts/hour - Added HEALTHY_CYCLES counter: after 10 consecutive healthy cycles (5 min), reset restart tracking state (system considered stable) - Rollback now handles the case where Docker restart policy already revived the container (rm -f before re-creating)
365 lines
12 KiB
Bash
365 lines
12 KiB
Bash
#!/bin/bash
|
|
# ============================================================
|
|
# Falah Mobile — Auto-Healing Agent (Dedicated Container)
|
|
# ============================================================
|
|
# Real-time fault detection + autonomous remediation.
|
|
# Runs as a sidecar container alongside falah-mobile-staging.
|
|
#
|
|
# Detects:
|
|
# Container crash, health 5xx, DB disconnect, OOM,
|
|
# Gateway 502, 5xx spikes, disk pressure
|
|
#
|
|
# Fixes:
|
|
# Restart, rollback, nginx reload, prune, escalate
|
|
# ============================================================
|
|
|
|
# NO set -e — we handle errors explicitly to avoid the entire agent crashing
|
|
# on a single failed remediation step. Each function manages its own errors.
|
|
|
|
CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
|
|
HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
|
|
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
|
|
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
|
|
MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
|
|
LOG="/var/log/falah/agent.log"
|
|
STATE_DIR="/var/state/falah"
|
|
STATE_FILE="$STATE_DIR/restart-count"
|
|
ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"
|
|
|
|
mkdir -p "$STATE_DIR" "$(dirname "$LOG")"
|
|
|
|
# ── Logging ──────────────────────────────────────────────────
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$LOG"
|
|
}
|
|
|
|
alert() {
|
|
local severity="$1" msg="$2"
|
|
log "[$severity] $msg"
|
|
# Future: emit to webhook, ntfy, Slack, etc.
|
|
# curl -sf -X POST -H "Content-Type: application/json" \
|
|
# -d "{\"severity\":\"$severity\",\"message\":\"$msg\",\"service\":\"$CONTAINER_NAME\"}" \
|
|
# "$ALERT_WEBHOOK" &
|
|
}
|
|
|
|
# ── Track restarts (sliding window per hour) ─────────────────
|
|
|
|
count_restarts() {
|
|
[ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
|
|
local now window_start count
|
|
now=$(date +%s)
|
|
window_start=$((now - 3600))
|
|
|
|
# Filter entries within the last hour
|
|
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
|
|
}
|
|
|
|
record_restart() {
|
|
date +%s >> "$STATE_FILE"
|
|
# Trim old entries
|
|
local now window_start
|
|
now=$(date +%s)
|
|
window_start=$((now - 3600))
|
|
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
|
}
|
|
|
|
# ── Fault Detection ──────────────────────────────────────────
|
|
|
|
check_http() {
|
|
local url="$1" label="$2"
|
|
local code body
|
|
body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
|
|
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
|
|
|
|
if [ "$code" = "200" ]; then
|
|
echo "$body"
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
check_db() {
|
|
local body
|
|
body=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null) || return 1
|
|
echo "$body" | grep -q '"db":true' && return 0 || return 1
|
|
}
|
|
|
|
check_docker_events() {
|
|
# Check if the container recently crashed
|
|
local since_seconds="${1:-120}"
|
|
local since_epoch
|
|
since_epoch=$(date -d "-${since_seconds} seconds" +%s 2>/dev/null || echo $(( $(date +%s) - since_seconds )))
|
|
|
|
# Use docker events --since to check for recent die/oom events
|
|
docker events --since "${since_epoch}s" --filter "container=$CONTAINER_NAME" \
|
|
--filter "event=die" --format '{{.Status}}' 2>/dev/null | head -1
|
|
}
|
|
|
|
check_disk() {
|
|
df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
|
|
}
|
|
|
|
# ── Fault Diagnosis ──────────────────────────────────────────
|
|
|
|
diagnose() {
|
|
local fault_reason=""
|
|
local health_body
|
|
|
|
# 1. Check if container exists
|
|
if ! docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
|
|
echo "container_missing"
|
|
return
|
|
fi
|
|
|
|
# 2. Check if container is running
|
|
local status
|
|
status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "missing")
|
|
if [ "$status" != "running" ]; then
|
|
local exit_code
|
|
exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
|
|
local oom
|
|
oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
|
|
|
|
if [ "$oom" = "true" ]; then
|
|
echo "oom_killed"
|
|
elif [ "$exit_code" = "137" ]; then
|
|
echo "sigkill_oom"
|
|
elif [ "$exit_code" != "0" ]; then
|
|
echo "crashed_exit_${exit_code}"
|
|
else
|
|
echo "not_running"
|
|
fi
|
|
return
|
|
fi
|
|
|
|
# 3. Check health endpoint
|
|
health_body=$(check_http "$HEALTH_URL" "direct" 2>/dev/null) || true
|
|
if [ -z "$health_body" ]; then
|
|
echo "health_unreachable"
|
|
return
|
|
fi
|
|
|
|
# 4. Check DB connectivity
|
|
local db_status
|
|
db_status=$(echo "$health_body" | grep -o '"db":false\|"db":true' || echo "unknown")
|
|
if [ "$db_status" = '"db":false' ]; then
|
|
echo "db_disconnected"
|
|
return
|
|
fi
|
|
|
|
# 5. Check response content
|
|
if ! echo "$health_body" | grep -q '"status":"ok"'; then
|
|
echo "health_invalid"
|
|
return
|
|
fi
|
|
|
|
# 6. Check gateway
|
|
if ! check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
|
|
echo "gateway_down"
|
|
return
|
|
fi
|
|
|
|
echo "healthy"
|
|
}
|
|
|
|
# ── Remediations ─────────────────────────────────────────────
|
|
|
|
remediate_container_restart() {
|
|
local diagnosis="$1"
|
|
local restarts
|
|
restarts=$(count_restarts)
|
|
|
|
log "🔄 Remediation: restart container (diagnosis=$diagnosis, restarts_last_hour=$restarts)"
|
|
|
|
if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
|
|
alert "CRITICAL" "Excessive restarts ($restarts/h) — escalating to rollback"
|
|
remediate_rollback
|
|
return
|
|
fi
|
|
|
|
docker restart "$CONTAINER_NAME" >> "$LOG" 2>&1
|
|
record_restart
|
|
sleep 10
|
|
|
|
# Verify
|
|
if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
|
|
log "✅ Restart successful — container healthy"
|
|
alert "INFO" "Auto-healed: restart $CONTAINER_NAME (diagnosis=$diagnosis)"
|
|
return 0
|
|
else
|
|
log "⚠️ Restart didn't fix — escalating"
|
|
remediate_escalate
|
|
fi
|
|
}
|
|
|
|
remediate_rollback() {
|
|
log "🔄 Remediation: rollback to previous image"
|
|
|
|
if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
|
|
log "🔄 Rolling back to $ROLLBACK_IMAGE..."
|
|
|
|
# Stop and remove old container if it exists (may have been auto-restarted by Docker)
|
|
if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
|
|
docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
|
|
sleep 2
|
|
docker rm -f "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
|
|
sleep 2
|
|
fi
|
|
|
|
# Extract original run config from inspect (if available) or use defaults
|
|
local ports volumes network env_vars
|
|
ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000")
|
|
volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null)
|
|
network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge")
|
|
|
|
# Run rollback image with same config
|
|
docker run -d \
|
|
--name "$CONTAINER_NAME" \
|
|
--restart unless-stopped \
|
|
-p 4014:3000 \
|
|
$volumes \
|
|
--network "$network" \
|
|
"$ROLLBACK_IMAGE" >> "$LOG" 2>&1
|
|
|
|
sleep 15
|
|
|
|
if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
|
|
log "✅ Rollback successful — previous image restored"
|
|
alert "INFO" "Auto-healed: rolled back to $ROLLBACK_IMAGE"
|
|
return 0
|
|
else
|
|
alert "CRITICAL" "Rollback also failed — manual intervention required"
|
|
return 1
|
|
fi
|
|
else
|
|
log "⚠️ No rollback image found — cannot rollback"
|
|
alert "WARN" "No rollback image available"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
remediate_gateway() {
|
|
log "🔄 Remediation: reload nginx gateway"
|
|
|
|
local gw_container
|
|
gw_container=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
|
|
|
|
if [ -n "$gw_container" ]; then
|
|
docker exec "$gw_container" nginx -s reload >> "$LOG" 2>&1
|
|
sleep 3
|
|
if check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
|
|
log "✅ Gateway reload successful"
|
|
alert "INFO" "Auto-healed: reloaded nginx in $gw_container"
|
|
return 0
|
|
fi
|
|
fi
|
|
log "⚠️ Could not reload gateway"
|
|
return 1
|
|
}
|
|
|
|
remediate_prune() {
|
|
log "🔄 Remediation: docker system prune (disk pressure)"
|
|
docker system prune -f >> "$LOG" 2>&1
|
|
log "✅ Prune completed"
|
|
alert "INFO" "Auto-healed: pruned Docker resources"
|
|
}
|
|
|
|
remediate_escalate() {
|
|
alert "CRITICAL" "Auto-heal unable to fix — escalation needed"
|
|
# Future: send to PagerDuty, OpsGenie, Slack, etc.
|
|
log "🚨 ESCALATION: Manual intervention required for $CONTAINER_NAME"
|
|
}
|
|
|
|
# ── Main Loop ────────────────────────────────────────────────
|
|
|
|
log "══════════════════════════════════════════════════════"
|
|
log "🚀 Auto-Healing Agent starting"
|
|
log " Container: $CONTAINER_NAME"
|
|
log " Health URL: $HEALTH_URL"
|
|
log " Gateway URL: $GATEWAY_URL"
|
|
log " Interval: ${CHECK_INTERVAL}s"
|
|
log " Max restarts: $MAX_RESTARTS/hour"
|
|
log "══════════════════════════════════════════════════════"
|
|
|
|
while true; do
|
|
# ── Phase 1: Detect ──
|
|
DIAGNOSIS=$(diagnose)
|
|
|
|
# ── Phase 2: Act ──
|
|
case "$DIAGNOSIS" in
|
|
healthy)
|
|
# All good — reset restart count if we've had consecutive healthy cycles
|
|
if [ -z "$HEALTHY_CYCLES" ]; then HEALTHY_CYCLES=0; fi
|
|
HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1))
|
|
# After 10 healthy cycles (5 min), reset restart counter
|
|
if [ "$HEALTHY_CYCLES" -ge 10 ]; then
|
|
HEALTHY_CYCLES=0
|
|
# Reset state file — system is stable now
|
|
echo 0 > "$STATE_FILE" 2>/dev/null || true
|
|
fi
|
|
;;
|
|
|
|
container_missing)
|
|
alert "CRITICAL" "Container $CONTAINER_NAME does not exist"
|
|
;;
|
|
|
|
oom_killed|sigkill_oom)
|
|
alert "WARN" "Container killed ($DIAGNOSIS) — restarting"
|
|
# If we've had too many restarts, escalate to rollback
|
|
local restarts
|
|
restarts=$(count_restarts)
|
|
if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
|
|
alert "CRITICAL" "OOM repeated ($restarts restarts/hour) — rolling back"
|
|
remediate_rollback
|
|
else
|
|
remediate_container_restart "$DIAGNOSIS"
|
|
fi
|
|
;;
|
|
|
|
crashed_*|not_running)
|
|
alert "WARN" "Container crashed ($DIAGNOSIS)"
|
|
remediate_container_restart "$DIAGNOSIS"
|
|
;;
|
|
|
|
health_unreachable)
|
|
alert "WARN" "Health endpoint unreachable"
|
|
remediate_container_restart "$DIAGNOSIS"
|
|
;;
|
|
|
|
db_disconnected)
|
|
alert "WARN" "Database disconnected"
|
|
remediate_container_restart "$DIAGNOSIS"
|
|
;;
|
|
|
|
health_invalid)
|
|
alert "WARN" "Health response invalid"
|
|
remediate_container_restart "$DIAGNOSIS"
|
|
;;
|
|
|
|
gateway_down)
|
|
alert "WARN" "Gateway down"
|
|
remediate_gateway
|
|
;;
|
|
|
|
*)
|
|
alert "WARN" "Unknown diagnosis: $DIAGNOSIS"
|
|
remediate_container_restart "$DIAGNOSIS"
|
|
;;
|
|
esac
|
|
|
|
# ── Periodic maintenance ──
|
|
# Check disk every 10 cycles (5 min at 30s interval)
|
|
CYCLE_COUNTER=$((CYCLE_COUNTER + 1))
|
|
if [ "$CYCLE_COUNTER" -ge 10 ]; then
|
|
CYCLE_COUNTER=0
|
|
DISK_PCT=$(check_disk)
|
|
if [ "$DISK_PCT" -gt 90 ]; then
|
|
alert "WARN" "Disk at ${DISK_PCT}% — pruning"
|
|
remediate_prune
|
|
fi
|
|
fi
|
|
|
|
sleep "$CHECK_INTERVAL"
|
|
done
|