pi-agent
a0068d238b
- FIXED core bug: log() used tee -a, writing to both log file AND stdout, corrupting DIAGNOSIS=$(detect_fault) and causing infinite fault loops - ADDED 120s cooldown after restore to prevent detection loops during startup - ADDED swarm service detection to auto-healer (handles both standalone containers and Docker Swarm services) - FIXED swarm service detection: awk now uses $2 (service name) not $1 (ID) - FIXED exact match for service names (grep -x) - FIXED restore protocol to use docker service update --force for swarm services - DEPLOYED auto-healers to both Synology (v15) and Contabo (v15) - DNS healer also deployed to Contabo (v3) - DNS healer has swarm health monitoring (checks every 2 cycles) - All 3 nodes in Docker Swarm: Synology (manager), colima (worker), Contabo (leader)
599 lines
23 KiB
Bash
599 lines
23 KiB
Bash
#!/bin/bash
|
||
# ============================================================
|
||
# Falah Mobile — SRE Auto-Healing Agent
|
||
# ============================================================
|
||
# Log-aware fault diagnosis + restore protocol.
|
||
#
|
||
# On fault:
|
||
# 1. FETCH logs from failing container
|
||
# 2. ANALYZE logs against error knowledge base
|
||
# 3. DECIDE: quick hotfix or restore protocol?
|
||
# 4. RESTORE PROTOCOL: rollback to known-good image
|
||
# 5. While rolling back, capture full diagnostics for later fix
|
||
#
|
||
# This is NOT a blind restarter — it's a diagnostic-first SRE agent.
|
||
# ============================================================
|
||
|
||
# ── Configuration ────────────────────────────────────────────
|
||
|
||
CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
|
||
HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
|
||
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
|
||
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
|
||
MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
|
||
ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"
|
||
|
||
AGENT_LOG="/var/log/falah/agent.log"
|
||
STATE_DIR="/var/state/falah"
|
||
STATE_FILE="$STATE_DIR/restart-count"
|
||
INCIDENTS_DIR="/var/log/falah/incidents"
|
||
|
||
mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")" "$INCIDENTS_DIR"
|
||
|
||
# ── Logging ──────────────────────────────────────────────────
|
||
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" >> "$AGENT_LOG"
|
||
}
|
||
|
||
alert() {
|
||
local severity="$1" msg="$2"
|
||
log "[$severity] $msg"
|
||
# Future: webhook, ntfy, Slack, PagerDuty
|
||
}
|
||
|
||
# ── Restart Tracking (sliding window) ────────────────────────
|
||
|
||
count_restarts() {
|
||
[ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
|
||
local now window_start
|
||
now=$(date +%s)
|
||
window_start=$((now - 3600))
|
||
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
|
||
}
|
||
|
||
record_restart() {
|
||
date +%s >> "$STATE_FILE"
|
||
local now window_start
|
||
now=$(date +%s)
|
||
window_start=$((now - 3600))
|
||
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
||
}
|
||
|
||
# ── Basic Health Checks ──────────────────────────────────────
|
||
|
||
check_http() {
|
||
local url="$1"
|
||
local body code
|
||
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
|
||
body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
|
||
[ "$code" = "200" ] && { echo "$body"; return 0; } || return 1
|
||
}
|
||
|
||
check_disk() {
|
||
df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
|
||
}
|
||
|
||
# ── Phase 1: Detect Fault ────────────────────────────────────
|
||
|
||
detect_fault() {
|
||
# Cooldown: skip detection if restore happened within 60s
|
||
local cooldown=120
|
||
if [ -f /var/state/falah/last_restore_epoch ]; then
|
||
local last_restore
|
||
last_restore=$(cat /var/state/falah/last_restore_epoch 2>/dev/null || echo 0)
|
||
local now
|
||
now=$(date +%s)
|
||
local elapsed=$((now - last_restore))
|
||
if [ $elapsed -lt $cooldown ]; then
|
||
log " (cooldown active — last restore ${elapsed}s ago, skipping detection)"
|
||
echo "healthy"
|
||
return
|
||
fi
|
||
fi
|
||
|
||
# Returns: diagnosis string (or "healthy")
|
||
|
||
# 0a. Check if CONTAINER_NAME is a Swarm service
|
||
local is_swarm_service=false
|
||
if docker service ls 2>/dev/null | awk '{print $2}' | grep -x "${CONTAINER_NAME}"; then
|
||
is_swarm_service=true
|
||
fi
|
||
|
||
if [ "$is_swarm_service" = true ]; then
|
||
# Swarm service mode — check via docker service ps
|
||
local unhealthy
|
||
unhealthy=$(docker service ps "$CONTAINER_NAME" --filter 'desired-state=running' --format '{{.CurrentState}}' 2>/dev/null | grep -v "^Running" | head -1)
|
||
if [ -n "$unhealthy" ]; then
|
||
echo "swarm_service_unhealthy"
|
||
return
|
||
fi
|
||
# Service is running, now check HTTP health
|
||
local health_body
|
||
health_body=$(check_http "$HEALTH_URL") || { echo "health_down"; return; }
|
||
echo "$health_body" | grep -q '"db":false' && { echo "db_down"; return; }
|
||
echo "$health_body" | grep -q '"status":"ok"' || { echo "health_invalid"; return; }
|
||
echo "healthy"
|
||
return
|
||
fi
|
||
|
||
# 1. Container exists? (standalone mode)
|
||
if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "${CONTAINER_NAME}"; then
|
||
echo "container_missing"
|
||
return
|
||
fi
|
||
|
||
# 2. Container running?
|
||
local status exit_code oom
|
||
status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "unknown")
|
||
|
||
if [ "$status" != "running" ]; then
|
||
exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
|
||
oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
|
||
[ "$oom" = "true" ] && { echo "oom_killed"; return; }
|
||
[ "$exit_code" = "137" ] && { echo "sigkill"; return; }
|
||
echo "crashed_exit_$exit_code"
|
||
return
|
||
fi
|
||
|
||
# 3. Health endpoint responds?
|
||
local health_body
|
||
health_body=$(check_http "$HEALTH_URL") || { echo "health_down"; return; }
|
||
|
||
# 4. DB connected?
|
||
echo "$health_body" | grep -q '"db":false' && { echo "db_down"; return; }
|
||
|
||
# 5. Status ok?
|
||
echo "$health_body" | grep -q '"status":"ok"' || { echo "health_invalid"; return; }
|
||
|
||
# 6. Gateway?
|
||
check_http "$GATEWAY_URL" > /dev/null || { echo "gateway_down"; return; }
|
||
|
||
echo "healthy"
|
||
}
|
||
|
||
# ── Phase 2: Fetch Logs ──────────────────────────────────────
|
||
|
||
fetch_logs() {
|
||
local lines="${1:-100}"
|
||
docker logs "$CONTAINER_NAME" --tail "$lines" 2>&1 || echo "LOG_FETCH_FAILED"
|
||
}
|
||
|
||
# ── Phase 3: Log Analysis (Knowledge Base) ───────────────────
|
||
|
||
analyze_logs() {
|
||
local diagnosis="$1"
|
||
local logs="$2"
|
||
|
||
# Build a diagnostic report
|
||
local root_cause="unknown"
|
||
local certainty="low"
|
||
local suggested_action="rollback"
|
||
local details=""
|
||
|
||
# ── Error Pattern Knowledge Base ───────────────────────────
|
||
# Each pattern maps to: root_cause | certainty | suggested_action
|
||
|
||
# Node.js crashes
|
||
if echo "$logs" | grep -qi "Cannot find module"; then
|
||
root_cause="missing_dependency"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "Cannot find module" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "Module not found"; then
|
||
root_cause="missing_dependency"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "Module not found" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "unexpected token"; then
|
||
root_cause="js_parse_error"
|
||
certainty="high"
|
||
suggested_action="rollback"
|
||
details=$(echo "$logs" | grep -i "unexpected token" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "SyntaxError"; then
|
||
root_cause="js_syntax_error"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "SyntaxError" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "ReferenceError"; then
|
||
root_cause="js_reference_error"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "ReferenceError" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "TypeError"; then
|
||
root_cause="js_type_error"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "TypeError" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "heap out of memory\|FATAL ERROR\|Allocation failed"; then
|
||
root_cause="out_of_memory"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "heap out of memory\|FATAL ERROR\|Allocation failed" | head -3 | tr '\n' '; ')
|
||
|
||
# Prisma / DB errors
|
||
elif echo "$logs" | grep -qi "PrismaClientInitializationError\|prisma.*connect.*ECONNREFUSED"; then
|
||
root_cause="db_connection_refused"
|
||
certainty="high"
|
||
suggested_action="restart_container"
|
||
details=$(echo "$logs" | grep -i "PrismaClientInitializationError\|ECONNREFUSED" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "prisma.*ENOENT\|prisma.*does not exist"; then
|
||
root_cause="prisma_schema_mismatch"
|
||
certainty="high"
|
||
details=$(echo "$logs" | grep -i "prisma.*ENOENT\|does not exist" | head -3 | tr '\n' '; ')
|
||
|
||
elif echo "$logs" | grep -qi "Can't reach database server\|getaddrinfo ENOTFOUND.*db\|postgres.*connect"; then
|
||
root_cause="db_unreachable"
|
||
certainty="high"
|
||
suggested_action="restart_container"
|
||
details=$(echo "$logs" | grep -i "ENOTFOUND\|Can't reach database" | head -3 | tr '\n' '; ')
|
||
|
||
# Network / connectivity
|
||
elif echo "$logs" | grep -qi "ECONNREFUSED\|ECONNRESET\|ETIMEDOUT\|ENOTFOUND"; then
|
||
root_cause="downstream_unreachable"
|
||
certainty="medium"
|
||
suggested_action="restart_container"
|
||
details=$(echo "$logs" | grep -i "ECONNREFUSED\|ECONNRESET\|ETIMEDOUT\|ENOTFOUND" | head -3 | tr '\n' '; ')
|
||
|
||
# Rate limiting
|
||
elif echo "$logs" | grep -qi "rate limit\|too many requests\|429"; then
|
||
root_cause="rate_limit_exceeded"
|
||
certainty="medium"
|
||
suggested_action="hotfix_rate_limit"
|
||
details=$(echo "$logs" | grep -i "rate limit\|too many" | head -3 | tr '\n' '; ')
|
||
|
||
# Auth / config
|
||
elif echo "$logs" | grep -qi "jwt expired\|invalid token\|Unauthorized"; then
|
||
root_cause="auth_config_error"
|
||
certainty="medium"
|
||
suggested_action="check_config"
|
||
details=$(echo "$logs" | grep -i "jwt expired\|invalid token\|Unauthorized" | head -3 | tr '\n' '; ')
|
||
|
||
# 5xx patterns from health endpoint
|
||
elif echo "$logs" | grep -qi "500\|Internal Server Error\|unhandled rejection\|uncaught exception"; then
|
||
root_cause="unhandled_exception"
|
||
certainty="medium"
|
||
details=$(echo "$logs" | grep -i "unhandled\|uncaught\|500\|Internal Server Error" | head -3 | tr '\n' '; ')
|
||
|
||
# Next.js specific
|
||
elif echo "$logs" | grep -qi "next.*error\|Error:.*page\|render.*error\|application.*error"; then
|
||
root_cause="application_error"
|
||
certainty="medium"
|
||
details=$(echo "$logs" | grep -i "Error:\|render.*error" | head -3 | tr '\n' '; ')
|
||
fi
|
||
|
||
# If diagnosis gives strong signal, incorporate it
|
||
case "$diagnosis" in
|
||
oom_killed|sigkill)
|
||
[ "$root_cause" = "unknown" ] && root_cause="container_killed_sigkill"
|
||
suggested_action="rollback"
|
||
;;
|
||
db_down)
|
||
[ "$root_cause" = "unknown" ] && root_cause="database_disconnected"
|
||
suggested_action="restart_container"
|
||
;;
|
||
health_down)
|
||
[ "$root_cause" = "unknown" ] && root_cause="container_unresponsive"
|
||
;;
|
||
gateway_down)
|
||
root_cause="nginx_gateway_down"
|
||
suggested_action="reload_gateway"
|
||
certainty="high"
|
||
;;
|
||
esac
|
||
|
||
# Return structured report as JSON-like lines
|
||
echo "ROOT_CAUSE=$root_cause"
|
||
echo "CERTAINTY=$certainty"
|
||
echo "ACTION=$suggested_action"
|
||
echo "DETAILS=$details"
|
||
}
|
||
|
||
# ── Phase 4: Restore Protocol ────────────────────────────────
|
||
# Fast rollback to last known-good state, while capturing
|
||
# full diagnostics for permanent fix later.
|
||
|
||
restore_protocol() {
|
||
local diagnosis="$1"
|
||
local root_cause="$2"
|
||
local action="$3"
|
||
local details="$4"
|
||
local logs="$5"
|
||
local incident_id
|
||
incident_id="incident_$(date +%Y%m%d_%H%M%S)"
|
||
|
||
log "🚨 RESTORE PROTOCOL triggered — incident: $incident_id"
|
||
log " Diagnosis: $diagnosis"
|
||
log " Root cause: $root_cause"
|
||
log " Action: $action"
|
||
log " Details: $details"
|
||
|
||
# ── Step 0: Save full diagnostic context for later fix ─────
|
||
{
|
||
echo "=== INCIDENT REPORT: $incident_id ==="
|
||
echo "Timestamp: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||
echo "Container: $CONTAINER_NAME"
|
||
echo "Diagnosis: $diagnosis"
|
||
echo "Root cause: $root_cause"
|
||
echo "Certainty: $certainty"
|
||
echo "Action: $action"
|
||
echo "Details: $details"
|
||
echo ""
|
||
echo "=== Container Logs ==="
|
||
echo "$logs"
|
||
echo ""
|
||
echo "=== Container Inspect ==="
|
||
docker inspect "$CONTAINER_NAME" 2>/dev/null || docker service inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED"
|
||
echo ""
|
||
echo "=== Docker Events (last 5 min) ==="
|
||
timeout 5 docker events --since "${CHECK_INTERVAL}s" --until "$(date +%s)" \
|
||
--filter "container=$CONTAINER_NAME" \
|
||
--filter "service=$CONTAINER_NAME" \
|
||
--format '{{.Time}} {{.Type}} {{.Action}} {{.Status}}' 2>/dev/null || echo "EVENTS_FAILED"
|
||
echo ""
|
||
echo "=== System Resources ==="
|
||
df -h / | tail -1
|
||
free -m 2>/dev/null || echo "free_unavailable"
|
||
echo "=== END INCIDENT REPORT ==="
|
||
} > "$INCIDENTS_DIR/$incident_id.log"
|
||
|
||
log "📝 Incident saved to $INCIDENTS_DIR/$incident_id.log"
|
||
alert "INFO" "Incident $incident_id logged — root cause: $root_cause"
|
||
|
||
# Always save cooldown timestamp to prevent detection loops
|
||
date +%s > /var/state/falah/last_restore_epoch
|
||
|
||
# ── Step 1: Check if this is a Swarm service (don't manage containers directly) ──
|
||
if docker service ls 2>/dev/null | awk '{print $2}' | grep -x "${CONTAINER_NAME}"; then
|
||
log "ℹ️ Swarm service detected — skipping container management, Swarm handles recovery"
|
||
date +%s > /var/state/falah/last_restore_epoch
|
||
log "🔄 RESTORE: forcing service update to trigger redeploy..."
|
||
docker service update --force "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
alert "INFO" "Swarm service $CONTAINER_NAME force-updated (incident: $incident_id)"
|
||
return 0
|
||
fi
|
||
|
||
# ── Step 2: Apply action ───────────────────────────────────
|
||
case "$action" in
|
||
rollback)
|
||
log "🔄 RESTORE: rolling back to $ROLLBACK_IMAGE..."
|
||
if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
|
||
# Remove old container
|
||
docker stop "$CONTAINER_NAME" 2>/dev/null || true
|
||
sleep 2
|
||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||
sleep 2
|
||
|
||
# Recreate from rollback image (omit --network; default bridge works reliably)
|
||
docker run -d \
|
||
--name "$CONTAINER_NAME" \
|
||
--restart unless-stopped \
|
||
-p 4014:3000 \
|
||
-v /var/services/homes/admin/falah-mobile/data:/app/data \
|
||
"$ROLLBACK_IMAGE" >> "$AGENT_LOG" 2>&1
|
||
|
||
sleep 15
|
||
if check_http "$HEALTH_URL" > /dev/null 2>&1; then
|
||
date +%s > /var/state/falah/last_restore_epoch
|
||
log "✅ RESTORE: rollback successful — service restored"
|
||
alert "INFO" "Restore protocol: rolled back to $ROLLBACK_IMAGE (incident: $incident_id)"
|
||
record_restart
|
||
return 0
|
||
else
|
||
log "❌ RESTORE: rollback also failed — escalation needed"
|
||
alert "CRITICAL" "Rollback failed for incident $incident_id — manual intervention"
|
||
return 1
|
||
fi
|
||
else
|
||
log "⚠️ RESTORE: no rollback image found — attempting restart instead"
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
sleep 10
|
||
if check_http "$HEALTH_URL" > /dev/null 2>&1; then
|
||
date +%s > /var/state/falah/last_restore_epoch
|
||
log "✅ RESTORE: restart successful (fallback)"
|
||
record_restart
|
||
return 0
|
||
fi
|
||
return 1
|
||
fi
|
||
;;
|
||
|
||
restart_container)
|
||
log "🔄 RESTORE: restarting container..."
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
record_restart
|
||
sleep 10
|
||
if check_http "$HEALTH_URL" > /dev/null 2>&1; then
|
||
log "✅ RESTORE: restart successful"
|
||
return 0
|
||
else
|
||
log "⚠️ RESTORE: restart didn't fix — escalating to rollback"
|
||
restore_protocol "$diagnosis" "restart_failed" "rollback" "$details" "$logs"
|
||
fi
|
||
;;
|
||
|
||
reload_gateway)
|
||
log "🔄 RESTORE: reloading nginx gateway..."
|
||
local gw
|
||
gw=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
|
||
[ -n "$gw" ] && docker exec "$gw" nginx -s reload >> "$AGENT_LOG" 2>&1
|
||
sleep 3
|
||
check_http "$GATEWAY_URL" > /dev/null && log "✅ RESTORE: gateway restored" || log "⚠️ Gateway still down"
|
||
;;
|
||
|
||
hotfix_rate_limit)
|
||
log "🔄 RESTORE: rate limit issue — applying hotfix"
|
||
# Rate limiter auto-clears stale entries every 5 min (code-level fix)
|
||
# For immediate relief, restart container
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
record_restart
|
||
sleep 10
|
||
;;
|
||
|
||
check_config)
|
||
log "🔄 RESTORE: config issue detected — rolling back to safe state"
|
||
restore_protocol "$diagnosis" "config_error" "rollback" "$details" "$logs"
|
||
;;
|
||
|
||
*)
|
||
log "🔄 RESTORE: unknown action '$action' — attempting restart as safe default"
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
record_restart
|
||
sleep 10
|
||
;;
|
||
esac
|
||
}
|
||
|
||
# ── Phase 5: Apply Quick Hotfix (when possible) ──────────────
|
||
|
||
apply_hotfix() {
|
||
local root_cause="$1"
|
||
local logs="$2"
|
||
|
||
case "$root_cause" in
|
||
rate_limit_exceeded)
|
||
log "🔧 HOTFIX: rate limit exceeded — restarting to clear state"
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
sleep 5
|
||
log "🔧 HOTFIX: rate limiter now auto-cleans every 5 min (code fix in place)"
|
||
;;
|
||
|
||
db_connection_refused|db_unreachable)
|
||
log "🔧 HOTFIX: DB connection issue — restarting container"
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
sleep 10
|
||
;;
|
||
|
||
*)
|
||
log "🔧 HOTFIX: no quick fix for '$root_cause' — relying on restore protocol"
|
||
return 1
|
||
;;
|
||
esac
|
||
return 0
|
||
}
|
||
|
||
# ── Periodic Maintenance ────────────────────────────────────
|
||
|
||
periodic_maintenance() {
|
||
local disk_pct
|
||
disk_pct=$(check_disk)
|
||
if [ "$disk_pct" -gt 90 ]; then
|
||
log "⚠️ Disk at ${disk_pct}% — running prune"
|
||
docker system prune -f >> "$AGENT_LOG" 2>&1
|
||
log "✅ Prune completed"
|
||
fi
|
||
}
|
||
|
||
# ── Main Loop ────────────────────────────────────────────────
|
||
|
||
log "══════════════════════════════════════════════════════"
|
||
log "🚀 SRE Auto-Healing Agent starting"
|
||
log " Container: $CONTAINER_NAME"
|
||
log " Health URL: $HEALTH_URL"
|
||
log " Gateway URL: $GATEWAY_URL"
|
||
log " Interval: ${CHECK_INTERVAL}s"
|
||
log " Max restarts: $MAX_RESTARTS/hour"
|
||
log " Incidents: $INCIDENTS_DIR"
|
||
log "══════════════════════════════════════════════════════"
|
||
|
||
CYCLE=0
|
||
HEALTHY_CYCLES=0
|
||
|
||
while true; do
|
||
# ── Detect ──────────────────────────────────────────────
|
||
DIAGNOSIS=$(detect_fault)
|
||
|
||
if [ "$DIAGNOSIS" = "healthy" ]; then
|
||
# Stability counter — reset restart tracking after 5 min healthy
|
||
HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1))
|
||
if [ "$HEALTHY_CYCLES" -ge 10 ]; then
|
||
HEALTHY_CYCLES=0
|
||
echo 0 > "$STATE_FILE" 2>/dev/null || true
|
||
fi
|
||
else
|
||
# ── FAULT DETECTED ────────────────────────────────────
|
||
HEALTHY_CYCLES=0
|
||
log ""
|
||
log "⚠️ FAULT DETECTED: $DIAGNOSIS"
|
||
|
||
# Step 1: Fetch logs
|
||
log "📋 Fetching container logs..."
|
||
CONTAINER_LOGS=$(fetch_logs 100)
|
||
|
||
# Step 2: Analyze logs against knowledge base
|
||
log "🔍 Analyzing logs for root cause..."
|
||
ANALYSIS=$(analyze_logs "$DIAGNOSIS" "$CONTAINER_LOGS")
|
||
|
||
# Parse analysis output
|
||
ROOT_CAUSE=$(echo "$ANALYSIS" | grep "^ROOT_CAUSE=" | cut -d= -f2-)
|
||
CERTAINTY=$(echo "$ANALYSIS" | grep "^CERTAINTY=" | cut -d= -f2-)
|
||
ACTION=$(echo "$ANALYSIS" | grep "^ACTION=" | cut -d= -f2-)
|
||
DETAILS=$(echo "$ANALYSIS" | grep "^DETAILS=" | cut -d= -f2-)
|
||
|
||
log " Root cause: $ROOT_CAUSE (certainty: $CERTAINTY)"
|
||
log " Action: $ACTION"
|
||
[ -n "$DETAILS" ] && log " Details: $DETAILS"
|
||
|
||
# Step 3: Decision — can we hotfix or need restore protocol?
|
||
restarts=$(count_restarts)
|
||
|
||
if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
|
||
# Too many restarts — go straight to restore protocol
|
||
log "⚠️ $restarts restarts in last hour — exceeding limit of $MAX_RESTARTS"
|
||
log "🚨 SKIPPING hotfix attempt — going directly to restore protocol"
|
||
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
|
||
|
||
elif [ "$ACTION" = "rollback" ]; then
|
||
# Knowledge base says rollback — run restore protocol
|
||
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "$ACTION" "$DETAILS" "$CONTAINER_LOGS"
|
||
|
||
elif [ "$ACTION" = "restart_container" ]; then
|
||
# Try restart first, escalate if fails
|
||
log "🔄 Attempting restart (restart #$restarts)..."
|
||
apply_hotfix "$ROOT_CAUSE" "$CONTAINER_LOGS"
|
||
sleep 10
|
||
if [ "$(detect_fault)" != "healthy" ]; then
|
||
log "⚠️ Restart didn't resolve — escalating to restore protocol"
|
||
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
|
||
fi
|
||
|
||
elif [ "$ACTION" = "reload_gateway" ]; then
|
||
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "$ACTION" "$DETAILS" "$CONTAINER_LOGS"
|
||
|
||
elif [ "$ACTION" = "hotfix_rate_limit" ]; then
|
||
apply_hotfix "$ROOT_CAUSE" "$CONTAINER_LOGS"
|
||
|
||
else
|
||
# No specific action — safe default: restart, then rollback
|
||
log "🔄 No specific fix for '$ROOT_CAUSE' — attempting restart"
|
||
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
||
record_restart
|
||
sleep 10
|
||
[ "$(detect_fault)" != "healthy" ] && restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
|
||
fi
|
||
fi
|
||
|
||
# ── Periodic maintenance (every 10 cycles = 5 min) ─────
|
||
CYCLE=$((CYCLE + 1))
|
||
[ "$CYCLE" -ge 10 ] && { CYCLE=0; periodic_maintenance; }
|
||
|
||
# ── Cross-check: DNS healer alive? ────────────────────
|
||
if ! docker ps --filter 'name=falah-dns-healer' --format '{{.Status}}' 2>/dev/null | grep -q '^Up'; then
|
||
log "⚠️ Sibling DNS healer DOWN — restarting..."
|
||
docker restart falah-dns-healer 2>/dev/null || \
|
||
docker run -d --name falah-dns-healer --restart unless-stopped \
|
||
--network host \
|
||
-v /var/run/docker.sock:/var/run/docker.sock:ro \
|
||
-v /var/log/falah:/var/log/falah \
|
||
-e CHECK_INTERVAL=60 \
|
||
falah-dns-healer:latest >> "$AGENT_LOG" 2>&1
|
||
sleep 3
|
||
if docker ps --filter 'name=falah-dns-healer' --format '{{.Status}}' | grep -q '^Up'; then
|
||
alert "INFO" "Cross-heal: restarted DNS healer"
|
||
fi
|
||
fi
|
||
|
||
sleep "$CHECK_INTERVAL"
|
||
done
|