pi-agent
6d946571b0
Deploy Staging / build (push) Failing after 1h1m21s
- Fixed blocking command in incident reporting (added timeout 5s + --until now) Was causing the entire agent script to hang forever after saving each incident - Full SRE flow now works end-to-end: 1. Detect fault → 2. Fetch logs → 3. Analyze against knowledge base 4. Save incident with full context (logs, inspect, events, resources) 5. Execute restore protocol (rollback to :rollback image) 6. Verify service restored - 3 incidents successfully captured and logged with diagnostics - All containers healthy, auto-healer monitoring every 30s
532 lines
20 KiB
Bash
532 lines
20 KiB
Bash
#!/bin/bash
|
|
# ============================================================
|
|
# Falah Mobile — SRE Auto-Healing Agent
|
|
# ============================================================
|
|
# Log-aware fault diagnosis + restore protocol.
|
|
#
|
|
# On fault:
|
|
# 1. FETCH logs from failing container
|
|
# 2. ANALYZE logs against error knowledge base
|
|
# 3. DECIDE: quick hotfix or restore protocol?
|
|
# 4. RESTORE PROTOCOL: rollback to known-good image
|
|
# 5. While rolling back, capture full diagnostics for later fix
|
|
#
|
|
# This is NOT a blind restarter — it's a diagnostic-first SRE agent.
|
|
# ============================================================
|
|
|
|
# ── Configuration ────────────────────────────────────────────
|
|
|
|
CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
|
|
HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
|
|
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
|
|
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
|
|
MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
|
|
ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"
|
|
|
|
AGENT_LOG="/var/log/falah/agent.log"
|
|
STATE_DIR="/var/state/falah"
|
|
STATE_FILE="$STATE_DIR/restart-count"
|
|
INCIDENTS_DIR="/var/log/falah/incidents"
|
|
|
|
mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")" "$INCIDENTS_DIR"
|
|
|
|
# ── Logging ──────────────────────────────────────────────────
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$AGENT_LOG"
|
|
}
|
|
|
|
alert() {
|
|
local severity="$1" msg="$2"
|
|
log "[$severity] $msg"
|
|
# Future: webhook, ntfy, Slack, PagerDuty
|
|
}
|
|
|
|
# ── Restart Tracking (sliding window) ────────────────────────
|
|
|
|
count_restarts() {
|
|
[ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
|
|
local now window_start
|
|
now=$(date +%s)
|
|
window_start=$((now - 3600))
|
|
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
|
|
}
|
|
|
|
record_restart() {
|
|
date +%s >> "$STATE_FILE"
|
|
local now window_start
|
|
now=$(date +%s)
|
|
window_start=$((now - 3600))
|
|
awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
|
|
}
|
|
|
|
# ── Basic Health Checks ──────────────────────────────────────
|
|
|
|
check_http() {
|
|
local url="$1"
|
|
local body code
|
|
code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
|
|
body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
|
|
[ "$code" = "200" ] && { echo "$body"; return 0; } || return 1
|
|
}
|
|
|
|
check_disk() {
|
|
df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
|
|
}
|
|
|
|
# ── Phase 1: Detect Fault ────────────────────────────────────
|
|
|
|
detect_fault() {
|
|
# Returns: diagnosis string (or "healthy")
|
|
|
|
# 1. Container exists?
|
|
if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER_NAME}$"; then
|
|
echo "container_missing"
|
|
return
|
|
fi
|
|
|
|
# 2. Container running?
|
|
local status exit_code oom
|
|
status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "unknown")
|
|
|
|
if [ "$status" != "running" ]; then
|
|
exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
|
|
oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
|
|
[ "$oom" = "true" ] && { echo "oom_killed"; return; }
|
|
[ "$exit_code" = "137" ] && { echo "sigkill"; return; }
|
|
echo "crashed_exit_$exit_code"
|
|
return
|
|
fi
|
|
|
|
# 3. Health endpoint responds?
|
|
local health_body
|
|
health_body=$(check_http "$HEALTH_URL") || { echo "health_down"; return; }
|
|
|
|
# 4. DB connected?
|
|
echo "$health_body" | grep -q '"db":false' && { echo "db_down"; return; }
|
|
|
|
# 5. Status ok?
|
|
echo "$health_body" | grep -q '"status":"ok"' || { echo "health_invalid"; return; }
|
|
|
|
# 6. Gateway?
|
|
check_http "$GATEWAY_URL" > /dev/null || { echo "gateway_down"; return; }
|
|
|
|
echo "healthy"
|
|
}
|
|
|
|
# ── Phase 2: Fetch Logs ──────────────────────────────────────
|
|
|
|
fetch_logs() {
|
|
local lines="${1:-100}"
|
|
docker logs "$CONTAINER_NAME" --tail "$lines" 2>&1 || echo "LOG_FETCH_FAILED"
|
|
}
|
|
|
|
# ── Phase 3: Log Analysis (Knowledge Base) ───────────────────
|
|
|
|
analyze_logs() {
|
|
local diagnosis="$1"
|
|
local logs="$2"
|
|
|
|
# Build a diagnostic report
|
|
local root_cause="unknown"
|
|
local certainty="low"
|
|
local suggested_action="rollback"
|
|
local details=""
|
|
|
|
# ── Error Pattern Knowledge Base ───────────────────────────
|
|
# Each pattern maps to: root_cause | certainty | suggested_action
|
|
|
|
# Node.js crashes
|
|
if echo "$logs" | grep -qi "Cannot find module"; then
|
|
root_cause="missing_dependency"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "Cannot find module" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "Module not found"; then
|
|
root_cause="missing_dependency"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "Module not found" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "unexpected token"; then
|
|
root_cause="js_parse_error"
|
|
certainty="high"
|
|
suggested_action="rollback"
|
|
details=$(echo "$logs" | grep -i "unexpected token" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "SyntaxError"; then
|
|
root_cause="js_syntax_error"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "SyntaxError" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "ReferenceError"; then
|
|
root_cause="js_reference_error"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "ReferenceError" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "TypeError"; then
|
|
root_cause="js_type_error"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "TypeError" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "heap out of memory\|FATAL ERROR\|Allocation failed"; then
|
|
root_cause="out_of_memory"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "heap out of memory\|FATAL ERROR\|Allocation failed" | head -3 | tr '\n' '; ')
|
|
|
|
# Prisma / DB errors
|
|
elif echo "$logs" | grep -qi "PrismaClientInitializationError\|prisma.*connect.*ECONNREFUSED"; then
|
|
root_cause="db_connection_refused"
|
|
certainty="high"
|
|
suggested_action="restart_container"
|
|
details=$(echo "$logs" | grep -i "PrismaClientInitializationError\|ECONNREFUSED" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "prisma.*ENOENT\|prisma.*does not exist"; then
|
|
root_cause="prisma_schema_mismatch"
|
|
certainty="high"
|
|
details=$(echo "$logs" | grep -i "prisma.*ENOENT\|does not exist" | head -3 | tr '\n' '; ')
|
|
|
|
elif echo "$logs" | grep -qi "Can't reach database server\|getaddrinfo ENOTFOUND.*db\|postgres.*connect"; then
|
|
root_cause="db_unreachable"
|
|
certainty="high"
|
|
suggested_action="restart_container"
|
|
details=$(echo "$logs" | grep -i "ENOTFOUND\|Can't reach database" | head -3 | tr '\n' '; ')
|
|
|
|
# Network / connectivity
|
|
elif echo "$logs" | grep -qi "ECONNREFUSED\|ECONNRESET\|ETIMEDOUT\|ENOTFOUND"; then
|
|
root_cause="downstream_unreachable"
|
|
certainty="medium"
|
|
suggested_action="restart_container"
|
|
details=$(echo "$logs" | grep -i "ECONNREFUSED\|ECONNRESET\|ETIMEDOUT\|ENOTFOUND" | head -3 | tr '\n' '; ')
|
|
|
|
# Rate limiting
|
|
elif echo "$logs" | grep -qi "rate limit\|too many requests\|429"; then
|
|
root_cause="rate_limit_exceeded"
|
|
certainty="medium"
|
|
suggested_action="hotfix_rate_limit"
|
|
details=$(echo "$logs" | grep -i "rate limit\|too many" | head -3 | tr '\n' '; ')
|
|
|
|
# Auth / config
|
|
elif echo "$logs" | grep -qi "jwt expired\|invalid token\|Unauthorized"; then
|
|
root_cause="auth_config_error"
|
|
certainty="medium"
|
|
suggested_action="check_config"
|
|
details=$(echo "$logs" | grep -i "jwt expired\|invalid token\|Unauthorized" | head -3 | tr '\n' '; ')
|
|
|
|
# 5xx patterns from health endpoint
|
|
elif echo "$logs" | grep -qi "500\|Internal Server Error\|unhandled rejection\|uncaught exception"; then
|
|
root_cause="unhandled_exception"
|
|
certainty="medium"
|
|
details=$(echo "$logs" | grep -i "unhandled\|uncaught\|500\|Internal Server Error" | head -3 | tr '\n' '; ')
|
|
|
|
# Next.js specific
|
|
elif echo "$logs" | grep -qi "next.*error\|Error:.*page\|render.*error\|application.*error"; then
|
|
root_cause="application_error"
|
|
certainty="medium"
|
|
details=$(echo "$logs" | grep -i "Error:\|render.*error" | head -3 | tr '\n' '; ')
|
|
fi
|
|
|
|
# If diagnosis gives strong signal, incorporate it
|
|
case "$diagnosis" in
|
|
oom_killed|sigkill)
|
|
[ "$root_cause" = "unknown" ] && root_cause="container_killed_sigkill"
|
|
suggested_action="rollback"
|
|
;;
|
|
db_down)
|
|
[ "$root_cause" = "unknown" ] && root_cause="database_disconnected"
|
|
suggested_action="restart_container"
|
|
;;
|
|
health_down)
|
|
[ "$root_cause" = "unknown" ] && root_cause="container_unresponsive"
|
|
;;
|
|
gateway_down)
|
|
root_cause="nginx_gateway_down"
|
|
suggested_action="reload_gateway"
|
|
certainty="high"
|
|
;;
|
|
esac
|
|
|
|
# Return structured report as JSON-like lines
|
|
echo "ROOT_CAUSE=$root_cause"
|
|
echo "CERTAINTY=$certainty"
|
|
echo "ACTION=$suggested_action"
|
|
echo "DETAILS=$details"
|
|
}
|
|
|
|
# ── Phase 4: Restore Protocol ────────────────────────────────
|
|
# Fast rollback to last known-good state, while capturing
|
|
# full diagnostics for permanent fix later.
|
|
|
|
restore_protocol() {
|
|
local diagnosis="$1"
|
|
local root_cause="$2"
|
|
local action="$3"
|
|
local details="$4"
|
|
local logs="$5"
|
|
local incident_id
|
|
incident_id="incident_$(date +%Y%m%d_%H%M%S)"
|
|
|
|
log "🚨 RESTORE PROTOCOL triggered — incident: $incident_id"
|
|
log " Diagnosis: $diagnosis"
|
|
log " Root cause: $root_cause"
|
|
log " Action: $action"
|
|
log " Details: $details"
|
|
|
|
# ── Step 0: Save full diagnostic context for later fix ─────
|
|
{
|
|
echo "=== INCIDENT REPORT: $incident_id ==="
|
|
echo "Timestamp: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
|
echo "Container: $CONTAINER_NAME"
|
|
echo "Diagnosis: $diagnosis"
|
|
echo "Root cause: $root_cause"
|
|
echo "Certainty: $certainty"
|
|
echo "Action: $action"
|
|
echo "Details: $details"
|
|
echo ""
|
|
echo "=== Container Logs ==="
|
|
echo "$logs"
|
|
echo ""
|
|
echo "=== Container Inspect ==="
|
|
docker inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED"
|
|
echo ""
|
|
echo "=== Docker Events (last 5 min) ==="
|
|
timeout 5 docker events --since "${CHECK_INTERVAL}s" --until "$(date +%s)" \
|
|
--filter "container=$CONTAINER_NAME" \
|
|
--format '{{.Time}} {{.Type}} {{.Action}} {{.Status}}' 2>/dev/null || echo "EVENTS_FAILED"
|
|
echo ""
|
|
echo "=== System Resources ==="
|
|
df -h / | tail -1
|
|
free -m 2>/dev/null || echo "free_unavailable"
|
|
echo "=== END INCIDENT REPORT ==="
|
|
} > "$INCIDENTS_DIR/$incident_id.log"
|
|
|
|
log "📝 Incident saved to $INCIDENTS_DIR/$incident_id.log"
|
|
alert "INFO" "Incident $incident_id logged — root cause: $root_cause"
|
|
|
|
# ── Step 1: Apply action ───────────────────────────────────
|
|
case "$action" in
|
|
rollback)
|
|
log "🔄 RESTORE: rolling back to $ROLLBACK_IMAGE..."
|
|
if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
|
|
# Remove old container
|
|
docker stop "$CONTAINER_NAME" 2>/dev/null || true
|
|
sleep 2
|
|
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
|
sleep 2
|
|
|
|
# Recreate from rollback image
|
|
local network
|
|
network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge")
|
|
docker run -d \
|
|
--name "$CONTAINER_NAME" \
|
|
--restart unless-stopped \
|
|
-p 4014:3000 \
|
|
-v /var/services/homes/admin/falah-mobile/data:/app/data \
|
|
--network "$network" \
|
|
"$ROLLBACK_IMAGE" >> "$AGENT_LOG" 2>&1
|
|
|
|
sleep 15
|
|
if check_http "$HEALTH_URL" > /dev/null 2>&1; then
|
|
log "✅ RESTORE: rollback successful — service restored"
|
|
alert "INFO" "Restore protocol: rolled back to $ROLLBACK_IMAGE (incident: $incident_id)"
|
|
record_restart
|
|
return 0
|
|
else
|
|
log "❌ RESTORE: rollback also failed — escalation needed"
|
|
alert "CRITICAL" "Rollback failed for incident $incident_id — manual intervention"
|
|
return 1
|
|
fi
|
|
else
|
|
log "⚠️ RESTORE: no rollback image found — attempting restart instead"
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
sleep 10
|
|
if check_http "$HEALTH_URL" > /dev/null 2>&1; then
|
|
log "✅ RESTORE: restart successful (fallback)"
|
|
record_restart
|
|
return 0
|
|
fi
|
|
return 1
|
|
fi
|
|
;;
|
|
|
|
restart_container)
|
|
log "🔄 RESTORE: restarting container..."
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
record_restart
|
|
sleep 10
|
|
if check_http "$HEALTH_URL" > /dev/null 2>&1; then
|
|
log "✅ RESTORE: restart successful"
|
|
return 0
|
|
else
|
|
log "⚠️ RESTORE: restart didn't fix — escalating to rollback"
|
|
restore_protocol "$diagnosis" "restart_failed" "rollback" "$details" "$logs"
|
|
fi
|
|
;;
|
|
|
|
reload_gateway)
|
|
log "🔄 RESTORE: reloading nginx gateway..."
|
|
local gw
|
|
gw=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
|
|
[ -n "$gw" ] && docker exec "$gw" nginx -s reload >> "$AGENT_LOG" 2>&1
|
|
sleep 3
|
|
check_http "$GATEWAY_URL" > /dev/null && log "✅ RESTORE: gateway restored" || log "⚠️ Gateway still down"
|
|
;;
|
|
|
|
hotfix_rate_limit)
|
|
log "🔄 RESTORE: rate limit issue — applying hotfix"
|
|
# Rate limiter auto-clears stale entries every 5 min (code-level fix)
|
|
# For immediate relief, restart container
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
record_restart
|
|
sleep 10
|
|
;;
|
|
|
|
check_config)
|
|
log "🔄 RESTORE: config issue detected — rolling back to safe state"
|
|
restore_protocol "$diagnosis" "config_error" "rollback" "$details" "$logs"
|
|
;;
|
|
|
|
*)
|
|
log "🔄 RESTORE: unknown action '$action' — attempting restart as safe default"
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
record_restart
|
|
sleep 10
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ── Phase 5: Apply Quick Hotfix (when possible) ──────────────
|
|
|
|
apply_hotfix() {
|
|
local root_cause="$1"
|
|
local logs="$2"
|
|
|
|
case "$root_cause" in
|
|
rate_limit_exceeded)
|
|
log "🔧 HOTFIX: rate limit exceeded — restarting to clear state"
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
sleep 5
|
|
log "🔧 HOTFIX: rate limiter now auto-cleans every 5 min (code fix in place)"
|
|
;;
|
|
|
|
db_connection_refused|db_unreachable)
|
|
log "🔧 HOTFIX: DB connection issue — restarting container"
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
sleep 10
|
|
;;
|
|
|
|
*)
|
|
log "🔧 HOTFIX: no quick fix for '$root_cause' — relying on restore protocol"
|
|
return 1
|
|
;;
|
|
esac
|
|
return 0
|
|
}
|
|
|
|
# ── Periodic Maintenance ────────────────────────────────────
|
|
|
|
periodic_maintenance() {
|
|
local disk_pct
|
|
disk_pct=$(check_disk)
|
|
if [ "$disk_pct" -gt 90 ]; then
|
|
log "⚠️ Disk at ${disk_pct}% — running prune"
|
|
docker system prune -f >> "$AGENT_LOG" 2>&1
|
|
log "✅ Prune completed"
|
|
fi
|
|
}
|
|
|
|
# ── Main Loop ────────────────────────────────────────────────
|
|
|
|
log "══════════════════════════════════════════════════════"
|
|
log "🚀 SRE Auto-Healing Agent starting"
|
|
log " Container: $CONTAINER_NAME"
|
|
log " Health URL: $HEALTH_URL"
|
|
log " Gateway URL: $GATEWAY_URL"
|
|
log " Interval: ${CHECK_INTERVAL}s"
|
|
log " Max restarts: $MAX_RESTARTS/hour"
|
|
log " Incidents: $INCIDENTS_DIR"
|
|
log "══════════════════════════════════════════════════════"
|
|
|
|
CYCLE=0
|
|
HEALTHY_CYCLES=0
|
|
|
|
while true; do
|
|
# ── Detect ──────────────────────────────────────────────
|
|
DIAGNOSIS=$(detect_fault)
|
|
|
|
if [ "$DIAGNOSIS" = "healthy" ]; then
|
|
# Stability counter — reset restart tracking after 5 min healthy
|
|
HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1))
|
|
if [ "$HEALTHY_CYCLES" -ge 10 ]; then
|
|
HEALTHY_CYCLES=0
|
|
echo 0 > "$STATE_FILE" 2>/dev/null || true
|
|
fi
|
|
else
|
|
# ── FAULT DETECTED ────────────────────────────────────
|
|
HEALTHY_CYCLES=0
|
|
log ""
|
|
log "⚠️ FAULT DETECTED: $DIAGNOSIS"
|
|
|
|
# Step 1: Fetch logs
|
|
log "📋 Fetching container logs..."
|
|
CONTAINER_LOGS=$(fetch_logs 100)
|
|
|
|
# Step 2: Analyze logs against knowledge base
|
|
log "🔍 Analyzing logs for root cause..."
|
|
ANALYSIS=$(analyze_logs "$DIAGNOSIS" "$CONTAINER_LOGS")
|
|
|
|
# Parse analysis output
|
|
ROOT_CAUSE=$(echo "$ANALYSIS" | grep "^ROOT_CAUSE=" | cut -d= -f2-)
|
|
CERTAINTY=$(echo "$ANALYSIS" | grep "^CERTAINTY=" | cut -d= -f2-)
|
|
ACTION=$(echo "$ANALYSIS" | grep "^ACTION=" | cut -d= -f2-)
|
|
DETAILS=$(echo "$ANALYSIS" | grep "^DETAILS=" | cut -d= -f2-)
|
|
|
|
log " Root cause: $ROOT_CAUSE (certainty: $CERTAINTY)"
|
|
log " Action: $ACTION"
|
|
[ -n "$DETAILS" ] && log " Details: $DETAILS"
|
|
|
|
# Step 3: Decision — can we hotfix or need restore protocol?
|
|
restarts=$(count_restarts)
|
|
|
|
if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
|
|
# Too many restarts — go straight to restore protocol
|
|
log "⚠️ $restarts restarts in last hour — exceeding limit of $MAX_RESTARTS"
|
|
log "🚨 SKIPPING hotfix attempt — going directly to restore protocol"
|
|
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
|
|
|
|
elif [ "$ACTION" = "rollback" ]; then
|
|
# Knowledge base says rollback — run restore protocol
|
|
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "$ACTION" "$DETAILS" "$CONTAINER_LOGS"
|
|
|
|
elif [ "$ACTION" = "restart_container" ]; then
|
|
# Try restart first, escalate if fails
|
|
log "🔄 Attempting restart (restart #$restarts)..."
|
|
apply_hotfix "$ROOT_CAUSE" "$CONTAINER_LOGS"
|
|
sleep 10
|
|
if [ "$(detect_fault)" != "healthy" ]; then
|
|
log "⚠️ Restart didn't resolve — escalating to restore protocol"
|
|
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
|
|
fi
|
|
|
|
elif [ "$ACTION" = "reload_gateway" ]; then
|
|
restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "$ACTION" "$DETAILS" "$CONTAINER_LOGS"
|
|
|
|
elif [ "$ACTION" = "hotfix_rate_limit" ]; then
|
|
apply_hotfix "$ROOT_CAUSE" "$CONTAINER_LOGS"
|
|
|
|
else
|
|
# No specific action — safe default: restart, then rollback
|
|
log "🔄 No specific fix for '$ROOT_CAUSE' — attempting restart"
|
|
docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
|
|
record_restart
|
|
sleep 10
|
|
[ "$(detect_fault)" != "healthy" ] && restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
|
|
fi
|
|
fi
|
|
|
|
# ── Periodic maintenance (every 10 cycles = 5 min) ─────
|
|
CYCLE=$((CYCLE + 1))
|
|
[ "$CYCLE" -ge 10 ] && { CYCLE=0; periodic_maintenance; }
|
|
|
|
sleep "$CHECK_INTERVAL"
|
|
done
|