feat: auto-healing system — HEALTHCHECK, rate-limiter cleanup, auto-rollback CI, watchdog

- Dockerfile: add HEALTHCHECK (30s interval, 10s timeout, 3 retries) - rate-limit.ts: auto-cleanup stale entries every 5 min (prevents memory leak) - scripts/auto-heal.sh: watchdog for Synology cron — auto-restarts container, reloads nginx gateway, escalates on excessive failures - CI workflow: auto-rollback on deploy failure (tags :rollback image, reverts if health check fails after deploy)
2026-06-28 02:52:11 +08:00
parent aac485ba30
commit 0098d9ca4a
4 changed files with 249 additions and 13 deletions
@@ -0,0 +1,162 @@
+#!/bin/bash
+# ============================================================
+# Falah Mobile — Auto-Healing Watchdog
+# ============================================================
+# Runs as a cron job or systemd timer on the Synology.
+# Detects failures and auto-fixes without human intervention.
+#
+# Install (as root on Synology):
+#   cp scripts/auto-heal.sh /usr/local/bin/falah-auto-heal
+#   chmod +x /usr/local/bin/falah-auto-heal
+#   echo "*/5 * * * * root /usr/local/bin/falah-auto-heal" > /etc/cron.d/falah-auto-heal
+# ============================================================
+
+DOCKER=/var/packages/Docker/target/usr/bin/docker
+CONTAINER=falah-mobile-staging
+HEALTH_URL="http://localhost:4014/mobile/api/health"
+GW_URL="http://localhost:7878/mobile/api/health"
+LOG_DIR="/var/log/falah"
+LOG="$LOG_DIR/auto-heal.log"
+STATE="$LOG_DIR/auto-heal.state"
+MAX_RESTARTS_PER_HOUR=3
+
+mkdir -p "$LOG_DIR"
+
+log() {
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"
+  echo "$*"
+}
+
+# ── 1. HEALTH CHECK ──────────────────────────────────────────
+
+check_health() {
+  local url="$1"
+  local label="$2"
+  curl -sf --max-time 10 "$url" > /dev/null 2>&1
+}
+
+# ── 2. TRACK RESTART COUNT ──────────────────────────────────
+
+track_restart() {
+  local now
+  now=$(date +%s)
+  # Keep only restarts in the last hour
+  if [ -f "$STATE" ]; then
+    sed -i "/^[0-9]*$/!d" "$STATE" 2>/dev/null
+    awk -v now="$now" -v limit=3600 '$1 > now - limit' "$STATE" > "${STATE}.tmp" && mv "${STATE}.tmp" "$STATE"
+  fi
+  echo "$now" >> "$STATE"
+  local count
+  count=$(wc -l < "$STATE" 2>/dev/null || echo 0)
+  echo "$count"
+}
+
+# ── 3. AUTO-HEAL ACTIONS ────────────────────────────────────
+
+heal_container() {
+  log "⚠️  Starting auto-heal sequence for $CONTAINER..."
+
+  # Check if container exists
+  if ! $DOCKER ps -a --format '{{.Names}}' | grep -q "^$CONTAINER$"; then
+    log "❌ Container $CONTAINER does not exist. Cannot auto-heal."
+    return 1
+  fi
+
+  local restarts
+  restarts=$(track_restart)
+  log "🔄 Restarts in last hour: $restarts / $MAX_RESTARTS_PER_HOUR"
+
+  if [ "$restarts" -gt "$MAX_RESTARTS_PER_HOUR" ]; then
+    log "🚨 Too many restarts ($restarts) — escalating instead of restarting"
+    return 2
+  fi
+
+  # Step 1: Try restarting the container
+  log "🔄 Restarting $CONTAINER..."
+  $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
+  sleep 10
+
+  # Step 2: Verify recovery
+  if check_health "$HEALTH_URL" "direct"; then
+    log "✅ Auto-heal successful — container healthy after restart"
+    return 0
+  fi
+
+  # Step 3: If still failing, try docker-compose (rebuild from compose definition)
+  log "⚠️  Restart not enough — trying compose down/up..."
+  cd /volume1/docker/falah-core-staging/maifors-falah-core-*/ 2>/dev/null || \
+    cd /var/services/homes/admin/falah-mobile/ 2>/dev/null || true
+  
+  $DOCKER compose -f docker-compose.staging.yml down >> "$LOG" 2>&1
+  sleep 3
+  $DOCKER compose -f docker-compose.staging.yml up -d >> "$LOG" 2>&1
+  sleep 15
+
+  if check_health "$HEALTH_URL" "direct"; then
+    log "✅ Auto-heal successful — compose recovery worked"
+    return 0
+  fi
+
+  log "❌ Auto-heal failed — manual intervention required"
+  return 3
+}
+
+heal_gateway() {
+  log "⚠️  Gateway unhealthy — reloading nginx..."
+  
+  # Find the nginx container
+  GW_CONTAINER=$($DOCKER ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
+  if [ -n "$GW_CONTAINER" ]; then
+    $DOCKER exec "$GW_CONTAINER" nginx -s reload >> "$LOG" 2>&1
+    log "✅ nginx reloaded in $GW_CONTAINER"
+    return 0
+  fi
+  log "⚠️  No nginx/gateway container found"
+  return 1
+}
+
+# ── 4. MAIN ──────────────────────────────────────────────────
+
+main() {
+  log "═══════════════════════════════════════════════"
+  log "🔍 Auto-heal check starting..."
+
+  local direct_ok=true
+  local gateway_ok=true
+
+  # Check direct container health
+  if ! check_health "$HEALTH_URL" "direct"; then
+    direct_ok=false
+    log "❌ Direct health check FAILED"
+    heal_container
+  else
+    log "✅ Direct container healthy"
+  fi
+
+  # Check gateway health
+  if ! check_health "$GW_URL" "gateway"; then
+    gateway_ok=false
+    log "❌ Gateway health check FAILED"
+    heal_gateway
+  else
+    log "✅ Gateway healthy"
+  fi
+
+  # Check db connectivity via direct health
+  HEALTH_BODY=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null)
+  if echo "$HEALTH_BODY" | grep -q '"db":false'; then
+    log "⚠️  Database disconnected! Attempting recovery..."
+    $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
+    log "🔄 Restarted container to recover DB connection"
+  fi
+
+  if $direct_ok && $gateway_ok; then
+    log "✅ All systems healthy"
+  else
+    log "⚠️  Some systems unhealthy after healing attempts"
+  fi
+
+  log "═══════════════════════════════════════════════"
+}
+
+main "$@"