falah-mobile/scripts/auto-heal.sh

#!/bin/bash
# ============================================================
# Falah Mobile — Auto-Healing Watchdog
# ============================================================
# Runs as a cron job or systemd timer on the Synology.
# Detects failures and auto-fixes without human intervention.
#
# Install (as root on Synology):
#   cp scripts/auto-heal.sh /usr/local/bin/falah-auto-heal
#   chmod +x /usr/local/bin/falah-auto-heal
#   echo "*/5 * * * * root /usr/local/bin/falah-auto-heal" > /etc/cron.d/falah-auto-heal
# ============================================================

DOCKER=/var/packages/Docker/target/usr/bin/docker
CONTAINER=falah-mobile-staging
HEALTH_URL="http://localhost:4014/mobile/api/health"
GW_URL="http://localhost:7878/mobile/api/health"
LOG_DIR="/var/log/falah"
LOG="$LOG_DIR/auto-heal.log"
STATE="$LOG_DIR/auto-heal.state"
MAX_RESTARTS_PER_HOUR=3

mkdir -p "$LOG_DIR"

log() {
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"
  echo "$*"
}

# ── 1. HEALTH CHECK ──────────────────────────────────────────

check_health() {
  local url="$1"
  local label="$2"
  curl -sf --max-time 10 "$url" > /dev/null 2>&1
}

# ── 2. TRACK RESTART COUNT ──────────────────────────────────

track_restart() {
  local now
  now=$(date +%s)
  # Keep only restarts in the last hour
  if [ -f "$STATE" ]; then
    sed -i "/^[0-9]*$/!d" "$STATE" 2>/dev/null
    awk -v now="$now" -v limit=3600 '$1 > now - limit' "$STATE" > "${STATE}.tmp" && mv "${STATE}.tmp" "$STATE"
  fi
  echo "$now" >> "$STATE"
  local count
  count=$(wc -l < "$STATE" 2>/dev/null || echo 0)
  echo "$count"
}

# ── 3. AUTO-HEAL ACTIONS ────────────────────────────────────

heal_container() {
  log "⚠️  Starting auto-heal sequence for $CONTAINER..."

  # Check if container exists
  if ! $DOCKER ps -a --format '{{.Names}}' | grep -q "^$CONTAINER$"; then
    log "❌ Container $CONTAINER does not exist. Cannot auto-heal."
    return 1
  fi

  local restarts
  restarts=$(track_restart)
  log "🔄 Restarts in last hour: $restarts / $MAX_RESTARTS_PER_HOUR"

  if [ "$restarts" -gt "$MAX_RESTARTS_PER_HOUR" ]; then
    log "🚨 Too many restarts ($restarts) — escalating instead of restarting"
    return 2
  fi

  # Step 1: Try restarting the container
  log "🔄 Restarting $CONTAINER..."
  $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
  sleep 10

  # Step 2: Verify recovery
  if check_health "$HEALTH_URL" "direct"; then
    log "✅ Auto-heal successful — container healthy after restart"
    return 0
  fi

  # Step 3: If still failing, try docker-compose (rebuild from compose definition)
  log "⚠️  Restart not enough — trying compose down/up..."
  cd /volume1/docker/falah-core-staging/maifors-falah-core-*/ 2>/dev/null || \
    cd /var/services/homes/admin/falah-mobile/ 2>/dev/null || true
  
  $DOCKER compose -f docker-compose.staging.yml down >> "$LOG" 2>&1
  sleep 3
  $DOCKER compose -f docker-compose.staging.yml up -d >> "$LOG" 2>&1
  sleep 15

  if check_health "$HEALTH_URL" "direct"; then
    log "✅ Auto-heal successful — compose recovery worked"
    return 0
  fi

  log "❌ Auto-heal failed — manual intervention required"
  return 3
}

heal_gateway() {
  log "⚠️  Gateway unhealthy — reloading nginx..."
  
  # Find the nginx container
  GW_CONTAINER=$($DOCKER ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
  if [ -n "$GW_CONTAINER" ]; then
    $DOCKER exec "$GW_CONTAINER" nginx -s reload >> "$LOG" 2>&1
    log "✅ nginx reloaded in $GW_CONTAINER"
    return 0
  fi
  log "⚠️  No nginx/gateway container found"
  return 1
}

# ── 4. MAIN ──────────────────────────────────────────────────

main() {
  log "═══════════════════════════════════════════════"
  log "🔍 Auto-heal check starting..."

  local direct_ok=true
  local gateway_ok=true

  # Check direct container health
  if ! check_health "$HEALTH_URL" "direct"; then
    direct_ok=false
    log "❌ Direct health check FAILED"
    heal_container
  else
    log "✅ Direct container healthy"
  fi

  # Check gateway health
  if ! check_health "$GW_URL" "gateway"; then
    gateway_ok=false
    log "❌ Gateway health check FAILED"
    heal_gateway
  else
    log "✅ Gateway healthy"
  fi

  # Check db connectivity via direct health
  HEALTH_BODY=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null)
  if echo "$HEALTH_BODY" | grep -q '"db":false'; then
    log "⚠️  Database disconnected! Attempting recovery..."
    $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
    log "🔄 Restarted container to recover DB connection"
  fi

  if $direct_ok && $gateway_ok; then
    log "✅ All systems healthy"
  else
    log "⚠️  Some systems unhealthy after healing attempts"
  fi

  log "═══════════════════════════════════════════════"
}

main "$@"