#!/bin/bash
# ============================================================
# Falah Mobile — Auto-Healing Agent (Dedicated Container)
# ============================================================
# Real-time fault detection + autonomous remediation.
# Runs as a sidecar container alongside falah-mobile-staging.
#
# Detects:
#   Container crash, health 5xx, DB disconnect, OOM,
#   Gateway 502, 5xx spikes, disk pressure
#
# Fixes:
#   Restart, rollback, nginx reload, prune, escalate
# ============================================================

# NO set -e — we handle errors explicitly to avoid the entire agent crashing
# on a single failed remediation step. Each function manages its own errors.

CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
LOG="/var/log/falah/agent.log"
STATE_DIR="/var/state/falah"
STATE_FILE="$STATE_DIR/restart-count"
ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"

mkdir -p "$STATE_DIR" "$(dirname "$LOG")"

# ── Logging ──────────────────────────────────────────────────

log() {
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$LOG"
}

alert() {
  local severity="$1" msg="$2"
  log "[$severity] $msg"
  # Future: emit to webhook, ntfy, Slack, etc.
  # curl -sf -X POST -H "Content-Type: application/json" \
  #   -d "{\"severity\":\"$severity\",\"message\":\"$msg\",\"service\":\"$CONTAINER_NAME\"}" \
  #   "$ALERT_WEBHOOK" &
}

# ── Track restarts (sliding window per hour) ─────────────────

count_restarts() {
  [ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
  local now window_start count
  now=$(date +%s)
  window_start=$((now - 3600))
  
  # Filter entries within the last hour
  awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
}

record_restart() {
  date +%s >> "$STATE_FILE"
  # Trim old entries
  local now window_start
  now=$(date +%s)
  window_start=$((now - 3600))
  awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
}

# ── Fault Detection ──────────────────────────────────────────

check_http() {
  local url="$1" label="$2"
  local code body
  body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
  code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
  
  if [ "$code" = "200" ]; then
    echo "$body"
    return 0
  fi
  return 1
}

check_db() {
  local body
  body=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null) || return 1
  echo "$body" | grep -q '"db":true' && return 0 || return 1
}

check_docker_events() {
  # Check if the container recently crashed
  local since_seconds="${1:-120}"
  local since_epoch
  since_epoch=$(date -d "-${since_seconds} seconds" +%s 2>/dev/null || echo $(( $(date +%s) - since_seconds )))
  
  # Use docker events --since to check for recent die/oom events
  docker events --since "${since_epoch}s" --filter "container=$CONTAINER_NAME" \
    --filter "event=die" --format '{{.Status}}' 2>/dev/null | head -1
}

check_disk() {
  df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
}

# ── Fault Diagnosis ──────────────────────────────────────────

diagnose() {
  local fault_reason=""
  local health_body
  
  # 1. Check if container exists
  if ! docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
    echo "container_missing"
    return
  fi
  
  # 2. Check if container is running
  local status
  status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "missing")
  if [ "$status" != "running" ]; then
    local exit_code
    exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
    local oom
    oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
    
    if [ "$oom" = "true" ]; then
      echo "oom_killed"
    elif [ "$exit_code" = "137" ]; then
      echo "sigkill_oom"
    elif [ "$exit_code" != "0" ]; then
      echo "crashed_exit_${exit_code}"
    else
      echo "not_running"
    fi
    return
  fi
  
  # 3. Check health endpoint
  health_body=$(check_http "$HEALTH_URL" "direct" 2>/dev/null) || true
  if [ -z "$health_body" ]; then
    echo "health_unreachable"
    return
  fi
  
  # 4. Check DB connectivity
  local db_status
  db_status=$(echo "$health_body" | grep -o '"db":false\|"db":true' || echo "unknown")
  if [ "$db_status" = '"db":false' ]; then
    echo "db_disconnected"
    return
  fi
  
  # 5. Check response content
  if ! echo "$health_body" | grep -q '"status":"ok"'; then
    echo "health_invalid"
    return
  fi
  
  # 6. Check gateway
  if ! check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
    echo "gateway_down"
    return
  fi
  
  echo "healthy"
}

# ── Remediations ─────────────────────────────────────────────

remediate_container_restart() {
  local diagnosis="$1"
  local restarts
  restarts=$(count_restarts)
  
  log "🔄 Remediation: restart container (diagnosis=$diagnosis, restarts_last_hour=$restarts)"
  
  if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
    alert "CRITICAL" "Excessive restarts ($restarts/h) — escalating to rollback"
    remediate_rollback
    return
  fi
  
  docker restart "$CONTAINER_NAME" >> "$LOG" 2>&1
  record_restart
  sleep 10
  
  # Verify
  if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
    log "✅ Restart successful — container healthy"
    alert "INFO" "Auto-healed: restart $CONTAINER_NAME (diagnosis=$diagnosis)"
    return 0
  else
    log "⚠️  Restart didn't fix — escalating"
    remediate_escalate
  fi
}

remediate_rollback() {
  log "🔄 Remediation: rollback to previous image"
  
  if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
    log "🔄 Rolling back to $ROLLBACK_IMAGE..."
    
    # Stop and remove old container if it exists (may have been auto-restarted by Docker)
    if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
      docker stop "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
      sleep 2
      docker rm -f "$CONTAINER_NAME" >> "$LOG" 2>&1 || true
      sleep 2
    fi
    
    # Extract original run config from inspect (if available) or use defaults
    local ports volumes network env_vars
    ports=$(docker inspect "$CONTAINER_NAME" --format '{{range $p, $conf := .HostConfig.PortBindings}}{{$p}} {{end}}' 2>/dev/null || echo "4014:3000")
    volumes=$(docker inspect "$CONTAINER_NAME" --format '{{range $v := .Mounts}}{{$v.Source}}:{{$v.Destination}} {{end}}' 2>/dev/null)
    network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge")
    
    # Run rollback image with same config
    docker run -d \
      --name "$CONTAINER_NAME" \
      --restart unless-stopped \
      -p 4014:3000 \
      $volumes \
      --network "$network" \
      "$ROLLBACK_IMAGE" >> "$LOG" 2>&1
    
    sleep 15
    
    if check_http "$HEALTH_URL" "direct" > /dev/null 2>&1; then
      log "✅ Rollback successful — previous image restored"
      alert "INFO" "Auto-healed: rolled back to $ROLLBACK_IMAGE"
      return 0
    else
      alert "CRITICAL" "Rollback also failed — manual intervention required"
      return 1
    fi
  else
    log "⚠️  No rollback image found — cannot rollback"
    alert "WARN" "No rollback image available"
    return 1
  fi
}

remediate_gateway() {
  log "🔄 Remediation: reload nginx gateway"
  
  local gw_container
  gw_container=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
  
  if [ -n "$gw_container" ]; then
    docker exec "$gw_container" nginx -s reload >> "$LOG" 2>&1
    sleep 3
    if check_http "$GATEWAY_URL" "gateway" > /dev/null 2>&1; then
      log "✅ Gateway reload successful"
      alert "INFO" "Auto-healed: reloaded nginx in $gw_container"
      return 0
    fi
  fi
  log "⚠️  Could not reload gateway"
  return 1
}

remediate_prune() {
  log "🔄 Remediation: docker system prune (disk pressure)"
  docker system prune -f >> "$LOG" 2>&1
  log "✅ Prune completed"
  alert "INFO" "Auto-healed: pruned Docker resources"
}

remediate_escalate() {
  alert "CRITICAL" "Auto-heal unable to fix — escalation needed"
  # Future: send to PagerDuty, OpsGenie, Slack, etc.
  log "🚨 ESCALATION: Manual intervention required for $CONTAINER_NAME"
}

# ── Main Loop ────────────────────────────────────────────────

log "══════════════════════════════════════════════════════"
log "🚀 Auto-Healing Agent starting"
log "   Container:     $CONTAINER_NAME"
log "   Health URL:    $HEALTH_URL"
log "   Gateway URL:   $GATEWAY_URL"
log "   Interval:      ${CHECK_INTERVAL}s"
log "   Max restarts:  $MAX_RESTARTS/hour"
log "══════════════════════════════════════════════════════"

while true; do
  # ── Phase 1: Detect ──
  DIAGNOSIS=$(diagnose)
  
  # ── Phase 2: Act ──
  case "$DIAGNOSIS" in
    healthy)
      # All good — reset restart count if we've had consecutive healthy cycles
      if [ -z "$HEALTHY_CYCLES" ]; then HEALTHY_CYCLES=0; fi
      HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1))
      # After 10 healthy cycles (5 min), reset restart counter
      if [ "$HEALTHY_CYCLES" -ge 10 ]; then
        HEALTHY_CYCLES=0
        # Reset state file — system is stable now
        echo 0 > "$STATE_FILE" 2>/dev/null || true
      fi
      ;;
      
    container_missing)
      alert "CRITICAL" "Container $CONTAINER_NAME does not exist"
      ;;
      
    oom_killed|sigkill_oom)
      alert "WARN" "Container killed ($DIAGNOSIS) — restarting"
      # If we've had too many restarts, escalate to rollback
      local restarts
      restarts=$(count_restarts)
      if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
        alert "CRITICAL" "OOM repeated ($restarts restarts/hour) — rolling back"
        remediate_rollback
      else
        remediate_container_restart "$DIAGNOSIS"
      fi
      ;;
      
    crashed_*|not_running)
      alert "WARN" "Container crashed ($DIAGNOSIS)"
      remediate_container_restart "$DIAGNOSIS"
      ;;
      
    health_unreachable)
      alert "WARN" "Health endpoint unreachable"
      remediate_container_restart "$DIAGNOSIS"
      ;;
      
    db_disconnected)
      alert "WARN" "Database disconnected"
      remediate_container_restart "$DIAGNOSIS"
      ;;
      
    health_invalid)
      alert "WARN" "Health response invalid"
      remediate_container_restart "$DIAGNOSIS"
      ;;
      
    gateway_down)
      alert "WARN" "Gateway down"
      remediate_gateway
      ;;
      
    *)
      alert "WARN" "Unknown diagnosis: $DIAGNOSIS"
      remediate_container_restart "$DIAGNOSIS"
      ;;
  esac
  
  # ── Periodic maintenance ──
  # Check disk every 10 cycles (5 min at 30s interval)
  CYCLE_COUNTER=$((CYCLE_COUNTER + 1))
  if [ "$CYCLE_COUNTER" -ge 10 ]; then
    CYCLE_COUNTER=0
    DISK_PCT=$(check_disk)
    if [ "$DISK_PCT" -gt 90 ]; then
      alert "WARN" "Disk at ${DISK_PCT}% — pruning"
      remediate_prune
    fi
  fi
  
  sleep "$CHECK_INTERVAL"
done