#!/bin/bash
# ============================================================
# Falah Mobile — SRE Auto-Healing Agent
# ============================================================
# Log-aware fault diagnosis + restore protocol.
#
# On fault:
#   1. FETCH logs from failing container
#   2. ANALYZE logs against error knowledge base
#   3. DECIDE: quick hotfix or restore protocol?
#   4. RESTORE PROTOCOL: rollback to known-good image
#   5. While rolling back, capture full diagnostics for later fix
#
# This is NOT a blind restarter — it's a diagnostic-first SRE agent.
# ============================================================

# ── Configuration ────────────────────────────────────────────

CONTAINER_NAME="${CONTAINER_NAME:-falah-mobile-staging}"
HEALTH_URL="${HEALTH_URL:-http://localhost:4014/mobile/api/health}"
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
MAX_RESTARTS="${MAX_RESTARTS_PER_HOUR:-3}"
ROLLBACK_IMAGE="${ROLLBACK_IMAGE:-falah-mobile:rollback}"

AGENT_LOG="/var/log/falah/agent.log"
STATE_DIR="/var/state/falah"
STATE_FILE="$STATE_DIR/restart-count"
INCIDENTS_DIR="/var/log/falah/incidents"

mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")" "$INCIDENTS_DIR"

# ── Logging ──────────────────────────────────────────────────

log() {
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [agent] $*" | tee -a "$AGENT_LOG"
}

alert() {
  local severity="$1" msg="$2"
  log "[$severity] $msg"
  # Future: webhook, ntfy, Slack, PagerDuty
}

# ── Restart Tracking (sliding window) ────────────────────────

count_restarts() {
  [ -f "$STATE_FILE" ] || echo 0 > "$STATE_FILE"
  local now window_start
  now=$(date +%s)
  window_start=$((now - 3600))
  awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" 2>/dev/null | wc -l
}

record_restart() {
  date +%s >> "$STATE_FILE"
  local now window_start
  now=$(date +%s)
  window_start=$((now - 3600))
  awk -v ws="$window_start" '$1 > ws' "$STATE_FILE" > "${STATE_FILE}.tmp" && mv "${STATE_FILE}.tmp" "$STATE_FILE"
}

# ── Basic Health Checks ──────────────────────────────────────

check_http() {
  local url="$1"
  local body code
  code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "$url" 2>/dev/null || echo "000")
  body=$(curl -sf --max-time 10 "$url" 2>/dev/null) || true
  [ "$code" = "200" ] && { echo "$body"; return 0; } || return 1
}

check_disk() {
  df / | awk 'NR==2 {gsub(/%/,"",$5); print $5}'
}

# ── Phase 1: Detect Fault ────────────────────────────────────

detect_fault() {
  # Returns: diagnosis string (or "healthy")
  
  # 1. Container exists?
  if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "^${CONTAINER_NAME}$"; then
    echo "container_missing"
    return
  fi

  # 2. Container running?
  local status exit_code oom
  status=$(docker inspect "$CONTAINER_NAME" --format '{{.State.Status}}' 2>/dev/null || echo "unknown")
  
  if [ "$status" != "running" ]; then
    exit_code=$(docker inspect "$CONTAINER_NAME" --format '{{.State.ExitCode}}' 2>/dev/null || echo "0")
    oom=$(docker inspect "$CONTAINER_NAME" --format '{{.State.OOMKilled}}' 2>/dev/null || echo "false")
    [ "$oom" = "true" ] && { echo "oom_killed"; return; }
    [ "$exit_code" = "137" ] && { echo "sigkill"; return; }
    echo "crashed_exit_$exit_code"
    return
  fi

  # 3. Health endpoint responds?
  local health_body
  health_body=$(check_http "$HEALTH_URL") || { echo "health_down"; return; }

  # 4. DB connected?
  echo "$health_body" | grep -q '"db":false' && { echo "db_down"; return; }

  # 5. Status ok?
  echo "$health_body" | grep -q '"status":"ok"' || { echo "health_invalid"; return; }

  # 6. Gateway?
  check_http "$GATEWAY_URL" > /dev/null || { echo "gateway_down"; return; }

  echo "healthy"
}

# ── Phase 2: Fetch Logs ──────────────────────────────────────

fetch_logs() {
  local lines="${1:-100}"
  docker logs "$CONTAINER_NAME" --tail "$lines" 2>&1 || echo "LOG_FETCH_FAILED"
}

# ── Phase 3: Log Analysis (Knowledge Base) ───────────────────

analyze_logs() {
  local diagnosis="$1"
  local logs="$2"

  # Build a diagnostic report
  local root_cause="unknown"
  local certainty="low"
  local suggested_action="rollback"
  local details=""

  # ── Error Pattern Knowledge Base ───────────────────────────
  # Each pattern maps to: root_cause | certainty | suggested_action

  # Node.js crashes
  if echo "$logs" | grep -qi "Cannot find module"; then
    root_cause="missing_dependency"
    certainty="high"
    details=$(echo "$logs" | grep -i "Cannot find module" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "Module not found"; then
    root_cause="missing_dependency"
    certainty="high"
    details=$(echo "$logs" | grep -i "Module not found" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "unexpected token"; then
    root_cause="js_parse_error"
    certainty="high"
    suggested_action="rollback"
    details=$(echo "$logs" | grep -i "unexpected token" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "SyntaxError"; then
    root_cause="js_syntax_error"
    certainty="high"
    details=$(echo "$logs" | grep -i "SyntaxError" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "ReferenceError"; then
    root_cause="js_reference_error"
    certainty="high"
    details=$(echo "$logs" | grep -i "ReferenceError" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "TypeError"; then
    root_cause="js_type_error"
    certainty="high"
    details=$(echo "$logs" | grep -i "TypeError" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "heap out of memory\|FATAL ERROR\|Allocation failed"; then
    root_cause="out_of_memory"
    certainty="high"
    details=$(echo "$logs" | grep -i "heap out of memory\|FATAL ERROR\|Allocation failed" | head -3 | tr '\n' '; ')
  
  # Prisma / DB errors
  elif echo "$logs" | grep -qi "PrismaClientInitializationError\|prisma.*connect.*ECONNREFUSED"; then
    root_cause="db_connection_refused"
    certainty="high"
    suggested_action="restart_container"
    details=$(echo "$logs" | grep -i "PrismaClientInitializationError\|ECONNREFUSED" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "prisma.*ENOENT\|prisma.*does not exist"; then
    root_cause="prisma_schema_mismatch"
    certainty="high"
    details=$(echo "$logs" | grep -i "prisma.*ENOENT\|does not exist" | head -3 | tr '\n' '; ')
  
  elif echo "$logs" | grep -qi "Can't reach database server\|getaddrinfo ENOTFOUND.*db\|postgres.*connect"; then
    root_cause="db_unreachable"
    certainty="high"
    suggested_action="restart_container"
    details=$(echo "$logs" | grep -i "ENOTFOUND\|Can't reach database" | head -3 | tr '\n' '; ')
  
  # Network / connectivity
  elif echo "$logs" | grep -qi "ECONNREFUSED\|ECONNRESET\|ETIMEDOUT\|ENOTFOUND"; then
    root_cause="downstream_unreachable"
    certainty="medium"
    suggested_action="restart_container"
    details=$(echo "$logs" | grep -i "ECONNREFUSED\|ECONNRESET\|ETIMEDOUT\|ENOTFOUND" | head -3 | tr '\n' '; ')
  
  # Rate limiting
  elif echo "$logs" | grep -qi "rate limit\|too many requests\|429"; then
    root_cause="rate_limit_exceeded"
    certainty="medium"
    suggested_action="hotfix_rate_limit"
    details=$(echo "$logs" | grep -i "rate limit\|too many" | head -3 | tr '\n' '; ')
  
  # Auth / config
  elif echo "$logs" | grep -qi "jwt expired\|invalid token\|Unauthorized"; then
    root_cause="auth_config_error"
    certainty="medium"
    suggested_action="check_config"
    details=$(echo "$logs" | grep -i "jwt expired\|invalid token\|Unauthorized" | head -3 | tr '\n' '; ')
  
  # 5xx patterns from health endpoint
  elif echo "$logs" | grep -qi "500\|Internal Server Error\|unhandled rejection\|uncaught exception"; then
    root_cause="unhandled_exception"
    certainty="medium"
    details=$(echo "$logs" | grep -i "unhandled\|uncaught\|500\|Internal Server Error" | head -3 | tr '\n' '; ')
  
  # Next.js specific
  elif echo "$logs" | grep -qi "next.*error\|Error:.*page\|render.*error\|application.*error"; then
    root_cause="application_error"
    certainty="medium"
    details=$(echo "$logs" | grep -i "Error:\|render.*error" | head -3 | tr '\n' '; ')
  fi

  # If diagnosis gives strong signal, incorporate it
  case "$diagnosis" in
    oom_killed|sigkill)
      [ "$root_cause" = "unknown" ] && root_cause="container_killed_sigkill"
      suggested_action="rollback"
      ;;
    db_down)
      [ "$root_cause" = "unknown" ] && root_cause="database_disconnected"
      suggested_action="restart_container"
      ;;
    health_down)
      [ "$root_cause" = "unknown" ] && root_cause="container_unresponsive"
      ;;
    gateway_down)
      root_cause="nginx_gateway_down"
      suggested_action="reload_gateway"
      certainty="high"
      ;;
  esac

  # Return structured report as JSON-like lines
  echo "ROOT_CAUSE=$root_cause"
  echo "CERTAINTY=$certainty"
  echo "ACTION=$suggested_action"
  echo "DETAILS=$details"
}

# ── Phase 4: Restore Protocol ────────────────────────────────
# Fast rollback to last known-good state, while capturing
# full diagnostics for permanent fix later.

restore_protocol() {
  local diagnosis="$1"
  local root_cause="$2"
  local action="$3"
  local details="$4"
  local logs="$5"
  local incident_id
  incident_id="incident_$(date +%Y%m%d_%H%M%S)"

  log "🚨 RESTORE PROTOCOL triggered — incident: $incident_id"
  log "   Diagnosis:   $diagnosis"
  log "   Root cause:  $root_cause"
  log "   Action:      $action"
  log "   Details:     $details"

  # ── Step 0: Save full diagnostic context for later fix ─────
  {
    echo "=== INCIDENT REPORT: $incident_id ==="
    echo "Timestamp:   $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
    echo "Container:   $CONTAINER_NAME"
    echo "Diagnosis:   $diagnosis"
    echo "Root cause:  $root_cause"
    echo "Certainty:   $certainty"
    echo "Action:      $action"
    echo "Details:     $details"
    echo ""
    echo "=== Container Logs ==="
    echo "$logs"
    echo ""
    echo "=== Container Inspect ==="
    docker inspect "$CONTAINER_NAME" 2>/dev/null || echo "INSPECT_FAILED"
    echo ""
    echo "=== Docker Events (last 5 min) ==="
    timeout 5 docker events --since "${CHECK_INTERVAL}s" --until "$(date +%s)" \
      --filter "container=$CONTAINER_NAME" \
      --format '{{.Time}} {{.Type}} {{.Action}} {{.Status}}' 2>/dev/null || echo "EVENTS_FAILED"
    echo ""
    echo "=== System Resources ==="
    df -h / | tail -1
    free -m 2>/dev/null || echo "free_unavailable"
    echo "=== END INCIDENT REPORT ==="
  } > "$INCIDENTS_DIR/$incident_id.log"

  log "📝 Incident saved to $INCIDENTS_DIR/$incident_id.log"
  alert "INFO" "Incident $incident_id logged — root cause: $root_cause"

  # ── Step 1: Apply action ───────────────────────────────────
  case "$action" in
    rollback)
      log "🔄 RESTORE: rolling back to $ROLLBACK_IMAGE..."
      if docker images "$ROLLBACK_IMAGE" --format '{{.ID}}' | head -1 | grep -q .; then
        # Remove old container
        docker stop "$CONTAINER_NAME" 2>/dev/null || true
        sleep 2
        docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
        sleep 2

        # Recreate from rollback image
        local network
        network=$(docker inspect "$CONTAINER_NAME" --format '{{.HostConfig.NetworkMode}}' 2>/dev/null || echo "bridge")
        docker run -d \
          --name "$CONTAINER_NAME" \
          --restart unless-stopped \
          -p 4014:3000 \
          -v /var/services/homes/admin/falah-mobile/data:/app/data \
          --network "$network" \
          "$ROLLBACK_IMAGE" >> "$AGENT_LOG" 2>&1

        sleep 15
        if check_http "$HEALTH_URL" > /dev/null 2>&1; then
          log "✅ RESTORE: rollback successful — service restored"
          alert "INFO" "Restore protocol: rolled back to $ROLLBACK_IMAGE (incident: $incident_id)"
          record_restart
          return 0
        else
          log "❌ RESTORE: rollback also failed — escalation needed"
          alert "CRITICAL" "Rollback failed for incident $incident_id — manual intervention"
          return 1
        fi
      else
        log "⚠️  RESTORE: no rollback image found — attempting restart instead"
        docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
        sleep 10
        if check_http "$HEALTH_URL" > /dev/null 2>&1; then
          log "✅ RESTORE: restart successful (fallback)"
          record_restart
          return 0
        fi
        return 1
      fi
      ;;

    restart_container)
      log "🔄 RESTORE: restarting container..."
      docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
      record_restart
      sleep 10
      if check_http "$HEALTH_URL" > /dev/null 2>&1; then
        log "✅ RESTORE: restart successful"
        return 0
      else
        log "⚠️  RESTORE: restart didn't fix — escalating to rollback"
        restore_protocol "$diagnosis" "restart_failed" "rollback" "$details" "$logs"
      fi
      ;;

    reload_gateway)
      log "🔄 RESTORE: reloading nginx gateway..."
      local gw
      gw=$(docker ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
      [ -n "$gw" ] && docker exec "$gw" nginx -s reload >> "$AGENT_LOG" 2>&1
      sleep 3
      check_http "$GATEWAY_URL" > /dev/null && log "✅ RESTORE: gateway restored" || log "⚠️  Gateway still down"
      ;;

    hotfix_rate_limit)
      log "🔄 RESTORE: rate limit issue — applying hotfix"
      # Rate limiter auto-clears stale entries every 5 min (code-level fix)
      # For immediate relief, restart container
      docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
      record_restart
      sleep 10
      ;;

    check_config)
      log "🔄 RESTORE: config issue detected — rolling back to safe state"
      restore_protocol "$diagnosis" "config_error" "rollback" "$details" "$logs"
      ;;

    *)
      log "🔄 RESTORE: unknown action '$action' — attempting restart as safe default"
      docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
      record_restart
      sleep 10
      ;;
  esac
}

# ── Phase 5: Apply Quick Hotfix (when possible) ──────────────

apply_hotfix() {
  local root_cause="$1"
  local logs="$2"

  case "$root_cause" in
    rate_limit_exceeded)
      log "🔧 HOTFIX: rate limit exceeded — restarting to clear state"
      docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
      sleep 5
      log "🔧 HOTFIX: rate limiter now auto-cleans every 5 min (code fix in place)"
      ;;

    db_connection_refused|db_unreachable)
      log "🔧 HOTFIX: DB connection issue — restarting container"
      docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
      sleep 10
      ;;

    *)
      log "🔧 HOTFIX: no quick fix for '$root_cause' — relying on restore protocol"
      return 1
      ;;
  esac
  return 0
}

# ── Periodic Maintenance ────────────────────────────────────

periodic_maintenance() {
  local disk_pct
  disk_pct=$(check_disk)
  if [ "$disk_pct" -gt 90 ]; then
    log "⚠️  Disk at ${disk_pct}% — running prune"
    docker system prune -f >> "$AGENT_LOG" 2>&1
    log "✅ Prune completed"
  fi
}

# ── Main Loop ────────────────────────────────────────────────

log "══════════════════════════════════════════════════════"
log "🚀 SRE Auto-Healing Agent starting"
log "   Container:     $CONTAINER_NAME"
log "   Health URL:    $HEALTH_URL"
log "   Gateway URL:   $GATEWAY_URL"
log "   Interval:      ${CHECK_INTERVAL}s"
log "   Max restarts:  $MAX_RESTARTS/hour"
log "   Incidents:     $INCIDENTS_DIR"
log "══════════════════════════════════════════════════════"

CYCLE=0
HEALTHY_CYCLES=0

while true; do
  # ── Detect ──────────────────────────────────────────────
  DIAGNOSIS=$(detect_fault)

  if [ "$DIAGNOSIS" = "healthy" ]; then
    # Stability counter — reset restart tracking after 5 min healthy
    HEALTHY_CYCLES=$((HEALTHY_CYCLES + 1))
    if [ "$HEALTHY_CYCLES" -ge 10 ]; then
      HEALTHY_CYCLES=0
      echo 0 > "$STATE_FILE" 2>/dev/null || true
    fi
  else
    # ── FAULT DETECTED ────────────────────────────────────
    HEALTHY_CYCLES=0
    log ""
    log "⚠️  FAULT DETECTED: $DIAGNOSIS"

    # Step 1: Fetch logs
    log "📋 Fetching container logs..."
    CONTAINER_LOGS=$(fetch_logs 100)

    # Step 2: Analyze logs against knowledge base
    log "🔍 Analyzing logs for root cause..."
    ANALYSIS=$(analyze_logs "$DIAGNOSIS" "$CONTAINER_LOGS")
    
    # Parse analysis output
    ROOT_CAUSE=$(echo "$ANALYSIS" | grep "^ROOT_CAUSE=" | cut -d= -f2-)
    CERTAINTY=$(echo "$ANALYSIS" | grep "^CERTAINTY=" | cut -d= -f2-)
    ACTION=$(echo "$ANALYSIS" | grep "^ACTION=" | cut -d= -f2-)
    DETAILS=$(echo "$ANALYSIS" | grep "^DETAILS=" | cut -d= -f2-)

    log "   Root cause:  $ROOT_CAUSE  (certainty: $CERTAINTY)"
    log "   Action:      $ACTION"
    [ -n "$DETAILS" ] && log "   Details:     $DETAILS"

    # Step 3: Decision — can we hotfix or need restore protocol?
    restarts=$(count_restarts)

    if [ "$restarts" -ge "$MAX_RESTARTS" ]; then
      # Too many restarts — go straight to restore protocol
      log "⚠️  $restarts restarts in last hour — exceeding limit of $MAX_RESTARTS"
      log "🚨 SKIPPING hotfix attempt — going directly to restore protocol"
      restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"

    elif [ "$ACTION" = "rollback" ]; then
      # Knowledge base says rollback — run restore protocol
      restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "$ACTION" "$DETAILS" "$CONTAINER_LOGS"

    elif [ "$ACTION" = "restart_container" ]; then
      # Try restart first, escalate if fails
      log "🔄 Attempting restart (restart #$restarts)..."
      apply_hotfix "$ROOT_CAUSE" "$CONTAINER_LOGS"
      sleep 10
      if [ "$(detect_fault)" != "healthy" ]; then
        log "⚠️  Restart didn't resolve — escalating to restore protocol"
        restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
      fi

    elif [ "$ACTION" = "reload_gateway" ]; then
      restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "$ACTION" "$DETAILS" "$CONTAINER_LOGS"

    elif [ "$ACTION" = "hotfix_rate_limit" ]; then
      apply_hotfix "$ROOT_CAUSE" "$CONTAINER_LOGS"

    else
      # No specific action — safe default: restart, then rollback
      log "🔄 No specific fix for '$ROOT_CAUSE' — attempting restart"
      docker restart "$CONTAINER_NAME" >> "$AGENT_LOG" 2>&1
      record_restart
      sleep 10
      [ "$(detect_fault)" != "healthy" ] && restore_protocol "$DIAGNOSIS" "$ROOT_CAUSE" "rollback" "$DETAILS" "$CONTAINER_LOGS"
    fi
  fi

  # ── Periodic maintenance (every 10 cycles = 5 min) ─────
  CYCLE=$((CYCLE + 1))
  [ "$CYCLE" -ge 10 ] && { CYCLE=0; periodic_maintenance; }

  sleep "$CHECK_INTERVAL"
done