falah-mobile/dns-router-healer/dns-router-heal.sh

#!/bin/bash
# ============================================================
# Falah Mobile — DNS & Domain Routing Auto-Healer
# ============================================================
# Monitors and fixes:
#   - Cloudflare tunnels (primary + fallback)
#   - Traefik (if present — Swarm or standalone)
#   - DNS resolution for all falah domains
#   - nginx gateway routing to microservices
#   - SSL/TLS certificate validity
#   - End-to-end public URL health
# ============================================================

# ── Configuration ────────────────────────────────────────────

# Domains to monitor
DOMAINS="${DOMAINS:-falahos.my api.falahos.my git.falahos.my app.falahos.my}"

# Internal URLs to probe
DIRECT_HEALTH="${DIRECT_HEALTH:-http://localhost:4014/mobile/api/health}"
GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
TUNNEL_URL="${TUNNEL_URL:-https://falahos.my/mobile/api/health}"

# Tunnel container names
TUNNEL_PRIMARY="${TUNNEL_PRIMARY:-cloudflare-YT01}"
TUNNEL_FALLBACK="${TUNNEL_FALLBACK:-falah-tunnel}"

# Gateway container
GATEWAY_CONTAINER="${GATEWAY_CONTAINER:-falah-umbrel-api-gateway}"

# Traefik (if deployed - may be Swarm service or standalone)
TRAEFIK_CONTAINER="${TRAEFIK_CONTAINER:-falah_traefik}"

# Expected Cloudflare proxy IPs (anycast)
EXPECTED_CF_IPS="${EXPECTED_CF_IPS:-104.16.0.0/12 172.64.0.0/13}"

# Synology nginx SSL cert path
SYNO_CERT="${SYNO_CERT:-/usr/syno/etc/certificate/system/default/fullchain.pem}"

CHECK_INTERVAL="${CHECK_INTERVAL:-60}"   # 1 min for DNS
AGENT_LOG="/var/log/falah/dns-healer.log"
STATE_DIR="/var/state/falah"
mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")"

# ── Logging ──────────────────────────────────────────────────

log()  { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [dns] $*" | tee -a "$AGENT_LOG"; }
alert() { local s="$1" m="$2"; log "[$s] $m"; }

# ── Helper: run docker from inside or outside ───────────────

docker_cmd() {
  # Works whether running inside container (has docker socket) or on host
  docker "$@" 2>/dev/null || /var/packages/Docker/target/usr/bin/docker "$@" 2>/dev/null || return 1
}

# ── 1. CLOUDFLARE TUNNEL HEALTH ─────────────────────────────

check_tunnel() {
  local name="$1"
  local status
  
  status=$(docker_cmd ps --filter "name=$name" --format '{{.Status}}' 2>/dev/null)
  if echo "$status" | grep -q "^Up"; then
    log "  ✅ Tunnel $name: $status"
    return 0
  else
    log "  ❌ Tunnel $name: DOWN ($status)"
    return 1
  fi
}

fix_tunnel() {
  local name="$1"
  log "🔄 Restarting tunnel $name..."
  docker_cmd restart "$name" >> "$AGENT_LOG" 2>&1 && {
    sleep 5
    if check_tunnel "$name"; then
      alert "INFO" "Auto-healed: restarted tunnel $name"
      return 0
    fi
  }
  alert "CRITICAL" "Failed to restart tunnel $name"
  return 1
}

# ── 2. DNS RESOLUTION ───────────────────────────────────────

check_dns() {
  local all_ok=0
  
  for domain in $DOMAINS; do
    local ips
    ips=$(drill "$domain" 2>/dev/null | grep -A1 "ANSWER SECTION" | tail -1 | awk '{print $5}' || \
          dig +short "$domain" 2>/dev/null || \
          nslookup "$domain" 2>/dev/null | grep "Address:" | tail -1 | awk '{print $2}')
    
    if [ -z "$ips" ]; then
      log "  ❌ DNS: $domain → UNRESOLVABLE"
      all_ok=1
    elif echo "$ips" | grep -qE "^104\.|^172\.|^162\.|^198\."; then
      log "  ✅ DNS: $domain → $ips (Cloudflare)"
    else
      log "  ⚠️  DNS: $domain → $ips (non-Cloudflare IP — may be direct)"
    fi
  done
  
  return $all_ok
}

# ── 3. PUBLIC END-TO-END (via Cloudflare tunnel) ───────────

check_public_endpoint() {
  local code
  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 15 "$TUNNEL_URL" 2>/dev/null || echo "000")
  
  case "$code" in
    200|301|302|401|403)
      log "  ✅ Public endpoint: $TUNNEL_URL → HTTP $code"
      return 0
      ;;
    521|522|523|524|525|526|527)
      log "  ❌ Public endpoint: $TUNNEL_URL → Cloudflare origin error $code"
      return 2
      ;;
    502|503|504)
      log "  ❌ Public endpoint: $TUNNEL_URL → upstream error $code"
      return 3
      ;;
    000)
      log "  ❌ Public endpoint: $TUNNEL_URL → UNREACHABLE (DNS or network)"
      return 1
      ;;
    *)
      log "  ⚠️  Public endpoint: $TUNNEL_URL → HTTP $code"
      return 4
      ;;
  esac
}

# ── 4. GATEWAY HEALTH ──────────────────────────────────────

check_gateway() {
  local code body
  
  # Check container
  local status
  status=$(docker_cmd ps --filter "name=$GATEWAY_CONTAINER" --format '{{.Status}}' 2>/dev/null)
  if ! echo "$status" | grep -q "^Up"; then
    log "  ❌ Gateway container: DOWN ($status)"
    return 1
  fi
  
  # Check nginx inside
  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$GATEWAY_URL" 2>/dev/null || echo "000")
  if [ "$code" = "200" ]; then
    log "  ✅ Gateway: $GATEWAY_URL → HTTP $code"
    return 0
  else
    log "  ❌ Gateway: $GATEWAY_URL → HTTP $code"
    return 2
  fi
}

fix_gateway() {
  log "🔄 Reloading nginx in gateway..."
  docker_cmd exec "$GATEWAY_CONTAINER" nginx -s reload >> "$AGENT_LOG" 2>&1
  sleep 3
  
  # If reload failed, try restarting container
  local code
  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$GATEWAY_URL" 2>/dev/null || echo "000")
  if [ "$code" != "200" ]; then
    log "🔄 Gateway reload didn't fix — restarting container..."
    docker_cmd restart "$GATEWAY_CONTAINER" >> "$AGENT_LOG" 2>&1
    sleep 5
  fi
  
  if check_gateway; then
    alert "INFO" "Auto-healed: restored gateway routing"
    return 0
  fi
  return 1
}

# ── 5. TRAEFIK HEALTH (if deployed) ─────────────────────────

check_traefik() {
  # Traefik may be running as standalone container or Swarm service
  local status
  
  # Check standalone
  status=$(docker_cmd ps --filter "name=$TRAEFIK_CONTAINER" --format '{{.Status}}' 2>/dev/null)
  if echo "$status" | grep -q "^Up"; then
    log "  ✅ Traefik (standalone): $status"
    # Check Traefik API
    local traefik_health
    traefik_health=$(curl -sf --max-time 5 http://localhost:8080/api/version 2>/dev/null || echo "")
    if [ -n "$traefik_health" ]; then
      log "  ✅ Traefik API: responsive"
    fi
    return 0
  fi
  
  # Check Swarm service (if still in swarm)
  local svc_status
  svc_status=$(docker_cmd service ps "$TRAEFIK_CONTAINER" --format '{{.CurrentState}}' 2>/dev/null | head -1)
  if [ -n "$svc_status" ]; then
    log "  ⚠️  Traefik (Swarm): $svc_status"
    return 1
  fi
  
  log "  ⚪ Traefik: not deployed (skipped)"
  return 0
}

# ── 6. SSL CERTIFICATE CHECK ───────────────────────────────

check_cert() {
  # Check Synology cert expiry
  if [ -f "$SYNO_CERT" ]; then
    local expiry
    expiry=$(openssl x509 -enddate -noout -in "$SYNO_CERT" 2>/dev/null | cut -d= -f2)
    local expiry_epoch
    expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null || echo 0)
    local now
    now=$(date +%s)
    local days_left=$(( (expiry_epoch - now) / 86400 ))
    
    if [ "$days_left" -lt 0 ]; then
      log "  ❌ SSL Cert: EXPIRED ($days_left days ago)"
      return 2
    elif [ "$days_left" -lt 7 ]; then
      log "  ⚠️  SSL Cert: expires in $days_left days"
      return 1
    else
      log "  ✅ SSL Cert: valid for $days_left more days"
      return 0
    fi
  else
    log "  ⚪ SSL Cert: no local cert found"
    return 0
  fi
}

# ── 7. DIRECT SERVICE HEALTH ────────────────────────────────

check_direct() {
  local code
  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$DIRECT_HEALTH" 2>/dev/null || echo "000")
  if [ "$code" = "200" ]; then
    log "  ✅ Direct: $DIRECT_HEALTH → HTTP $code"
    return 0
  else
    log "  ❌ Direct: $DIRECT_HEALTH → HTTP $code"
    return 1
  fi
}

# ── 8. ROUTING INTEGRITY ────────────────────────────────────

check_routing_integrity() {
  # Verify the gateway actually proxies to the mobile service
  local gateway_body direct_body
  
  gateway_body=$(curl -sf --max-time 5 "$GATEWAY_URL" 2>/dev/null)
  direct_body=$(curl -sf --max-time 5 "$DIRECT_HEALTH" 2>/dev/null)
  
  if [ "$gateway_body" = "$direct_body" ] || echo "$gateway_body" | grep -q '"status":"ok"'; then
    log "  ✅ Routing: gateway → direct response matches"
    return 0
  else
    log "  ❌ Routing: gateway response differs from direct!"
    log "     Gateway: $(echo $gateway_body | head -c 100)"
    log "     Direct:  $(echo $direct_body | head -c 100)"
    return 1
  fi
}

# ── Main Loop ────────────────────────────────────────────────

log ""
log "══════════════════════════════════════════════════════"
log "🌐 DNS & Domain Routing Auto-Healer starting"
log "   Domains:         $DOMAINS"
log "   Tunnels:         $TUNNEL_PRIMARY, $TUNNEL_FALLBACK"
log "   Gateway:         $GATEWAY_CONTAINER"
log "   Traefik:         $TRAEFIK_CONTAINER (optional)"
log "   Public URL:      $TUNNEL_URL"
log "   Interval:        ${CHECK_INTERVAL}s"
log "══════════════════════════════════════════════════════"

CYCLE=0

while true; do
  CYCLE=$((CYCLE + 1))
  log ""
  log "── Cycle $CYCLE ──────────────────────────────────────"
  
  # ── 1. Check tunnels ─────────────────────────────────────
  log "1) Cloudflare Tunnels:"
  if ! check_tunnel "$TUNNEL_PRIMARY"; then
    fix_tunnel "$TUNNEL_PRIMARY"
    # If primary failed, check fallback
    if ! check_tunnel "$TUNNEL_FALLBACK"; then
      fix_tunnel "$TUNNEL_FALLBACK"
    fi
  fi
  
  # ── 2. Check DNS ─────────────────────────────────────────
  log "2) DNS Resolution:"
  check_dns
  
  # ── 3. Check direct service ──────────────────────────────
  log "3) Direct Service:"
  check_direct
  
  # ── 4. Check gateway ─────────────────────────────────────
  log "4) Gateway Routing:"
  if ! check_gateway; then
    fix_gateway
  fi
  
  # ── 5. Check routing integrity ────────────────────────────
  log "5) Routing Integrity:"
  check_routing_integrity
  
  # ── 6. Check public endpoint (every 5th cycle) ───────────
  log "6) Public Endpoint (via Cloudflare):"
  if [ $((CYCLE % 5)) -eq 0 ]; then
    check_public_endpoint
  else
    log "   (skipped — runs every 5 cycles)"
  fi
  
  # ── 7. Check SSL certs (every 10th cycle) ────────────────
  log "7) SSL Certificates:"
  if [ $((CYCLE % 10)) -eq 0 ]; then
    check_cert
  else
    log "   (skipped — runs every 10 cycles)"
  fi
  
  # ── 8. Check Traefik (every 5th cycle) ───────────────────
  log "8) Traefik:"
  if [ $((CYCLE % 5)) -eq 0 ]; then
    check_traefik
  else
    log "   (skipped — runs every 5 cycles)"
  fi
  
  log "── Cycle $CYCLE complete ─────────────────────────────"
  sleep "$CHECK_INTERVAL"
done