feat: DNS & domain routing auto-healer — Cloudflare tunnels, DNS, gateway, routing integrity, SSL, Traefik

- dns-router-healer/Dockerfile: Alpine + curl, docker-cli, bash, jq, drill - dns-router-healer/dns-router-heal.sh: 8 health checks in 60s cycle 1. Cloudflare tunnels (primary + fallback) — restart on failure 2. DNS resolution — verify all falah domains → Cloudflare IPs 3. Direct service health — local health endpoint 4. Gateway routing — nginx on port 7878 5. Routing integrity — gateway response matches direct 6. Public endpoint via Cloudflare (every 5 cycles) 7. SSL certificate expiry (every 10 cycles) 8. Traefik detection (every 5 cycles) — standalone or Swarm - docker-compose.dns-healer.yml: sidecar deployment config - .pi/agents/dns-healer.md: pi agent definition
2026-06-28 03:45:15 +08:00
parent 6a7ff5d12f
commit ad48309995
4 changed files with 441 additions and 0 deletions
@@ -0,0 +1,6 @@
+FROM alpine:3.19
+RUN apk add --no-cache curl docker-cli bash jq drill
+WORKDIR /app
+COPY dns-router-heal.sh /app/dns-router-heal.sh
+RUN chmod +x /app/dns-router-heal.sh && mkdir -p /var/log/falah /var/state/falah
+CMD ["/app/dns-router-heal.sh"]
@@ -0,0 +1,354 @@
+#!/bin/bash
+# ============================================================
+# Falah Mobile — DNS & Domain Routing Auto-Healer
+# ============================================================
+# Monitors and fixes:
+#   - Cloudflare tunnels (primary + fallback)
+#   - Traefik (if present — Swarm or standalone)
+#   - DNS resolution for all falah domains
+#   - nginx gateway routing to microservices
+#   - SSL/TLS certificate validity
+#   - End-to-end public URL health
+# ============================================================
+
+# ── Configuration ────────────────────────────────────────────
+
+# Domains to monitor
+DOMAINS="${DOMAINS:-falahos.my api.falahos.my git.falahos.my app.falahos.my}"
+
+# Internal URLs to probe
+DIRECT_HEALTH="${DIRECT_HEALTH:-http://localhost:4014/mobile/api/health}"
+GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}"
+TUNNEL_URL="${TUNNEL_URL:-https://falahos.my/mobile/api/health}"
+
+# Tunnel container names
+TUNNEL_PRIMARY="${TUNNEL_PRIMARY:-cloudflare-YT01}"
+TUNNEL_FALLBACK="${TUNNEL_FALLBACK:-falah-tunnel}"
+
+# Gateway container
+GATEWAY_CONTAINER="${GATEWAY_CONTAINER:-falah-umbrel-api-gateway}"
+
+# Traefik (if deployed - may be Swarm service or standalone)
+TRAEFIK_CONTAINER="${TRAEFIK_CONTAINER:-falah_traefik}"
+
+# Expected Cloudflare proxy IPs (anycast)
+EXPECTED_CF_IPS="${EXPECTED_CF_IPS:-104.16.0.0/12 172.64.0.0/13}"
+
+# Synology nginx SSL cert path
+SYNO_CERT="${SYNO_CERT:-/usr/syno/etc/certificate/system/default/fullchain.pem}"
+
+CHECK_INTERVAL="${CHECK_INTERVAL:-60}"   # 1 min for DNS
+AGENT_LOG="/var/log/falah/dns-healer.log"
+STATE_DIR="/var/state/falah"
+mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")"
+
+# ── Logging ──────────────────────────────────────────────────
+
+log()  { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [dns] $*" | tee -a "$AGENT_LOG"; }
+alert() { local s="$1" m="$2"; log "[$s] $m"; }
+
+# ── Helper: run docker from inside or outside ───────────────
+
+docker_cmd() {
+  # Works whether running inside container (has docker socket) or on host
+  docker "$@" 2>/dev/null || /var/packages/Docker/target/usr/bin/docker "$@" 2>/dev/null || return 1
+}
+
+# ── 1. CLOUDFLARE TUNNEL HEALTH ─────────────────────────────
+
+check_tunnel() {
+  local name="$1"
+  local status
+  
+  status=$(docker_cmd ps --filter "name=$name" --format '{{.Status}}' 2>/dev/null)
+  if echo "$status" | grep -q "^Up"; then
+    log "  ✅ Tunnel $name: $status"
+    return 0
+  else
+    log "  ❌ Tunnel $name: DOWN ($status)"
+    return 1
+  fi
+}
+
+fix_tunnel() {
+  local name="$1"
+  log "🔄 Restarting tunnel $name..."
+  docker_cmd restart "$name" >> "$AGENT_LOG" 2>&1 && {
+    sleep 5
+    if check_tunnel "$name"; then
+      alert "INFO" "Auto-healed: restarted tunnel $name"
+      return 0
+    fi
+  }
+  alert "CRITICAL" "Failed to restart tunnel $name"
+  return 1
+}
+
+# ── 2. DNS RESOLUTION ───────────────────────────────────────
+
+check_dns() {
+  local all_ok=0
+  
+  for domain in $DOMAINS; do
+    local ips
+    ips=$(drill "$domain" 2>/dev/null | grep -A1 "ANSWER SECTION" | tail -1 | awk '{print $5}' || \
+          dig +short "$domain" 2>/dev/null || \
+          nslookup "$domain" 2>/dev/null | grep "Address:" | tail -1 | awk '{print $2}')
+    
+    if [ -z "$ips" ]; then
+      log "  ❌ DNS: $domain → UNRESOLVABLE"
+      all_ok=1
+    elif echo "$ips" | grep -qE "^104\.|^172\.|^162\.|^198\."; then
+      log "  ✅ DNS: $domain → $ips (Cloudflare)"
+    else
+      log "  ⚠️  DNS: $domain → $ips (non-Cloudflare IP — may be direct)"
+    fi
+  done
+  
+  return $all_ok
+}
+
+# ── 3. PUBLIC END-TO-END (via Cloudflare tunnel) ───────────
+
+check_public_endpoint() {
+  local code
+  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 15 "$TUNNEL_URL" 2>/dev/null || echo "000")
+  
+  case "$code" in
+    200|301|302|401|403)
+      log "  ✅ Public endpoint: $TUNNEL_URL → HTTP $code"
+      return 0
+      ;;
+    521|522|523|524|525|526|527)
+      log "  ❌ Public endpoint: $TUNNEL_URL → Cloudflare origin error $code"
+      return 2
+      ;;
+    502|503|504)
+      log "  ❌ Public endpoint: $TUNNEL_URL → upstream error $code"
+      return 3
+      ;;
+    000)
+      log "  ❌ Public endpoint: $TUNNEL_URL → UNREACHABLE (DNS or network)"
+      return 1
+      ;;
+    *)
+      log "  ⚠️  Public endpoint: $TUNNEL_URL → HTTP $code"
+      return 4
+      ;;
+  esac
+}
+
+# ── 4. GATEWAY HEALTH ──────────────────────────────────────
+
+check_gateway() {
+  local code body
+  
+  # Check container
+  local status
+  status=$(docker_cmd ps --filter "name=$GATEWAY_CONTAINER" --format '{{.Status}}' 2>/dev/null)
+  if ! echo "$status" | grep -q "^Up"; then
+    log "  ❌ Gateway container: DOWN ($status)"
+    return 1
+  fi
+  
+  # Check nginx inside
+  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$GATEWAY_URL" 2>/dev/null || echo "000")
+  if [ "$code" = "200" ]; then
+    log "  ✅ Gateway: $GATEWAY_URL → HTTP $code"
+    return 0
+  else
+    log "  ❌ Gateway: $GATEWAY_URL → HTTP $code"
+    return 2
+  fi
+}
+
+fix_gateway() {
+  log "🔄 Reloading nginx in gateway..."
+  docker_cmd exec "$GATEWAY_CONTAINER" nginx -s reload >> "$AGENT_LOG" 2>&1
+  sleep 3
+  
+  # If reload failed, try restarting container
+  local code
+  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$GATEWAY_URL" 2>/dev/null || echo "000")
+  if [ "$code" != "200" ]; then
+    log "🔄 Gateway reload didn't fix — restarting container..."
+    docker_cmd restart "$GATEWAY_CONTAINER" >> "$AGENT_LOG" 2>&1
+    sleep 5
+  fi
+  
+  if check_gateway; then
+    alert "INFO" "Auto-healed: restored gateway routing"
+    return 0
+  fi
+  return 1
+}
+
+# ── 5. TRAEFIK HEALTH (if deployed) ─────────────────────────
+
+check_traefik() {
+  # Traefik may be running as standalone container or Swarm service
+  local status
+  
+  # Check standalone
+  status=$(docker_cmd ps --filter "name=$TRAEFIK_CONTAINER" --format '{{.Status}}' 2>/dev/null)
+  if echo "$status" | grep -q "^Up"; then
+    log "  ✅ Traefik (standalone): $status"
+    # Check Traefik API
+    local traefik_health
+    traefik_health=$(curl -sf --max-time 5 http://localhost:8080/api/version 2>/dev/null || echo "")
+    if [ -n "$traefik_health" ]; then
+      log "  ✅ Traefik API: responsive"
+    fi
+    return 0
+  fi
+  
+  # Check Swarm service (if still in swarm)
+  local svc_status
+  svc_status=$(docker_cmd service ps "$TRAEFIK_CONTAINER" --format '{{.CurrentState}}' 2>/dev/null | head -1)
+  if [ -n "$svc_status" ]; then
+    log "  ⚠️  Traefik (Swarm): $svc_status"
+    return 1
+  fi
+  
+  log "  ⚪ Traefik: not deployed (skipped)"
+  return 0
+}
+
+# ── 6. SSL CERTIFICATE CHECK ───────────────────────────────
+
+check_cert() {
+  # Check Synology cert expiry
+  if [ -f "$SYNO_CERT" ]; then
+    local expiry
+    expiry=$(openssl x509 -enddate -noout -in "$SYNO_CERT" 2>/dev/null | cut -d= -f2)
+    local expiry_epoch
+    expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null || echo 0)
+    local now
+    now=$(date +%s)
+    local days_left=$(( (expiry_epoch - now) / 86400 ))
+    
+    if [ "$days_left" -lt 0 ]; then
+      log "  ❌ SSL Cert: EXPIRED ($days_left days ago)"
+      return 2
+    elif [ "$days_left" -lt 7 ]; then
+      log "  ⚠️  SSL Cert: expires in $days_left days"
+      return 1
+    else
+      log "  ✅ SSL Cert: valid for $days_left more days"
+      return 0
+    fi
+  else
+    log "  ⚪ SSL Cert: no local cert found"
+    return 0
+  fi
+}
+
+# ── 7. DIRECT SERVICE HEALTH ────────────────────────────────
+
+check_direct() {
+  local code
+  code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$DIRECT_HEALTH" 2>/dev/null || echo "000")
+  if [ "$code" = "200" ]; then
+    log "  ✅ Direct: $DIRECT_HEALTH → HTTP $code"
+    return 0
+  else
+    log "  ❌ Direct: $DIRECT_HEALTH → HTTP $code"
+    return 1
+  fi
+}
+
+# ── 8. ROUTING INTEGRITY ────────────────────────────────────
+
+check_routing_integrity() {
+  # Verify the gateway actually proxies to the mobile service
+  local gateway_body direct_body
+  
+  gateway_body=$(curl -sf --max-time 5 "$GATEWAY_URL" 2>/dev/null)
+  direct_body=$(curl -sf --max-time 5 "$DIRECT_HEALTH" 2>/dev/null)
+  
+  if [ "$gateway_body" = "$direct_body" ] || echo "$gateway_body" | grep -q '"status":"ok"'; then
+    log "  ✅ Routing: gateway → direct response matches"
+    return 0
+  else
+    log "  ❌ Routing: gateway response differs from direct!"
+    log "     Gateway: $(echo $gateway_body | head -c 100)"
+    log "     Direct:  $(echo $direct_body | head -c 100)"
+    return 1
+  fi
+}
+
+# ── Main Loop ────────────────────────────────────────────────
+
+log ""
+log "══════════════════════════════════════════════════════"
+log "🌐 DNS & Domain Routing Auto-Healer starting"
+log "   Domains:         $DOMAINS"
+log "   Tunnels:         $TUNNEL_PRIMARY, $TUNNEL_FALLBACK"
+log "   Gateway:         $GATEWAY_CONTAINER"
+log "   Traefik:         $TRAEFIK_CONTAINER (optional)"
+log "   Public URL:      $TUNNEL_URL"
+log "   Interval:        ${CHECK_INTERVAL}s"
+log "══════════════════════════════════════════════════════"
+
+CYCLE=0
+
+while true; do
+  CYCLE=$((CYCLE + 1))
+  log ""
+  log "── Cycle $CYCLE ──────────────────────────────────────"
+  
+  # ── 1. Check tunnels ─────────────────────────────────────
+  log "1) Cloudflare Tunnels:"
+  if ! check_tunnel "$TUNNEL_PRIMARY"; then
+    fix_tunnel "$TUNNEL_PRIMARY"
+    # If primary failed, check fallback
+    if ! check_tunnel "$TUNNEL_FALLBACK"; then
+      fix_tunnel "$TUNNEL_FALLBACK"
+    fi
+  fi
+  
+  # ── 2. Check DNS ─────────────────────────────────────────
+  log "2) DNS Resolution:"
+  check_dns
+  
+  # ── 3. Check direct service ──────────────────────────────
+  log "3) Direct Service:"
+  check_direct
+  
+  # ── 4. Check gateway ─────────────────────────────────────
+  log "4) Gateway Routing:"
+  if ! check_gateway; then
+    fix_gateway
+  fi
+  
+  # ── 5. Check routing integrity ────────────────────────────
+  log "5) Routing Integrity:"
+  check_routing_integrity
+  
+  # ── 6. Check public endpoint (every 5th cycle) ───────────
+  log "6) Public Endpoint (via Cloudflare):"
+  if [ $((CYCLE % 5)) -eq 0 ]; then
+    check_public_endpoint
+  else
+    log "   (skipped — runs every 5 cycles)"
+  fi
+  
+  # ── 7. Check SSL certs (every 10th cycle) ────────────────
+  log "7) SSL Certificates:"
+  if [ $((CYCLE % 10)) -eq 0 ]; then
+    check_cert
+  else
+    log "   (skipped — runs every 10 cycles)"
+  fi
+  
+  # ── 8. Check Traefik (every 5th cycle) ───────────────────
+  log "8) Traefik:"
+  if [ $((CYCLE % 5)) -eq 0 ]; then
+    check_traefik
+  else
+    log "   (skipped — runs every 5 cycles)"
+  fi
+  
+  log "── Cycle $CYCLE complete ─────────────────────────────"
+  sleep "$CHECK_INTERVAL"
+done
@@ -0,0 +1,29 @@
+version: "3.8"
+services:
+  falah-dns-healer:
+    build:
+      context: ./dns-router-healer
+      dockerfile: Dockerfile
+    image: falah-dns-healer:latest
+    container_name: falah-dns-healer
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /var/log/falah:/var/log/falah
+      - /var/state/falah:/var/state/falah
+    environment:
+      - DOMAINS=falahos.my api.falahos.my git.falahos.my app.falahos.my
+      - DIRECT_HEALTH=http://localhost:4014/mobile/api/health
+      - GATEWAY_URL=http://localhost:7878/mobile/api/health
+      - TUNNEL_URL=https://falahos.my/mobile/api/health
+      - TUNNEL_PRIMARY=cloudflare-YT01
+      - TUNNEL_FALLBACK=falah-tunnel
+      - GATEWAY_CONTAINER=falah-umbrel-api-gateway
+      - TRAEFIK_CONTAINER=falah_traefik
+      - CHECK_INTERVAL=60
+    network_mode: "host"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"