diff --git a/.pi/agents/dns-healer.md b/.pi/agents/dns-healer.md new file mode 100644 index 0000000..3b74461 --- /dev/null +++ b/.pi/agents/dns-healer.md @@ -0,0 +1,52 @@ +--- +name: dns-healer +description: DNS & domain routing auto-healer — Cloudflare tunnels, Traefik, nginx gateway, SSL certs, DNS resolution +tools: read, bash +--- + +# DNS & Domain Routing Auto-Healer + +Monitors and auto-fixes the full routing chain: + +``` +User → Cloudflare DNS → Cloudflare Tunnel → Gateway nginx → Microservice +``` + +## What It Checks (every 60s) + +| Check | What | Auto-Fix | +|---|---|---| +| Cloudflare Tunnel | Container status | Restart tunnel container | +| DNS Resolution | Domains resolve to Cloudflare IPs | Alert only (needs API token) | +| Direct Service | Local health endpoint | Already handled by app auto-healer | +| Gateway Routing | nginx proxies correctly | Reload nginx, restart container | +| Routing Integrity | Gateway response matches direct | Alert on mismatch | +| Public Endpoint | URL accessible via Cloudflare | Alert on origin errors | +| SSL Certs | Certificate expiration | Alert if <7 days | +| Traefik | If deployed (standalone or Swarm) | Restart container | + +## Common Fixes + +```bash +# Restart tunnel +docker restart cloudflare-YT01 + +# Reload nginx gateway +docker exec falah-umbrel-api-gateway nginx -s reload + +# Restart gateway +docker restart falah-umbrel-api-gateway + +# Verify tunnel config from Cloudflare dashboard +# Tunnel token: eyJhIjoiZjM1ZGI5M2YwMTkwZjIwMzA3NzM4OTEzNjE2OWNjMGI... + +# Check tunnel ingress rules +# Cloudflare Dashboard → Zero Trust → Tunnels → YT01 +``` + +## Escalation + +If tunnel fails to restart 3 times in an hour, or public endpoint returns 521/522 for 5+ minutes: +1. Log full diagnostics +2. Do NOT keep restarting (rate limit: 3 restarts/hour) +3. Check Cloudflare dashboard for origin status diff --git a/dns-router-healer/Dockerfile b/dns-router-healer/Dockerfile new file mode 100644 index 0000000..0f45aca --- /dev/null +++ b/dns-router-healer/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:3.19 +RUN apk add --no-cache curl docker-cli bash jq drill +WORKDIR /app +COPY dns-router-heal.sh /app/dns-router-heal.sh +RUN chmod +x /app/dns-router-heal.sh && mkdir -p /var/log/falah /var/state/falah +CMD ["/app/dns-router-heal.sh"] diff --git a/dns-router-healer/dns-router-heal.sh b/dns-router-healer/dns-router-heal.sh new file mode 100644 index 0000000..1b68324 --- /dev/null +++ b/dns-router-healer/dns-router-heal.sh @@ -0,0 +1,354 @@ +#!/bin/bash +# ============================================================ +# Falah Mobile — DNS & Domain Routing Auto-Healer +# ============================================================ +# Monitors and fixes: +# - Cloudflare tunnels (primary + fallback) +# - Traefik (if present — Swarm or standalone) +# - DNS resolution for all falah domains +# - nginx gateway routing to microservices +# - SSL/TLS certificate validity +# - End-to-end public URL health +# ============================================================ + +# ── Configuration ──────────────────────────────────────────── + +# Domains to monitor +DOMAINS="${DOMAINS:-falahos.my api.falahos.my git.falahos.my app.falahos.my}" + +# Internal URLs to probe +DIRECT_HEALTH="${DIRECT_HEALTH:-http://localhost:4014/mobile/api/health}" +GATEWAY_URL="${GATEWAY_URL:-http://localhost:7878/mobile/api/health}" +TUNNEL_URL="${TUNNEL_URL:-https://falahos.my/mobile/api/health}" + +# Tunnel container names +TUNNEL_PRIMARY="${TUNNEL_PRIMARY:-cloudflare-YT01}" +TUNNEL_FALLBACK="${TUNNEL_FALLBACK:-falah-tunnel}" + +# Gateway container +GATEWAY_CONTAINER="${GATEWAY_CONTAINER:-falah-umbrel-api-gateway}" + +# Traefik (if deployed - may be Swarm service or standalone) +TRAEFIK_CONTAINER="${TRAEFIK_CONTAINER:-falah_traefik}" + +# Expected Cloudflare proxy IPs (anycast) +EXPECTED_CF_IPS="${EXPECTED_CF_IPS:-104.16.0.0/12 172.64.0.0/13}" + +# Synology nginx SSL cert path +SYNO_CERT="${SYNO_CERT:-/usr/syno/etc/certificate/system/default/fullchain.pem}" + +CHECK_INTERVAL="${CHECK_INTERVAL:-60}" # 1 min for DNS +AGENT_LOG="/var/log/falah/dns-healer.log" +STATE_DIR="/var/state/falah" +mkdir -p "$STATE_DIR" "$(dirname "$AGENT_LOG")" + +# ── Logging ────────────────────────────────────────────────── + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [dns] $*" | tee -a "$AGENT_LOG"; } +alert() { local s="$1" m="$2"; log "[$s] $m"; } + +# ── Helper: run docker from inside or outside ─────────────── + +docker_cmd() { + # Works whether running inside container (has docker socket) or on host + docker "$@" 2>/dev/null || /var/packages/Docker/target/usr/bin/docker "$@" 2>/dev/null || return 1 +} + +# ── 1. CLOUDFLARE TUNNEL HEALTH ───────────────────────────── + +check_tunnel() { + local name="$1" + local status + + status=$(docker_cmd ps --filter "name=$name" --format '{{.Status}}' 2>/dev/null) + if echo "$status" | grep -q "^Up"; then + log " ✅ Tunnel $name: $status" + return 0 + else + log " ❌ Tunnel $name: DOWN ($status)" + return 1 + fi +} + +fix_tunnel() { + local name="$1" + log "🔄 Restarting tunnel $name..." + docker_cmd restart "$name" >> "$AGENT_LOG" 2>&1 && { + sleep 5 + if check_tunnel "$name"; then + alert "INFO" "Auto-healed: restarted tunnel $name" + return 0 + fi + } + alert "CRITICAL" "Failed to restart tunnel $name" + return 1 +} + +# ── 2. DNS RESOLUTION ─────────────────────────────────────── + +check_dns() { + local all_ok=0 + + for domain in $DOMAINS; do + local ips + ips=$(drill "$domain" 2>/dev/null | grep -A1 "ANSWER SECTION" | tail -1 | awk '{print $5}' || \ + dig +short "$domain" 2>/dev/null || \ + nslookup "$domain" 2>/dev/null | grep "Address:" | tail -1 | awk '{print $2}') + + if [ -z "$ips" ]; then + log " ❌ DNS: $domain → UNRESOLVABLE" + all_ok=1 + elif echo "$ips" | grep -qE "^104\.|^172\.|^162\.|^198\."; then + log " ✅ DNS: $domain → $ips (Cloudflare)" + else + log " ⚠️ DNS: $domain → $ips (non-Cloudflare IP — may be direct)" + fi + done + + return $all_ok +} + +# ── 3. PUBLIC END-TO-END (via Cloudflare tunnel) ─────────── + +check_public_endpoint() { + local code + code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 15 "$TUNNEL_URL" 2>/dev/null || echo "000") + + case "$code" in + 200|301|302|401|403) + log " ✅ Public endpoint: $TUNNEL_URL → HTTP $code" + return 0 + ;; + 521|522|523|524|525|526|527) + log " ❌ Public endpoint: $TUNNEL_URL → Cloudflare origin error $code" + return 2 + ;; + 502|503|504) + log " ❌ Public endpoint: $TUNNEL_URL → upstream error $code" + return 3 + ;; + 000) + log " ❌ Public endpoint: $TUNNEL_URL → UNREACHABLE (DNS or network)" + return 1 + ;; + *) + log " ⚠️ Public endpoint: $TUNNEL_URL → HTTP $code" + return 4 + ;; + esac +} + +# ── 4. GATEWAY HEALTH ────────────────────────────────────── + +check_gateway() { + local code body + + # Check container + local status + status=$(docker_cmd ps --filter "name=$GATEWAY_CONTAINER" --format '{{.Status}}' 2>/dev/null) + if ! echo "$status" | grep -q "^Up"; then + log " ❌ Gateway container: DOWN ($status)" + return 1 + fi + + # Check nginx inside + code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$GATEWAY_URL" 2>/dev/null || echo "000") + if [ "$code" = "200" ]; then + log " ✅ Gateway: $GATEWAY_URL → HTTP $code" + return 0 + else + log " ❌ Gateway: $GATEWAY_URL → HTTP $code" + return 2 + fi +} + +fix_gateway() { + log "🔄 Reloading nginx in gateway..." + docker_cmd exec "$GATEWAY_CONTAINER" nginx -s reload >> "$AGENT_LOG" 2>&1 + sleep 3 + + # If reload failed, try restarting container + local code + code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$GATEWAY_URL" 2>/dev/null || echo "000") + if [ "$code" != "200" ]; then + log "🔄 Gateway reload didn't fix — restarting container..." + docker_cmd restart "$GATEWAY_CONTAINER" >> "$AGENT_LOG" 2>&1 + sleep 5 + fi + + if check_gateway; then + alert "INFO" "Auto-healed: restored gateway routing" + return 0 + fi + return 1 +} + +# ── 5. TRAEFIK HEALTH (if deployed) ───────────────────────── + +check_traefik() { + # Traefik may be running as standalone container or Swarm service + local status + + # Check standalone + status=$(docker_cmd ps --filter "name=$TRAEFIK_CONTAINER" --format '{{.Status}}' 2>/dev/null) + if echo "$status" | grep -q "^Up"; then + log " ✅ Traefik (standalone): $status" + # Check Traefik API + local traefik_health + traefik_health=$(curl -sf --max-time 5 http://localhost:8080/api/version 2>/dev/null || echo "") + if [ -n "$traefik_health" ]; then + log " ✅ Traefik API: responsive" + fi + return 0 + fi + + # Check Swarm service (if still in swarm) + local svc_status + svc_status=$(docker_cmd service ps "$TRAEFIK_CONTAINER" --format '{{.CurrentState}}' 2>/dev/null | head -1) + if [ -n "$svc_status" ]; then + log " ⚠️ Traefik (Swarm): $svc_status" + return 1 + fi + + log " ⚪ Traefik: not deployed (skipped)" + return 0 +} + +# ── 6. SSL CERTIFICATE CHECK ─────────────────────────────── + +check_cert() { + # Check Synology cert expiry + if [ -f "$SYNO_CERT" ]; then + local expiry + expiry=$(openssl x509 -enddate -noout -in "$SYNO_CERT" 2>/dev/null | cut -d= -f2) + local expiry_epoch + expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null || echo 0) + local now + now=$(date +%s) + local days_left=$(( (expiry_epoch - now) / 86400 )) + + if [ "$days_left" -lt 0 ]; then + log " ❌ SSL Cert: EXPIRED ($days_left days ago)" + return 2 + elif [ "$days_left" -lt 7 ]; then + log " ⚠️ SSL Cert: expires in $days_left days" + return 1 + else + log " ✅ SSL Cert: valid for $days_left more days" + return 0 + fi + else + log " ⚪ SSL Cert: no local cert found" + return 0 + fi +} + +# ── 7. DIRECT SERVICE HEALTH ──────────────────────────────── + +check_direct() { + local code + code=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 5 "$DIRECT_HEALTH" 2>/dev/null || echo "000") + if [ "$code" = "200" ]; then + log " ✅ Direct: $DIRECT_HEALTH → HTTP $code" + return 0 + else + log " ❌ Direct: $DIRECT_HEALTH → HTTP $code" + return 1 + fi +} + +# ── 8. ROUTING INTEGRITY ──────────────────────────────────── + +check_routing_integrity() { + # Verify the gateway actually proxies to the mobile service + local gateway_body direct_body + + gateway_body=$(curl -sf --max-time 5 "$GATEWAY_URL" 2>/dev/null) + direct_body=$(curl -sf --max-time 5 "$DIRECT_HEALTH" 2>/dev/null) + + if [ "$gateway_body" = "$direct_body" ] || echo "$gateway_body" | grep -q '"status":"ok"'; then + log " ✅ Routing: gateway → direct response matches" + return 0 + else + log " ❌ Routing: gateway response differs from direct!" + log " Gateway: $(echo $gateway_body | head -c 100)" + log " Direct: $(echo $direct_body | head -c 100)" + return 1 + fi +} + +# ── Main Loop ──────────────────────────────────────────────── + +log "" +log "══════════════════════════════════════════════════════" +log "🌐 DNS & Domain Routing Auto-Healer starting" +log " Domains: $DOMAINS" +log " Tunnels: $TUNNEL_PRIMARY, $TUNNEL_FALLBACK" +log " Gateway: $GATEWAY_CONTAINER" +log " Traefik: $TRAEFIK_CONTAINER (optional)" +log " Public URL: $TUNNEL_URL" +log " Interval: ${CHECK_INTERVAL}s" +log "══════════════════════════════════════════════════════" + +CYCLE=0 + +while true; do + CYCLE=$((CYCLE + 1)) + log "" + log "── Cycle $CYCLE ──────────────────────────────────────" + + # ── 1. Check tunnels ───────────────────────────────────── + log "1) Cloudflare Tunnels:" + if ! check_tunnel "$TUNNEL_PRIMARY"; then + fix_tunnel "$TUNNEL_PRIMARY" + # If primary failed, check fallback + if ! check_tunnel "$TUNNEL_FALLBACK"; then + fix_tunnel "$TUNNEL_FALLBACK" + fi + fi + + # ── 2. Check DNS ───────────────────────────────────────── + log "2) DNS Resolution:" + check_dns + + # ── 3. Check direct service ────────────────────────────── + log "3) Direct Service:" + check_direct + + # ── 4. Check gateway ───────────────────────────────────── + log "4) Gateway Routing:" + if ! check_gateway; then + fix_gateway + fi + + # ── 5. Check routing integrity ──────────────────────────── + log "5) Routing Integrity:" + check_routing_integrity + + # ── 6. Check public endpoint (every 5th cycle) ─────────── + log "6) Public Endpoint (via Cloudflare):" + if [ $((CYCLE % 5)) -eq 0 ]; then + check_public_endpoint + else + log " (skipped — runs every 5 cycles)" + fi + + # ── 7. Check SSL certs (every 10th cycle) ──────────────── + log "7) SSL Certificates:" + if [ $((CYCLE % 10)) -eq 0 ]; then + check_cert + else + log " (skipped — runs every 10 cycles)" + fi + + # ── 8. Check Traefik (every 5th cycle) ─────────────────── + log "8) Traefik:" + if [ $((CYCLE % 5)) -eq 0 ]; then + check_traefik + else + log " (skipped — runs every 5 cycles)" + fi + + log "── Cycle $CYCLE complete ─────────────────────────────" + sleep "$CHECK_INTERVAL" +done diff --git a/dns-router-healer/docker-compose.dns-healer.yml b/dns-router-healer/docker-compose.dns-healer.yml new file mode 100644 index 0000000..84abb31 --- /dev/null +++ b/dns-router-healer/docker-compose.dns-healer.yml @@ -0,0 +1,29 @@ +version: "3.8" +services: + falah-dns-healer: + build: + context: ./dns-router-healer + dockerfile: Dockerfile + image: falah-dns-healer:latest + container_name: falah-dns-healer + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log/falah:/var/log/falah + - /var/state/falah:/var/state/falah + environment: + - DOMAINS=falahos.my api.falahos.my git.falahos.my app.falahos.my + - DIRECT_HEALTH=http://localhost:4014/mobile/api/health + - GATEWAY_URL=http://localhost:7878/mobile/api/health + - TUNNEL_URL=https://falahos.my/mobile/api/health + - TUNNEL_PRIMARY=cloudflare-YT01 + - TUNNEL_FALLBACK=falah-tunnel + - GATEWAY_CONTAINER=falah-umbrel-api-gateway + - TRAEFIK_CONTAINER=falah_traefik + - CHECK_INTERVAL=60 + network_mode: "host" + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3"