diff --git a/dns-router-healer/dns-router-heal.sh b/dns-router-healer/dns-router-heal.sh index ac994d0..e02d025 100644 --- a/dns-router-healer/dns-router-heal.sh +++ b/dns-router-healer/dns-router-heal.sh @@ -185,6 +185,53 @@ fix_gateway() { # ── 5. TRAEFIK HEALTH (if deployed) ───────────────────────── +check_swarm() { + local token_file="/var/state/falah/swarm-token" + local manager_file="/var/state/falah/swarm-manager" + + # Check if we're in a swarm + local swarm_status + swarm_status=$(docker info 2>/dev/null | grep "Swarm:" | awk '{print $2}') + + if [ "$swarm_status" != "active" ]; then + log " ❌ Swarm: INACTIVE — attempting rejoin..." + if [ -f "$token_file" ] && [ -f "$manager_file" ]; then + local token manager + token=$(cat "$token_file") + manager=$(cat "$manager_file") + docker swarm join --token "$token" "$manager" >> "$AGENT_LOG" 2>&1 + sleep 3 + swarm_status=$(docker info 2>/dev/null | grep "Swarm:" | awk '{print $2}') + if [ "$swarm_status" = "active" ]; then + log " ✅ Swarm: rejoined successfully" + alert "INFO" "Auto-healed: rejoined Docker Swarm" + else + log " ❌ Swarm: rejoin failed" + fi + else + log " ❌ Swarm: no join token found at $token_file" + fi + return + fi + + # Check this node's role + local is_manager + is_manager=$(docker info 2>/dev/null | grep "Is Manager" | awk '{print $3}') + if [ "$is_manager" = "true" ]; then + log " ✅ Swarm: active (manager)" + # Count nodes + local node_count + node_count=$(docker node ls 2>/dev/null | tail -n +2 | wc -l) + log " 📊 Swarm nodes: $node_count" + # Check if we can see Contabo + docker node ls 2>/dev/null | grep -q "vmi3361598" && log " ✅ Contabo: in swarm" || log " ⚠️ Contabo: missing from swarm" + # Check if Mac Mini is back + docker node ls 2>/dev/null | grep -q "colima" && log " ✅ Mac Mini: in swarm" || log " ⚪ Mac Mini: not in swarm (offline)" + else + log " ✅ Swarm: active (worker)" + fi +} + check_traefik() { # Traefik may be running as standalone container or Swarm service local status @@ -341,8 +388,16 @@ while true; do log " (skipped — runs every 10 cycles)" fi - # ── 8. Check Traefik (every 5th cycle) ─────────────────── - log "8) Traefik:" + # ── 8. Swarm membership check (every 2nd cycle) ────────── + log "8) Docker Swarm:" + if [ $((CYCLE % 2)) -eq 0 ]; then + check_swarm + else + log " (skipped — runs every 2 cycles)" + fi + + # ── 9. Check Traefik (every 5th cycle) ─────────────────── + log "9) Traefik:" if [ $((CYCLE % 5)) -eq 0 ]; then check_traefik else