fix: restore 3-node Docker Swarm + swarm health monitoring in DNS healer

- Rejoined Synology to Contabo-led swarm as manager (reachable)
- Swarm now has 2 managers: Synology (DS3617xs) + Contabo (vmi3361598)
- Saved join token to /var/state/falah/swarm-token for auto-rejoin
- DNS healer v3: checks swarm membership every 2 cycles (~2 min)
  - If swarm inactive, auto-rejoins using saved token
  - Reports node status (Contabo present, Mac Mini offline)
- Mac Mini (colima) was permanently removed from swarm while offline
  (needs manual re-join when back online with same token)

Current swarm:
  DS3617xs (Synology)  → Manager, Reachable, Engine 20.10.3
  vmi3361598 (Contabo)  → Manager, Leader, Engine 29.5.3
  colima (Mac Mini)     → GONE (offline >20d, need re-join)
This commit is contained in:
2026-06-28 04:26:30 +08:00
parent 59f892e73b
commit 33d85a56d2
+57 -2
View File
@@ -185,6 +185,53 @@ fix_gateway() {
# ── 5. TRAEFIK HEALTH (if deployed) ─────────────────────────
check_swarm() {
local token_file="/var/state/falah/swarm-token"
local manager_file="/var/state/falah/swarm-manager"
# Check if we're in a swarm
local swarm_status
swarm_status=$(docker info 2>/dev/null | grep "Swarm:" | awk '{print $2}')
if [ "$swarm_status" != "active" ]; then
log " ❌ Swarm: INACTIVE — attempting rejoin..."
if [ -f "$token_file" ] && [ -f "$manager_file" ]; then
local token manager
token=$(cat "$token_file")
manager=$(cat "$manager_file")
docker swarm join --token "$token" "$manager" >> "$AGENT_LOG" 2>&1
sleep 3
swarm_status=$(docker info 2>/dev/null | grep "Swarm:" | awk '{print $2}')
if [ "$swarm_status" = "active" ]; then
log " ✅ Swarm: rejoined successfully"
alert "INFO" "Auto-healed: rejoined Docker Swarm"
else
log " ❌ Swarm: rejoin failed"
fi
else
log " ❌ Swarm: no join token found at $token_file"
fi
return
fi
# Check this node's role
local is_manager
is_manager=$(docker info 2>/dev/null | grep "Is Manager" | awk '{print $3}')
if [ "$is_manager" = "true" ]; then
log " ✅ Swarm: active (manager)"
# Count nodes
local node_count
node_count=$(docker node ls 2>/dev/null | tail -n +2 | wc -l)
log " 📊 Swarm nodes: $node_count"
# Check if we can see Contabo
docker node ls 2>/dev/null | grep -q "vmi3361598" && log " ✅ Contabo: in swarm" || log " ⚠️ Contabo: missing from swarm"
# Check if Mac Mini is back
docker node ls 2>/dev/null | grep -q "colima" && log " ✅ Mac Mini: in swarm" || log " ⚪ Mac Mini: not in swarm (offline)"
else
log " ✅ Swarm: active (worker)"
fi
}
check_traefik() {
# Traefik may be running as standalone container or Swarm service
local status
@@ -341,8 +388,16 @@ while true; do
log " (skipped — runs every 10 cycles)"
fi
# ── 8. Check Traefik (every 5th cycle) ───────────────────
log "8) Traefik:"
# ── 8. Swarm membership check (every 2nd cycle) ──────────
log "8) Docker Swarm:"
if [ $((CYCLE % 2)) -eq 0 ]; then
check_swarm
else
log " (skipped — runs every 2 cycles)"
fi
# ── 9. Check Traefik (every 5th cycle) ───────────────────
log "9) Traefik:"
if [ $((CYCLE % 5)) -eq 0 ]; then
check_traefik
else