fix: restore 3-node Docker Swarm + swarm health monitoring in DNS healer
- Rejoined Synology to Contabo-led swarm as manager (reachable) - Swarm now has 2 managers: Synology (DS3617xs) + Contabo (vmi3361598) - Saved join token to /var/state/falah/swarm-token for auto-rejoin - DNS healer v3: checks swarm membership every 2 cycles (~2 min) - If swarm inactive, auto-rejoins using saved token - Reports node status (Contabo present, Mac Mini offline) - Mac Mini (colima) was permanently removed from swarm while offline (needs manual re-join when back online with same token) Current swarm: DS3617xs (Synology) → Manager, Reachable, Engine 20.10.3 vmi3361598 (Contabo) → Manager, Leader, Engine 29.5.3 colima (Mac Mini) → GONE (offline >20d, need re-join)
This commit is contained in:
@@ -185,6 +185,53 @@ fix_gateway() {
|
|||||||
|
|
||||||
# ── 5. TRAEFIK HEALTH (if deployed) ─────────────────────────
|
# ── 5. TRAEFIK HEALTH (if deployed) ─────────────────────────
|
||||||
|
|
||||||
|
check_swarm() {
|
||||||
|
local token_file="/var/state/falah/swarm-token"
|
||||||
|
local manager_file="/var/state/falah/swarm-manager"
|
||||||
|
|
||||||
|
# Check if we're in a swarm
|
||||||
|
local swarm_status
|
||||||
|
swarm_status=$(docker info 2>/dev/null | grep "Swarm:" | awk '{print $2}')
|
||||||
|
|
||||||
|
if [ "$swarm_status" != "active" ]; then
|
||||||
|
log " ❌ Swarm: INACTIVE — attempting rejoin..."
|
||||||
|
if [ -f "$token_file" ] && [ -f "$manager_file" ]; then
|
||||||
|
local token manager
|
||||||
|
token=$(cat "$token_file")
|
||||||
|
manager=$(cat "$manager_file")
|
||||||
|
docker swarm join --token "$token" "$manager" >> "$AGENT_LOG" 2>&1
|
||||||
|
sleep 3
|
||||||
|
swarm_status=$(docker info 2>/dev/null | grep "Swarm:" | awk '{print $2}')
|
||||||
|
if [ "$swarm_status" = "active" ]; then
|
||||||
|
log " ✅ Swarm: rejoined successfully"
|
||||||
|
alert "INFO" "Auto-healed: rejoined Docker Swarm"
|
||||||
|
else
|
||||||
|
log " ❌ Swarm: rejoin failed"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log " ❌ Swarm: no join token found at $token_file"
|
||||||
|
fi
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check this node's role
|
||||||
|
local is_manager
|
||||||
|
is_manager=$(docker info 2>/dev/null | grep "Is Manager" | awk '{print $3}')
|
||||||
|
if [ "$is_manager" = "true" ]; then
|
||||||
|
log " ✅ Swarm: active (manager)"
|
||||||
|
# Count nodes
|
||||||
|
local node_count
|
||||||
|
node_count=$(docker node ls 2>/dev/null | tail -n +2 | wc -l)
|
||||||
|
log " 📊 Swarm nodes: $node_count"
|
||||||
|
# Check if we can see Contabo
|
||||||
|
docker node ls 2>/dev/null | grep -q "vmi3361598" && log " ✅ Contabo: in swarm" || log " ⚠️ Contabo: missing from swarm"
|
||||||
|
# Check if Mac Mini is back
|
||||||
|
docker node ls 2>/dev/null | grep -q "colima" && log " ✅ Mac Mini: in swarm" || log " ⚪ Mac Mini: not in swarm (offline)"
|
||||||
|
else
|
||||||
|
log " ✅ Swarm: active (worker)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
check_traefik() {
|
check_traefik() {
|
||||||
# Traefik may be running as standalone container or Swarm service
|
# Traefik may be running as standalone container or Swarm service
|
||||||
local status
|
local status
|
||||||
@@ -341,8 +388,16 @@ while true; do
|
|||||||
log " (skipped — runs every 10 cycles)"
|
log " (skipped — runs every 10 cycles)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── 8. Check Traefik (every 5th cycle) ───────────────────
|
# ── 8. Swarm membership check (every 2nd cycle) ──────────
|
||||||
log "8) Traefik:"
|
log "8) Docker Swarm:"
|
||||||
|
if [ $((CYCLE % 2)) -eq 0 ]; then
|
||||||
|
check_swarm
|
||||||
|
else
|
||||||
|
log " (skipped — runs every 2 cycles)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 9. Check Traefik (every 5th cycle) ───────────────────
|
||||||
|
log "9) Traefik:"
|
||||||
if [ $((CYCLE % 5)) -eq 0 ]; then
|
if [ $((CYCLE % 5)) -eq 0 ]; then
|
||||||
check_traefik
|
check_traefik
|
||||||
else
|
else
|
||||||
|
|||||||
Reference in New Issue
Block a user