feat: auto-healing system — HEALTHCHECK, rate-limiter cleanup, auto-rollback CI, watchdog
- Dockerfile: add HEALTHCHECK (30s interval, 10s timeout, 3 retries) - rate-limit.ts: auto-cleanup stale entries every 5 min (prevents memory leak) - scripts/auto-heal.sh: watchdog for Synology cron — auto-restarts container, reloads nginx gateway, escalates on excessive failures - CI workflow: auto-rollback on deploy failure (tags :rollback image, reverts if health check fails after deploy)
This commit is contained in:
+61
-13
@@ -7,11 +7,12 @@ on:
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: falah-mobile
|
||||
STAGING_HOST: 192.168.0.17
|
||||
STAGING_HOST: 192.168.0.11
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -37,18 +38,65 @@ jobs:
|
||||
docker build -t ${{ env.IMAGE_NAME }}:staging .
|
||||
docker save ${{ env.IMAGE_NAME }}:staging | gzip > /tmp/${{ env.IMAGE_NAME }}.tar.gz
|
||||
|
||||
- name: Deploy to staging
|
||||
- name: Tag current image as rollback target
|
||||
run: |
|
||||
scp /tmp/${{ env.IMAGE_NAME }}.tar.gz root@${{ env.STAGING_HOST }}:/opt/${{ env.IMAGE_NAME }}/
|
||||
ssh root@${{ env.STAGING_HOST }} << 'EOF'
|
||||
cd /opt/${{ env.IMAGE_NAME }}
|
||||
docker load < ${{ env.IMAGE_NAME }}.tar.gz
|
||||
docker compose -f docker-compose.staging.yml down
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
for i in $(seq 1 30); do
|
||||
sleep 2
|
||||
curl -sf http://localhost:4014/mobile/api/health && echo "✅ Staging is healthy" && break
|
||||
done
|
||||
EOF
|
||||
ssh admin@${{ env.STAGING_HOST }} "\
|
||||
docker tag ${{ env.IMAGE_NAME }}:staging ${{ env.IMAGE_NAME }}:rollback 2>/dev/null; \
|
||||
echo 'Rollback tag created'"
|
||||
continue-on-error: true
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
- name: Deploy to staging
|
||||
id: deploy
|
||||
run: |
|
||||
scp /tmp/${{ env.IMAGE_NAME }}.tar.gz admin@${{ env.STAGING_HOST }}:/tmp/
|
||||
ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
|
||||
cd /var/services/homes/admin/falah-mobile
|
||||
docker load < /tmp/${{ env.IMAGE_NAME }}.tar.gz
|
||||
docker compose -f docker-compose.staging.yml down
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
# Wait for healthy with timeout
|
||||
for i in $(seq 1 60); do
|
||||
sleep 2
|
||||
HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
|
||||
if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
|
||||
echo "✅ Staging is healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting... attempt \$i"
|
||||
done
|
||||
echo "❌ Deployment failed health check"
|
||||
exit 1
|
||||
ENDSSH
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
- name: Auto-rollback on failure
|
||||
if: failure() && steps.deploy.outcome == 'failure'
|
||||
run: |
|
||||
echo "🚨 Deployment failed — initiating auto-rollback..."
|
||||
ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
|
||||
cd /var/services/homes/admin/falah-mobile
|
||||
if docker images ${{ env.IMAGE_NAME }}:rollback --format '{{.ID}}' | head -1; then
|
||||
echo "🔄 Rolling back to previous image..."
|
||||
docker compose -f docker-compose.staging.yml down
|
||||
docker tag ${{ env.IMAGE_NAME }}:rollback ${{ env.IMAGE_NAME }}:staging
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
for i in \$(seq 1 30); do
|
||||
sleep 2
|
||||
HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
|
||||
if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
|
||||
echo "✅ Rollback successful — service healthy"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "❌ Rollback also failed — manual intervention required"
|
||||
exit 1
|
||||
else
|
||||
echo "⚠️ No rollback image found — keeping current state"
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
fi
|
||||
ENDSSH
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
@@ -42,6 +42,11 @@ cd /app\n\
|
||||
node ./node_modules/prisma/build/index.js db push --skip-generate --accept-data-loss 2>&1\n\
|
||||
# Force 0.0.0.0 binding (Docker sets HOSTNAME to container ID)\nexport HOSTNAME=0.0.0.0\nexec node server.js\n' > /app/start.sh && chmod +x /app/start.sh
|
||||
|
||||
# ── Docker HEALTHCHECK ──────────────────────────────────────────
|
||||
# Auto-restarts container if health endpoint fails 3 times (30s interval, 10s timeout)
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
||||
CMD curl -sf http://localhost:3000/mobile/api/health || exit 1
|
||||
|
||||
USER nextjs
|
||||
EXPOSE 3000
|
||||
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# Falah Mobile — Auto-Healing Watchdog
|
||||
# ============================================================
|
||||
# Runs as a cron job or systemd timer on the Synology.
|
||||
# Detects failures and auto-fixes without human intervention.
|
||||
#
|
||||
# Install (as root on Synology):
|
||||
# cp scripts/auto-heal.sh /usr/local/bin/falah-auto-heal
|
||||
# chmod +x /usr/local/bin/falah-auto-heal
|
||||
# echo "*/5 * * * * root /usr/local/bin/falah-auto-heal" > /etc/cron.d/falah-auto-heal
|
||||
# ============================================================
|
||||
|
||||
DOCKER=/var/packages/Docker/target/usr/bin/docker
|
||||
CONTAINER=falah-mobile-staging
|
||||
HEALTH_URL="http://localhost:4014/mobile/api/health"
|
||||
GW_URL="http://localhost:7878/mobile/api/health"
|
||||
LOG_DIR="/var/log/falah"
|
||||
LOG="$LOG_DIR/auto-heal.log"
|
||||
STATE="$LOG_DIR/auto-heal.state"
|
||||
MAX_RESTARTS_PER_HOUR=3
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
# ── 1. HEALTH CHECK ──────────────────────────────────────────
|
||||
|
||||
check_health() {
|
||||
local url="$1"
|
||||
local label="$2"
|
||||
curl -sf --max-time 10 "$url" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# ── 2. TRACK RESTART COUNT ──────────────────────────────────
|
||||
|
||||
track_restart() {
|
||||
local now
|
||||
now=$(date +%s)
|
||||
# Keep only restarts in the last hour
|
||||
if [ -f "$STATE" ]; then
|
||||
sed -i "/^[0-9]*$/!d" "$STATE" 2>/dev/null
|
||||
awk -v now="$now" -v limit=3600 '$1 > now - limit' "$STATE" > "${STATE}.tmp" && mv "${STATE}.tmp" "$STATE"
|
||||
fi
|
||||
echo "$now" >> "$STATE"
|
||||
local count
|
||||
count=$(wc -l < "$STATE" 2>/dev/null || echo 0)
|
||||
echo "$count"
|
||||
}
|
||||
|
||||
# ── 3. AUTO-HEAL ACTIONS ────────────────────────────────────
|
||||
|
||||
heal_container() {
|
||||
log "⚠️ Starting auto-heal sequence for $CONTAINER..."
|
||||
|
||||
# Check if container exists
|
||||
if ! $DOCKER ps -a --format '{{.Names}}' | grep -q "^$CONTAINER$"; then
|
||||
log "❌ Container $CONTAINER does not exist. Cannot auto-heal."
|
||||
return 1
|
||||
fi
|
||||
|
||||
local restarts
|
||||
restarts=$(track_restart)
|
||||
log "🔄 Restarts in last hour: $restarts / $MAX_RESTARTS_PER_HOUR"
|
||||
|
||||
if [ "$restarts" -gt "$MAX_RESTARTS_PER_HOUR" ]; then
|
||||
log "🚨 Too many restarts ($restarts) — escalating instead of restarting"
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Step 1: Try restarting the container
|
||||
log "🔄 Restarting $CONTAINER..."
|
||||
$DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
|
||||
sleep 10
|
||||
|
||||
# Step 2: Verify recovery
|
||||
if check_health "$HEALTH_URL" "direct"; then
|
||||
log "✅ Auto-heal successful — container healthy after restart"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Step 3: If still failing, try docker-compose (rebuild from compose definition)
|
||||
log "⚠️ Restart not enough — trying compose down/up..."
|
||||
cd /volume1/docker/falah-core-staging/maifors-falah-core-*/ 2>/dev/null || \
|
||||
cd /var/services/homes/admin/falah-mobile/ 2>/dev/null || true
|
||||
|
||||
$DOCKER compose -f docker-compose.staging.yml down >> "$LOG" 2>&1
|
||||
sleep 3
|
||||
$DOCKER compose -f docker-compose.staging.yml up -d >> "$LOG" 2>&1
|
||||
sleep 15
|
||||
|
||||
if check_health "$HEALTH_URL" "direct"; then
|
||||
log "✅ Auto-heal successful — compose recovery worked"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "❌ Auto-heal failed — manual intervention required"
|
||||
return 3
|
||||
}
|
||||
|
||||
heal_gateway() {
|
||||
log "⚠️ Gateway unhealthy — reloading nginx..."
|
||||
|
||||
# Find the nginx container
|
||||
GW_CONTAINER=$($DOCKER ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
|
||||
if [ -n "$GW_CONTAINER" ]; then
|
||||
$DOCKER exec "$GW_CONTAINER" nginx -s reload >> "$LOG" 2>&1
|
||||
log "✅ nginx reloaded in $GW_CONTAINER"
|
||||
return 0
|
||||
fi
|
||||
log "⚠️ No nginx/gateway container found"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── 4. MAIN ──────────────────────────────────────────────────
|
||||
|
||||
main() {
|
||||
log "═══════════════════════════════════════════════"
|
||||
log "🔍 Auto-heal check starting..."
|
||||
|
||||
local direct_ok=true
|
||||
local gateway_ok=true
|
||||
|
||||
# Check direct container health
|
||||
if ! check_health "$HEALTH_URL" "direct"; then
|
||||
direct_ok=false
|
||||
log "❌ Direct health check FAILED"
|
||||
heal_container
|
||||
else
|
||||
log "✅ Direct container healthy"
|
||||
fi
|
||||
|
||||
# Check gateway health
|
||||
if ! check_health "$GW_URL" "gateway"; then
|
||||
gateway_ok=false
|
||||
log "❌ Gateway health check FAILED"
|
||||
heal_gateway
|
||||
else
|
||||
log "✅ Gateway healthy"
|
||||
fi
|
||||
|
||||
# Check db connectivity via direct health
|
||||
HEALTH_BODY=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null)
|
||||
if echo "$HEALTH_BODY" | grep -q '"db":false'; then
|
||||
log "⚠️ Database disconnected! Attempting recovery..."
|
||||
$DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
|
||||
log "🔄 Restarted container to recover DB connection"
|
||||
fi
|
||||
|
||||
if $direct_ok && $gateway_ok; then
|
||||
log "✅ All systems healthy"
|
||||
else
|
||||
log "⚠️ Some systems unhealthy after healing attempts"
|
||||
fi
|
||||
|
||||
log "═══════════════════════════════════════════════"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -3,8 +3,29 @@ const rateMap = new Map<string, { count: number; resetAt: number }>();
|
||||
const WINDOW_MS = 15 * 60 * 1000;
|
||||
const MAX_ATTEMPTS = 10;
|
||||
|
||||
// Auto-cleanup stale entries every 5 minutes to prevent memory leaks
|
||||
const CLEANUP_INTERVAL = 5 * 60 * 1000;
|
||||
let lastCleanup = Date.now();
|
||||
|
||||
function cleanupStaleEntries(): void {
|
||||
const now = Date.now();
|
||||
if (now - lastCleanup < CLEANUP_INTERVAL) return;
|
||||
lastCleanup = now;
|
||||
let removed = 0;
|
||||
for (const [ip, entry] of rateMap.entries()) {
|
||||
if (now > entry.resetAt) {
|
||||
rateMap.delete(ip);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
if (removed > 0) {
|
||||
console.debug(`[rate-limiter] Cleaned up ${removed} stale entries, map size: ${rateMap.size}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function checkRateLimit(ip: string): { allowed: boolean; remaining: number } {
|
||||
const now = Date.now();
|
||||
cleanupStaleEntries();
|
||||
const entry = rateMap.get(ip);
|
||||
|
||||
if (!entry || now > entry.resetAt) {
|
||||
|
||||
Reference in New Issue
Block a user