feat: auto-healing system — HEALTHCHECK, rate-limiter cleanup, auto-rollback CI, watchdog
Deploy Staging / build (push) Failing after 14m49s

- Dockerfile: add HEALTHCHECK (30s interval, 10s timeout, 3 retries)
- rate-limit.ts: auto-cleanup stale entries every 5 min (prevents memory leak)
- scripts/auto-heal.sh: watchdog for Synology cron — auto-restarts container,
  reloads nginx gateway, escalates on excessive failures
- CI workflow: auto-rollback on deploy failure (tags :rollback image,
  reverts if health check fails after deploy)
This commit is contained in:
2026-06-28 02:52:11 +08:00
parent aac485ba30
commit 0098d9ca4a
4 changed files with 249 additions and 13 deletions
+61 -13
View File
@@ -7,11 +7,12 @@ on:
env:
REGISTRY: ghcr.io
IMAGE_NAME: falah-mobile
STAGING_HOST: 192.168.0.17
STAGING_HOST: 192.168.0.11
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
@@ -37,18 +38,65 @@ jobs:
docker build -t ${{ env.IMAGE_NAME }}:staging .
docker save ${{ env.IMAGE_NAME }}:staging | gzip > /tmp/${{ env.IMAGE_NAME }}.tar.gz
- name: Deploy to staging
- name: Tag current image as rollback target
run: |
scp /tmp/${{ env.IMAGE_NAME }}.tar.gz root@${{ env.STAGING_HOST }}:/opt/${{ env.IMAGE_NAME }}/
ssh root@${{ env.STAGING_HOST }} << 'EOF'
cd /opt/${{ env.IMAGE_NAME }}
docker load < ${{ env.IMAGE_NAME }}.tar.gz
docker compose -f docker-compose.staging.yml down
docker compose -f docker-compose.staging.yml up -d
for i in $(seq 1 30); do
sleep 2
curl -sf http://localhost:4014/mobile/api/health && echo "✅ Staging is healthy" && break
done
EOF
ssh admin@${{ env.STAGING_HOST }} "\
docker tag ${{ env.IMAGE_NAME }}:staging ${{ env.IMAGE_NAME }}:rollback 2>/dev/null; \
echo 'Rollback tag created'"
continue-on-error: true
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
- name: Deploy to staging
id: deploy
run: |
scp /tmp/${{ env.IMAGE_NAME }}.tar.gz admin@${{ env.STAGING_HOST }}:/tmp/
ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
cd /var/services/homes/admin/falah-mobile
docker load < /tmp/${{ env.IMAGE_NAME }}.tar.gz
docker compose -f docker-compose.staging.yml down
docker compose -f docker-compose.staging.yml up -d
# Wait for healthy with timeout
for i in $(seq 1 60); do
sleep 2
HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
echo "✅ Staging is healthy"
exit 0
fi
echo "Waiting... attempt \$i"
done
echo "❌ Deployment failed health check"
exit 1
ENDSSH
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
- name: Auto-rollback on failure
if: failure() && steps.deploy.outcome == 'failure'
run: |
echo "🚨 Deployment failed — initiating auto-rollback..."
ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
cd /var/services/homes/admin/falah-mobile
if docker images ${{ env.IMAGE_NAME }}:rollback --format '{{.ID}}' | head -1; then
echo "🔄 Rolling back to previous image..."
docker compose -f docker-compose.staging.yml down
docker tag ${{ env.IMAGE_NAME }}:rollback ${{ env.IMAGE_NAME }}:staging
docker compose -f docker-compose.staging.yml up -d
for i in \$(seq 1 30); do
sleep 2
HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
echo "✅ Rollback successful — service healthy"
exit 0
fi
done
echo "❌ Rollback also failed — manual intervention required"
exit 1
else
echo "⚠️ No rollback image found — keeping current state"
docker compose -f docker-compose.staging.yml up -d
fi
ENDSSH
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}