feat: auto-healing system — HEALTHCHECK, rate-limiter cleanup, auto-rollback CI, watchdog
Deploy Staging / build (push) Failing after 14m49s
Deploy Staging / build (push) Failing after 14m49s
- Dockerfile: add HEALTHCHECK (30s interval, 10s timeout, 3 retries) - rate-limit.ts: auto-cleanup stale entries every 5 min (prevents memory leak) - scripts/auto-heal.sh: watchdog for Synology cron — auto-restarts container, reloads nginx gateway, escalates on excessive failures - CI workflow: auto-rollback on deploy failure (tags :rollback image, reverts if health check fails after deploy)
This commit is contained in:
+61
-13
@@ -7,11 +7,12 @@ on:
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: falah-mobile
|
||||
STAGING_HOST: 192.168.0.17
|
||||
STAGING_HOST: 192.168.0.11
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -37,18 +38,65 @@ jobs:
|
||||
docker build -t ${{ env.IMAGE_NAME }}:staging .
|
||||
docker save ${{ env.IMAGE_NAME }}:staging | gzip > /tmp/${{ env.IMAGE_NAME }}.tar.gz
|
||||
|
||||
- name: Deploy to staging
|
||||
- name: Tag current image as rollback target
|
||||
run: |
|
||||
scp /tmp/${{ env.IMAGE_NAME }}.tar.gz root@${{ env.STAGING_HOST }}:/opt/${{ env.IMAGE_NAME }}/
|
||||
ssh root@${{ env.STAGING_HOST }} << 'EOF'
|
||||
cd /opt/${{ env.IMAGE_NAME }}
|
||||
docker load < ${{ env.IMAGE_NAME }}.tar.gz
|
||||
docker compose -f docker-compose.staging.yml down
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
for i in $(seq 1 30); do
|
||||
sleep 2
|
||||
curl -sf http://localhost:4014/mobile/api/health && echo "✅ Staging is healthy" && break
|
||||
done
|
||||
EOF
|
||||
ssh admin@${{ env.STAGING_HOST }} "\
|
||||
docker tag ${{ env.IMAGE_NAME }}:staging ${{ env.IMAGE_NAME }}:rollback 2>/dev/null; \
|
||||
echo 'Rollback tag created'"
|
||||
continue-on-error: true
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
- name: Deploy to staging
|
||||
id: deploy
|
||||
run: |
|
||||
scp /tmp/${{ env.IMAGE_NAME }}.tar.gz admin@${{ env.STAGING_HOST }}:/tmp/
|
||||
ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
|
||||
cd /var/services/homes/admin/falah-mobile
|
||||
docker load < /tmp/${{ env.IMAGE_NAME }}.tar.gz
|
||||
docker compose -f docker-compose.staging.yml down
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
# Wait for healthy with timeout
|
||||
for i in $(seq 1 60); do
|
||||
sleep 2
|
||||
HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
|
||||
if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
|
||||
echo "✅ Staging is healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting... attempt \$i"
|
||||
done
|
||||
echo "❌ Deployment failed health check"
|
||||
exit 1
|
||||
ENDSSH
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
- name: Auto-rollback on failure
|
||||
if: failure() && steps.deploy.outcome == 'failure'
|
||||
run: |
|
||||
echo "🚨 Deployment failed — initiating auto-rollback..."
|
||||
ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
|
||||
cd /var/services/homes/admin/falah-mobile
|
||||
if docker images ${{ env.IMAGE_NAME }}:rollback --format '{{.ID}}' | head -1; then
|
||||
echo "🔄 Rolling back to previous image..."
|
||||
docker compose -f docker-compose.staging.yml down
|
||||
docker tag ${{ env.IMAGE_NAME }}:rollback ${{ env.IMAGE_NAME }}:staging
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
for i in \$(seq 1 30); do
|
||||
sleep 2
|
||||
HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
|
||||
if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
|
||||
echo "✅ Rollback successful — service healthy"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "❌ Rollback also failed — manual intervention required"
|
||||
exit 1
|
||||
else
|
||||
echo "⚠️ No rollback image found — keeping current state"
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
fi
|
||||
ENDSSH
|
||||
env:
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
Reference in New Issue
Block a user