feat: auto-healing system — HEALTHCHECK, rate-limiter cleanup, auto-rollback CI, watchdog

- Dockerfile: add HEALTHCHECK (30s interval, 10s timeout, 3 retries)
- rate-limit.ts: auto-cleanup stale entries every 5 min (prevents memory leak)
- scripts/auto-heal.sh: watchdog for Synology cron — auto-restarts container,
  reloads nginx gateway, escalates on excessive failures
- CI workflow: auto-rollback on deploy failure (tags :rollback image,
  reverts if health check fails after deploy)

.gitea/workflows/ci.yml

@@ -7,11 +7,12 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: falah-mobile
-  STAGING_HOST: 192.168.0.17
+  STAGING_HOST: 192.168.0.11
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
 
@@ -37,18 +38,65 @@ jobs:
           docker build -t ${{ env.IMAGE_NAME }}:staging .
           docker save ${{ env.IMAGE_NAME }}:staging | gzip > /tmp/${{ env.IMAGE_NAME }}.tar.gz
 
-      - name: Deploy to staging
+      - name: Tag current image as rollback target
         run: |
-          scp /tmp/${{ env.IMAGE_NAME }}.tar.gz root@${{ env.STAGING_HOST }}:/opt/${{ env.IMAGE_NAME }}/
+          ssh admin@${{ env.STAGING_HOST }} "\
-          ssh root@${{ env.STAGING_HOST }} << 'EOF'
+            docker tag ${{ env.IMAGE_NAME }}:staging ${{ env.IMAGE_NAME }}:rollback 2>/dev/null; \
-            cd /opt/${{ env.IMAGE_NAME }}
+            echo 'Rollback tag created'"
-            docker load < ${{ env.IMAGE_NAME }}.tar.gz
+        continue-on-error: true
-            docker compose -f docker-compose.staging.yml down
+        env:
-            docker compose -f docker-compose.staging.yml up -d
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
-            for i in $(seq 1 30); do
+
-              sleep 2
+      - name: Deploy to staging
-              curl -sf http://localhost:4014/mobile/api/health && echo "✅ Staging is healthy" && break
+        id: deploy
-            done
+        run: |
-          EOF
+          scp /tmp/${{ env.IMAGE_NAME }}.tar.gz admin@${{ env.STAGING_HOST }}:/tmp/
+          ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
+            cd /var/services/homes/admin/falah-mobile
+            docker load < /tmp/${{ env.IMAGE_NAME }}.tar.gz
+            docker compose -f docker-compose.staging.yml down
+            docker compose -f docker-compose.staging.yml up -d
+            # Wait for healthy with timeout
+            for i in $(seq 1 60); do
+              sleep 2
+              HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
+              if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
+                echo "✅ Staging is healthy"
+                exit 0
+              fi
+              echo "Waiting... attempt \$i"
+            done
+            echo "❌ Deployment failed health check"
+            exit 1
+          ENDSSH
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+
+      - name: Auto-rollback on failure
+        if: failure() && steps.deploy.outcome == 'failure'
+        run: |
+          echo "🚨 Deployment failed — initiating auto-rollback..."
+          ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH'
+            cd /var/services/homes/admin/falah-mobile
+            if docker images ${{ env.IMAGE_NAME }}:rollback --format '{{.ID}}' | head -1; then
+              echo "🔄 Rolling back to previous image..."
+              docker compose -f docker-compose.staging.yml down
+              docker tag ${{ env.IMAGE_NAME }}:rollback ${{ env.IMAGE_NAME }}:staging
+              docker compose -f docker-compose.staging.yml up -d
+              for i in \$(seq 1 30); do
+                sleep 2
+                HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null)
+                if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then
+                  echo "✅ Rollback successful — service healthy"
+                  exit 0
+                fi
+              done
+              echo "❌ Rollback also failed — manual intervention required"
+              exit 1
+            else
+              echo "⚠️  No rollback image found — keeping current state"
+              docker compose -f docker-compose.staging.yml up -d
+            fi
+          ENDSSH
         env:
           SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

Dockerfile

@@ -42,6 +42,11 @@ cd /app\n\
 node ./node_modules/prisma/build/index.js db push --skip-generate --accept-data-loss 2>&1\n\
 # Force 0.0.0.0 binding (Docker sets HOSTNAME to container ID)\nexport HOSTNAME=0.0.0.0\nexec node server.js\n' > /app/start.sh && chmod +x /app/start.sh
 
+# ── Docker HEALTHCHECK ──────────────────────────────────────────
+# Auto-restarts container if health endpoint fails 3 times (30s interval, 10s timeout)
+HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
+  CMD curl -sf http://localhost:3000/mobile/api/health || exit 1
+
 USER nextjs
 EXPOSE 3000
 

scripts/auto-heal.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+# ============================================================
+# Falah Mobile — Auto-Healing Watchdog
+# ============================================================
+# Runs as a cron job or systemd timer on the Synology.
+# Detects failures and auto-fixes without human intervention.
+#
+# Install (as root on Synology):
+#   cp scripts/auto-heal.sh /usr/local/bin/falah-auto-heal
+#   chmod +x /usr/local/bin/falah-auto-heal
+#   echo "*/5 * * * * root /usr/local/bin/falah-auto-heal" > /etc/cron.d/falah-auto-heal
+# ============================================================
+
+DOCKER=/var/packages/Docker/target/usr/bin/docker
+CONTAINER=falah-mobile-staging
+HEALTH_URL="http://localhost:4014/mobile/api/health"
+GW_URL="http://localhost:7878/mobile/api/health"
+LOG_DIR="/var/log/falah"
+LOG="$LOG_DIR/auto-heal.log"
+STATE="$LOG_DIR/auto-heal.state"
+MAX_RESTARTS_PER_HOUR=3
+
+mkdir -p "$LOG_DIR"
+
+log() {
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"
+  echo "$*"
+}
+
+# ── 1. HEALTH CHECK ──────────────────────────────────────────
+
+check_health() {
+  local url="$1"
+  local label="$2"
+  curl -sf --max-time 10 "$url" > /dev/null 2>&1
+}
+
+# ── 2. TRACK RESTART COUNT ──────────────────────────────────
+
+track_restart() {
+  local now
+  now=$(date +%s)
+  # Keep only restarts in the last hour
+  if [ -f "$STATE" ]; then
+    sed -i "/^[0-9]*$/!d" "$STATE" 2>/dev/null
+    awk -v now="$now" -v limit=3600 '$1 > now - limit' "$STATE" > "${STATE}.tmp" && mv "${STATE}.tmp" "$STATE"
+  fi
+  echo "$now" >> "$STATE"
+  local count
+  count=$(wc -l < "$STATE" 2>/dev/null || echo 0)
+  echo "$count"
+}
+
+# ── 3. AUTO-HEAL ACTIONS ────────────────────────────────────
+
+heal_container() {
+  log "⚠️  Starting auto-heal sequence for $CONTAINER..."
+
+  # Check if container exists
+  if ! $DOCKER ps -a --format '{{.Names}}' | grep -q "^$CONTAINER$"; then
+    log "❌ Container $CONTAINER does not exist. Cannot auto-heal."
+    return 1
+  fi
+
+  local restarts
+  restarts=$(track_restart)
+  log "🔄 Restarts in last hour: $restarts / $MAX_RESTARTS_PER_HOUR"
+
+  if [ "$restarts" -gt "$MAX_RESTARTS_PER_HOUR" ]; then
+    log "🚨 Too many restarts ($restarts) — escalating instead of restarting"
+    return 2
+  fi
+
+  # Step 1: Try restarting the container
+  log "🔄 Restarting $CONTAINER..."
+  $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
+  sleep 10
+
+  # Step 2: Verify recovery
+  if check_health "$HEALTH_URL" "direct"; then
+    log "✅ Auto-heal successful — container healthy after restart"
+    return 0
+  fi
+
+  # Step 3: If still failing, try docker-compose (rebuild from compose definition)
+  log "⚠️  Restart not enough — trying compose down/up..."
+  cd /volume1/docker/falah-core-staging/maifors-falah-core-*/ 2>/dev/null || \
+    cd /var/services/homes/admin/falah-mobile/ 2>/dev/null || true
+  
+  $DOCKER compose -f docker-compose.staging.yml down >> "$LOG" 2>&1
+  sleep 3
+  $DOCKER compose -f docker-compose.staging.yml up -d >> "$LOG" 2>&1
+  sleep 15
+
+  if check_health "$HEALTH_URL" "direct"; then
+    log "✅ Auto-heal successful — compose recovery worked"
+    return 0
+  fi
+
+  log "❌ Auto-heal failed — manual intervention required"
+  return 3
+}
+
+heal_gateway() {
+  log "⚠️  Gateway unhealthy — reloading nginx..."
+  
+  # Find the nginx container
+  GW_CONTAINER=$($DOCKER ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1)
+  if [ -n "$GW_CONTAINER" ]; then
+    $DOCKER exec "$GW_CONTAINER" nginx -s reload >> "$LOG" 2>&1
+    log "✅ nginx reloaded in $GW_CONTAINER"
+    return 0
+  fi
+  log "⚠️  No nginx/gateway container found"
+  return 1
+}
+
+# ── 4. MAIN ──────────────────────────────────────────────────
+
+main() {
+  log "═══════════════════════════════════════════════"
+  log "🔍 Auto-heal check starting..."
+
+  local direct_ok=true
+  local gateway_ok=true
+
+  # Check direct container health
+  if ! check_health "$HEALTH_URL" "direct"; then
+    direct_ok=false
+    log "❌ Direct health check FAILED"
+    heal_container
+  else
+    log "✅ Direct container healthy"
+  fi
+
+  # Check gateway health
+  if ! check_health "$GW_URL" "gateway"; then
+    gateway_ok=false
+    log "❌ Gateway health check FAILED"
+    heal_gateway
+  else
+    log "✅ Gateway healthy"
+  fi
+
+  # Check db connectivity via direct health
+  HEALTH_BODY=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null)
+  if echo "$HEALTH_BODY" | grep -q '"db":false'; then
+    log "⚠️  Database disconnected! Attempting recovery..."
+    $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1
+    log "🔄 Restarted container to recover DB connection"
+  fi
+
+  if $direct_ok && $gateway_ok; then
+    log "✅ All systems healthy"
+  else
+    log "⚠️  Some systems unhealthy after healing attempts"
+  fi
+
+  log "═══════════════════════════════════════════════"
+}
+
+main "$@"

src/lib/rate-limit.ts

@@ -3,8 +3,29 @@ const rateMap = new Map<string, { count: number; resetAt: number }>();
 const WINDOW_MS = 15 * 60 * 1000;
 const MAX_ATTEMPTS = 10;
 
+// Auto-cleanup stale entries every 5 minutes to prevent memory leaks
+const CLEANUP_INTERVAL = 5 * 60 * 1000;
+let lastCleanup = Date.now();
+
+function cleanupStaleEntries(): void {
+  const now = Date.now();
+  if (now - lastCleanup < CLEANUP_INTERVAL) return;
+  lastCleanup = now;
+  let removed = 0;
+  for (const [ip, entry] of rateMap.entries()) {
+    if (now > entry.resetAt) {
+      rateMap.delete(ip);
+      removed++;
+    }
+  }
+  if (removed > 0) {
+    console.debug(`[rate-limiter] Cleaned up ${removed} stale entries, map size: ${rateMap.size}`);
+  }
+}
+
 export function checkRateLimit(ip: string): { allowed: boolean; remaining: number } {
   const now = Date.now();
+  cleanupStaleEntries();
   const entry = rateMap.get(ip);
 
   if (!entry || now > entry.resetAt) {