diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index c0b0afe..c78053f 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -7,11 +7,12 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: falah-mobile - STAGING_HOST: 192.168.0.17 + STAGING_HOST: 192.168.0.11 jobs: build: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -37,18 +38,65 @@ jobs: docker build -t ${{ env.IMAGE_NAME }}:staging . docker save ${{ env.IMAGE_NAME }}:staging | gzip > /tmp/${{ env.IMAGE_NAME }}.tar.gz - - name: Deploy to staging + - name: Tag current image as rollback target run: | - scp /tmp/${{ env.IMAGE_NAME }}.tar.gz root@${{ env.STAGING_HOST }}:/opt/${{ env.IMAGE_NAME }}/ - ssh root@${{ env.STAGING_HOST }} << 'EOF' - cd /opt/${{ env.IMAGE_NAME }} - docker load < ${{ env.IMAGE_NAME }}.tar.gz - docker compose -f docker-compose.staging.yml down - docker compose -f docker-compose.staging.yml up -d - for i in $(seq 1 30); do - sleep 2 - curl -sf http://localhost:4014/mobile/api/health && echo "✅ Staging is healthy" && break - done - EOF + ssh admin@${{ env.STAGING_HOST }} "\ + docker tag ${{ env.IMAGE_NAME }}:staging ${{ env.IMAGE_NAME }}:rollback 2>/dev/null; \ + echo 'Rollback tag created'" + continue-on-error: true + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Deploy to staging + id: deploy + run: | + scp /tmp/${{ env.IMAGE_NAME }}.tar.gz admin@${{ env.STAGING_HOST }}:/tmp/ + ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH' + cd /var/services/homes/admin/falah-mobile + docker load < /tmp/${{ env.IMAGE_NAME }}.tar.gz + docker compose -f docker-compose.staging.yml down + docker compose -f docker-compose.staging.yml up -d + # Wait for healthy with timeout + for i in $(seq 1 60); do + sleep 2 + HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null) + if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then + echo "✅ Staging is healthy" + exit 0 + fi + echo "Waiting... attempt \$i" + done + echo "❌ Deployment failed health check" + exit 1 + ENDSSH + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Auto-rollback on failure + if: failure() && steps.deploy.outcome == 'failure' + run: | + echo "🚨 Deployment failed — initiating auto-rollback..." + ssh admin@${{ env.STAGING_HOST }} << 'ENDSSH' + cd /var/services/homes/admin/falah-mobile + if docker images ${{ env.IMAGE_NAME }}:rollback --format '{{.ID}}' | head -1; then + echo "🔄 Rolling back to previous image..." + docker compose -f docker-compose.staging.yml down + docker tag ${{ env.IMAGE_NAME }}:rollback ${{ env.IMAGE_NAME }}:staging + docker compose -f docker-compose.staging.yml up -d + for i in \$(seq 1 30); do + sleep 2 + HEALTH=\\$(curl -sf http://localhost:4014/mobile/api/health 2>/dev/null) + if echo \"\$HEALTH\" | grep -q '\"status\":\"ok\"'; then + echo "✅ Rollback successful — service healthy" + exit 0 + fi + done + echo "❌ Rollback also failed — manual intervention required" + exit 1 + else + echo "⚠️ No rollback image found — keeping current state" + docker compose -f docker-compose.staging.yml up -d + fi + ENDSSH env: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/Dockerfile b/Dockerfile index a74fb29..338ceb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,11 @@ cd /app\n\ node ./node_modules/prisma/build/index.js db push --skip-generate --accept-data-loss 2>&1\n\ # Force 0.0.0.0 binding (Docker sets HOSTNAME to container ID)\nexport HOSTNAME=0.0.0.0\nexec node server.js\n' > /app/start.sh && chmod +x /app/start.sh +# ── Docker HEALTHCHECK ────────────────────────────────────────── +# Auto-restarts container if health endpoint fails 3 times (30s interval, 10s timeout) +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD curl -sf http://localhost:3000/mobile/api/health || exit 1 + USER nextjs EXPOSE 3000 diff --git a/scripts/auto-heal.sh b/scripts/auto-heal.sh new file mode 100644 index 0000000..08eb260 --- /dev/null +++ b/scripts/auto-heal.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# ============================================================ +# Falah Mobile — Auto-Healing Watchdog +# ============================================================ +# Runs as a cron job or systemd timer on the Synology. +# Detects failures and auto-fixes without human intervention. +# +# Install (as root on Synology): +# cp scripts/auto-heal.sh /usr/local/bin/falah-auto-heal +# chmod +x /usr/local/bin/falah-auto-heal +# echo "*/5 * * * * root /usr/local/bin/falah-auto-heal" > /etc/cron.d/falah-auto-heal +# ============================================================ + +DOCKER=/var/packages/Docker/target/usr/bin/docker +CONTAINER=falah-mobile-staging +HEALTH_URL="http://localhost:4014/mobile/api/health" +GW_URL="http://localhost:7878/mobile/api/health" +LOG_DIR="/var/log/falah" +LOG="$LOG_DIR/auto-heal.log" +STATE="$LOG_DIR/auto-heal.state" +MAX_RESTARTS_PER_HOUR=3 + +mkdir -p "$LOG_DIR" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG" + echo "$*" +} + +# ── 1. HEALTH CHECK ────────────────────────────────────────── + +check_health() { + local url="$1" + local label="$2" + curl -sf --max-time 10 "$url" > /dev/null 2>&1 +} + +# ── 2. TRACK RESTART COUNT ────────────────────────────────── + +track_restart() { + local now + now=$(date +%s) + # Keep only restarts in the last hour + if [ -f "$STATE" ]; then + sed -i "/^[0-9]*$/!d" "$STATE" 2>/dev/null + awk -v now="$now" -v limit=3600 '$1 > now - limit' "$STATE" > "${STATE}.tmp" && mv "${STATE}.tmp" "$STATE" + fi + echo "$now" >> "$STATE" + local count + count=$(wc -l < "$STATE" 2>/dev/null || echo 0) + echo "$count" +} + +# ── 3. AUTO-HEAL ACTIONS ──────────────────────────────────── + +heal_container() { + log "⚠️ Starting auto-heal sequence for $CONTAINER..." + + # Check if container exists + if ! $DOCKER ps -a --format '{{.Names}}' | grep -q "^$CONTAINER$"; then + log "❌ Container $CONTAINER does not exist. Cannot auto-heal." + return 1 + fi + + local restarts + restarts=$(track_restart) + log "🔄 Restarts in last hour: $restarts / $MAX_RESTARTS_PER_HOUR" + + if [ "$restarts" -gt "$MAX_RESTARTS_PER_HOUR" ]; then + log "🚨 Too many restarts ($restarts) — escalating instead of restarting" + return 2 + fi + + # Step 1: Try restarting the container + log "🔄 Restarting $CONTAINER..." + $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1 + sleep 10 + + # Step 2: Verify recovery + if check_health "$HEALTH_URL" "direct"; then + log "✅ Auto-heal successful — container healthy after restart" + return 0 + fi + + # Step 3: If still failing, try docker-compose (rebuild from compose definition) + log "⚠️ Restart not enough — trying compose down/up..." + cd /volume1/docker/falah-core-staging/maifors-falah-core-*/ 2>/dev/null || \ + cd /var/services/homes/admin/falah-mobile/ 2>/dev/null || true + + $DOCKER compose -f docker-compose.staging.yml down >> "$LOG" 2>&1 + sleep 3 + $DOCKER compose -f docker-compose.staging.yml up -d >> "$LOG" 2>&1 + sleep 15 + + if check_health "$HEALTH_URL" "direct"; then + log "✅ Auto-heal successful — compose recovery worked" + return 0 + fi + + log "❌ Auto-heal failed — manual intervention required" + return 3 +} + +heal_gateway() { + log "⚠️ Gateway unhealthy — reloading nginx..." + + # Find the nginx container + GW_CONTAINER=$($DOCKER ps --format '{{.Names}}' | grep -i "gateway\|nginx" | head -1) + if [ -n "$GW_CONTAINER" ]; then + $DOCKER exec "$GW_CONTAINER" nginx -s reload >> "$LOG" 2>&1 + log "✅ nginx reloaded in $GW_CONTAINER" + return 0 + fi + log "⚠️ No nginx/gateway container found" + return 1 +} + +# ── 4. MAIN ────────────────────────────────────────────────── + +main() { + log "═══════════════════════════════════════════════" + log "🔍 Auto-heal check starting..." + + local direct_ok=true + local gateway_ok=true + + # Check direct container health + if ! check_health "$HEALTH_URL" "direct"; then + direct_ok=false + log "❌ Direct health check FAILED" + heal_container + else + log "✅ Direct container healthy" + fi + + # Check gateway health + if ! check_health "$GW_URL" "gateway"; then + gateway_ok=false + log "❌ Gateway health check FAILED" + heal_gateway + else + log "✅ Gateway healthy" + fi + + # Check db connectivity via direct health + HEALTH_BODY=$(curl -sf --max-time 10 "$HEALTH_URL" 2>/dev/null) + if echo "$HEALTH_BODY" | grep -q '"db":false'; then + log "⚠️ Database disconnected! Attempting recovery..." + $DOCKER restart "$CONTAINER" >> "$LOG" 2>&1 + log "🔄 Restarted container to recover DB connection" + fi + + if $direct_ok && $gateway_ok; then + log "✅ All systems healthy" + else + log "⚠️ Some systems unhealthy after healing attempts" + fi + + log "═══════════════════════════════════════════════" +} + +main "$@" diff --git a/src/lib/rate-limit.ts b/src/lib/rate-limit.ts index 4d243bc..8fd989c 100644 --- a/src/lib/rate-limit.ts +++ b/src/lib/rate-limit.ts @@ -3,8 +3,29 @@ const rateMap = new Map(); const WINDOW_MS = 15 * 60 * 1000; const MAX_ATTEMPTS = 10; +// Auto-cleanup stale entries every 5 minutes to prevent memory leaks +const CLEANUP_INTERVAL = 5 * 60 * 1000; +let lastCleanup = Date.now(); + +function cleanupStaleEntries(): void { + const now = Date.now(); + if (now - lastCleanup < CLEANUP_INTERVAL) return; + lastCleanup = now; + let removed = 0; + for (const [ip, entry] of rateMap.entries()) { + if (now > entry.resetAt) { + rateMap.delete(ip); + removed++; + } + } + if (removed > 0) { + console.debug(`[rate-limiter] Cleaned up ${removed} stale entries, map size: ${rateMap.size}`); + } +} + export function checkRateLimit(ip: string): { allowed: boolean; remaining: number } { const now = Date.now(); + cleanupStaleEntries(); const entry = rateMap.get(ip); if (!entry || now > entry.resetAt) {