root
14d7127e41
- Error Feedback Service: POST /api/feedback saves to JSONL, GET with admin token - ErrorFeedback component: warm amber tones, friendly messages, 'Report Issue' button - Replaced ALL red-themed error displays across 15+ pages with ErrorFeedback - Kind classification: network, auth, location, upgrade, default messages - QA test suite v2: 8-layer testing (system, API, render, scenario, edge, mobile, perf, security) - Browser-based visual QA with Playwright (Layer 9)
449 lines
18 KiB
Bash
449 lines
18 KiB
Bash
#!/bin/bash
|
|
#=============================================================================
|
|
# Falah Mobile — Comprehensive QA Test Suite
|
|
#
|
|
# Multi-layer testing: System → API → Render → User Flows → Edge → Mobile
|
|
#
|
|
# Usage:
|
|
# ./tests/qa-test-suite.sh # Full run
|
|
# ./tests/qa-test-suite.sh --layer system # Layer 1 only
|
|
# ./tests/qa-test-suite.sh --ci-exit # Exit 1 on failures
|
|
#
|
|
# Layers:
|
|
# 1. SYSTEM — HTTP status, route existence
|
|
# 2. API — Response shape, data type validation
|
|
# 3. RENDER — Page content validates (no error messages in HTML)
|
|
# 4. FLOW — Multi-step user journeys end-to-end
|
|
# 5. EDGE — Permission denied, missing params, rate limiting
|
|
# 6. MOBILE — Viewport, touch targets (≥44px), text size ≥12px
|
|
#=============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
BASE_URL="${BASE_URL:-https://falahos.my/mobile}"
|
|
TIMEOUT=15
|
|
PASS=0; FAIL=0; WARN=0
|
|
LAYER_FILTER="all"
|
|
CI_EXIT=false
|
|
REPORT_FILE=""
|
|
|
|
# Parse args
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--layer=*) LAYER_FILTER="${1#*=}" ;;
|
|
--ci-exit) CI_EXIT=true ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'
|
|
CYAN='\033[0;36m'; NC='\033[0m'
|
|
|
|
pass() { PASS=$((PASS+1)); echo -e " ${GREEN}✓${NC} $1"; }
|
|
fail() { FAIL=$((FAIL+1)); echo -e " ${RED}✗${NC} $1"; }
|
|
warn() { WARN=$((WARN+1)); echo -e " ${YELLOW}⚠${NC} $1"; }
|
|
|
|
section() { echo ""; echo -e "${CYAN}════════════════════════════════════════${NC}"; echo -e "${CYAN} $1${NC}"; echo -e "${CYAN}════════════════════════════════════════${NC}"; }
|
|
sub() { echo -e "\n${CYAN}── $1 ──${NC}"; }
|
|
|
|
should_run() { [ "$LAYER_FILTER" = "all" ] || [ "$LAYER_FILTER" = "$1" ]; }
|
|
|
|
fetch() { curl -s --connect-timeout "$TIMEOUT" "$@" 2>/dev/null || echo ""; }
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# LAYER 1: System — Route Existence & HTTP Status
|
|
#─────────────────────────────────────────────────────────────────
|
|
layer_system() {
|
|
section "LAYER 1: System — Route Existence & HTTP Status"
|
|
|
|
local routes=(
|
|
"/" "Landing"
|
|
"/prayer" "Prayer"
|
|
"/dhikr" "Dhikr"
|
|
"/qibla" "Qibla"
|
|
"/halal-monitor" "Halal Monitor"
|
|
"/nur" "Nur AI"
|
|
"/forum" "Forum"
|
|
"/souq" "Souq"
|
|
"/groups" "Groups"
|
|
"/api/health" "Health API"
|
|
)
|
|
|
|
for ((i=0; i<${#routes[@]}; i+=2)); do
|
|
local path="${routes[$i]}"
|
|
local label="${routes[$((i+1))]}"
|
|
local code
|
|
code=$(fetch -o /dev/null -w '%{http_code}' "${BASE_URL}${path}")
|
|
case "$code" in
|
|
200|308) pass "$label ($path → $code)" ;;
|
|
000) fail "$label ($path → CONNECTION FAILED)" ;;
|
|
404) fail "$label ($path → 404 NOT FOUND)" ;;
|
|
500) fail "$label ($path → 500 SERVER ERROR)" ;;
|
|
*) fail "$label ($path → $code)" ;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# LAYER 2: API Response Shape Validation
|
|
#─────────────────────────────────────────────────────────────────
|
|
layer_api() {
|
|
section "LAYER 2: API — Response Shape & Data Type Validation"
|
|
|
|
# —— Prayer API ——
|
|
sub "Prayer API — timings, date, hijri, city, country, timezone"
|
|
local resp
|
|
resp=$(fetch "${BASE_URL}/api/prayer?city=Kuala+Lumpur&country=MY")
|
|
if echo "$resp" | python3 -c "
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
checks = {
|
|
'timings (dict)': 'timings' in d and isinstance(d['timings'], dict),
|
|
'has Fajr time': 'Fajr' in d.get('timings', {}),
|
|
'date (string)': isinstance(d.get('date'), str) and len(d['date']) > 5,
|
|
'hijri (string)': isinstance(d.get('hijri'), str) and len(d['hijri']) > 5,
|
|
'city (string)': isinstance(d.get('city'), str) and len(d['city']) > 0,
|
|
}
|
|
all_ok = all(checks.values())
|
|
for name, ok in checks.items():
|
|
print(f' {\"✓\" if ok else \"✗\"} {name}')
|
|
sys.exit(0 if all_ok else 1)
|
|
" 2>&1; then
|
|
pass "Prayer API — shape valid"
|
|
else
|
|
fail "Prayer API — INVALID shape"
|
|
echo " Raw: $(echo "$resp" | head -c 200)"
|
|
fi
|
|
|
|
# —— Prayer API with empty params ——
|
|
sub "Prayer API — empty/missing params (should handle gracefully)"
|
|
resp=$(fetch "${BASE_URL}/api/prayer?city=&country=")
|
|
local has_err
|
|
has_err=*** "$resp" python3 -c "import sys,json; d=json.load(sys.stdin); print('error' in d)" 2>/dev/null || echo "true"
|
|
if [ "$has_err" = "True" ]; then
|
|
pass "Prayer API — returns error for empty params"
|
|
else
|
|
warn "Prayer API — no error for empty params (may silently default)"
|
|
fi
|
|
|
|
# —— Health API ——
|
|
sub "Health API — { status, uptime, timestamp }"
|
|
resp=$(fetch "${BASE_URL}/api/health")
|
|
if echo "$resp" | python3 -c "
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
assert d.get('status') == 'ok', f'status={d.get(\"status\")}'
|
|
sys.exit(0)
|
|
" 2>&1; then
|
|
pass "Health API — returns ok"
|
|
else
|
|
fail "Health API — unexpected response: $(echo "$resp" | head -c 100)"
|
|
fi
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# LAYER 3: Render Validation (content-level)
|
|
#─────────────────────────────────────────────────────────────────
|
|
layer_render() {
|
|
section "LAYER 3: Render — Page Content Validation"
|
|
echo " (Checking rendered HTML for error states, not just HTTP codes)"
|
|
|
|
local pages=(
|
|
"/prayer" "Prayer Times" "Invalid response from prayer API"
|
|
"/dhikr" "Dhikr" "Invalid response"
|
|
"/qibla" "Qibla" "Invalid response"
|
|
"/halal-monitor" "Halal Monitor" "Invalid response"
|
|
"/nur" "Nur" "Invalid response"
|
|
"/forum" "Forum" "Invalid response"
|
|
"/souq" "Souq" "Invalid response"
|
|
"/groups" "Groups" "Invalid response"
|
|
"/" "Falah" "Invalid response"
|
|
)
|
|
|
|
for ((i=0; i<${#pages[@]}; i+=3)); do
|
|
local path="${pages[$i]}"
|
|
local title="${pages[$((i+1))]}"
|
|
local error_pat="${pages[$((i+2))]}"
|
|
local content
|
|
content=$(fetch "${BASE_URL}${path}")
|
|
|
|
# Content length
|
|
local clen
|
|
clen=*** "$content" wc -c)
|
|
if [ "$clen" -lt 1000 ]; then
|
|
warn "$path — very short content (${clen}b), may be blank/error"
|
|
fi
|
|
|
|
# Error strings in rendered output
|
|
if echo "$content" | grep -qiF "$error_pat"; then
|
|
fail "$path — CONTAINS ERROR \"$error_pat\" in rendered HTML"
|
|
else
|
|
pass "$path — clean of \"$error_pat\""
|
|
fi
|
|
|
|
# Expected title
|
|
if echo "$content" | grep -qiF "$title"; then
|
|
pass "$path — contains title \"$title\""
|
|
else
|
|
fail "$path — MISSING expected title \"$title\""
|
|
fi
|
|
done
|
|
|
|
# Scan ALL pages for common error patterns
|
|
sub "Global scan — common error patterns across all pages"
|
|
local patterns=(
|
|
"Invalid response"
|
|
"Internal Server Error"
|
|
"Application error"
|
|
"Cannot read properties"
|
|
"undefined is not"
|
|
"TypeError"
|
|
"failed to fetch"
|
|
)
|
|
for pat in "${patterns[@]}"; do
|
|
local hit=0
|
|
for path in /prayer /dhikr /qibla /halal-monitor /nur /forum /souq /groups /; do
|
|
if fetch "${BASE_URL}${path}" | grep -qiF "$pat"; then
|
|
hit=$((hit+1))
|
|
fail "${path} → pattern: \"$pat\""
|
|
fi
|
|
done
|
|
[ "$hit" -eq 0 ] && pass "No page contains: \"$pat\""
|
|
done
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# LAYER 4: User Flow Testing
|
|
#─────────────────────────────────────────────────────────────────
|
|
layer_flow() {
|
|
section "LAYER 4: Flow — Multi-Step User Journeys"
|
|
|
|
# Try to get auth token via login API
|
|
sub "Flow: Login"
|
|
local login_resp token
|
|
login_resp=$(fetch -X POST -H "Content-Type: application/json" \
|
|
-d '{"email":"demo@falahos.my","password":"password123"}' \
|
|
"${BASE_URL}/api/auth/login")
|
|
token=*** "$login_resp" python3 -c "import sys,json; print(json.load(sys.stdin).get('token','NO_TOKEN'))" 2>/dev/null || echo "")
|
|
|
|
if [ -z "$token" ] || [ "$token" = "NO_TOKEN" ]; then
|
|
fail "Login — could not obtain auth token"
|
|
warn "Skipping authenticated flows"
|
|
return
|
|
fi
|
|
pass "Login — obtained auth token (${token:0:12}...)"
|
|
|
|
# —— Flow: Prayer page authenticated ——
|
|
sub "Flow: Prayer page (authenticated)"
|
|
local prayer_page
|
|
prayer_page=$(fetch -H "Authorization: Bearer ${token}" "${BASE_URL}/prayer")
|
|
if echo "$prayer_page" | grep -qF "Invalid response from prayer API"; then
|
|
fail "Prayer page — STILL shows API error when authenticated"
|
|
elif echo "$prayer_page" | grep -qF "Prayer Times"; then
|
|
pass "Prayer page — renders correctly when authenticated"
|
|
else
|
|
warn "Prayer page — unexpected content when authenticated"
|
|
fi
|
|
|
|
# —— Flow: Log a Dhikr session via API ——
|
|
sub "Flow: Dhikr counter — log session"
|
|
local dhikr_resp
|
|
dhikr_resp=$(fetch -X POST -H "Content-Type: application/json" \
|
|
-H "Authorization: Bearer ${token}" \
|
|
-d '{"type":"subhanallah","count":33,"target":33}' \
|
|
"${BASE_URL}/api/dhikr")
|
|
if echo "$dhikr_resp" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d.get('saved') or d.get('id') or d.get('success'), 'no success'; sys.exit(0)" 2>/dev/null; then
|
|
pass "Dhikr session — logged successfully"
|
|
else
|
|
warn "Dhikr session — response: $(echo "$dhikr_resp" | head -c 80)"
|
|
fi
|
|
|
|
# —— Flow: Qibla bearing calculation ——
|
|
sub "Flow: Qibla — bearing from Kuala Lumpur"
|
|
local qibla_resp
|
|
qibla_resp=$(fetch -H "Authorization: Bearer ${token}" \
|
|
"${BASE_URL}/api/qibla?lat=3.139&lng=101.6869")
|
|
if echo "$qibla_resp" | python3 -c "
|
|
import sys, json
|
|
d = json.load(sys.stdin)
|
|
b = d.get('bearing', 0)
|
|
dist = d.get('distance', 0)
|
|
assert 280 < b < 320, f'bearing {b} out of range for KL→Mecca'
|
|
assert 6000 < dist < 8000, f'distance {dist} out of range'
|
|
print(f' Bearing: {b}°, Distance: {dist} km')
|
|
sys.exit(0)
|
|
" 2>&1; then
|
|
pass "Qibla API — bearing correct for KL→Mecca"
|
|
else
|
|
fail "Qibla API — unexpected: $(echo "$qibla_resp" | head -c 100)"
|
|
fi
|
|
|
|
# —— Flow: Full journey (Prayer → Dhikr → Qibla) ——
|
|
sub "Flow: Full journey — all spiritual tools accessible"
|
|
local all_ok=0
|
|
for page in prayer dhikr qibla; do
|
|
local pg
|
|
pg=$(curl -s -o /dev/null -w '%{http_code}' \
|
|
-H "Authorization: Bearer ${token}" \
|
|
--connect-timeout 10 "${BASE_URL}/${page}" 2>/dev/null)
|
|
[ "$pg" = "200" ] && all_ok=$((all_ok+1))
|
|
done
|
|
if [ "$all_ok" -eq 3 ]; then
|
|
pass "Full journey — Prayer + Dhikr + Qibla all accessible (HTTP 200)"
|
|
else
|
|
warn "Full journey — only $all_ok/3 pages returned 200"
|
|
fi
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# LAYER 5: Edge Cases
|
|
#─────────────────────────────────────────────────────────────────
|
|
layer_edge() {
|
|
section "LAYER 5: Edge — Error States, Permissions, Resilience"
|
|
|
|
# —— Unauthenticated access to protected pages ——
|
|
sub "Edge: Protected pages redirect unauthenticated users"
|
|
local protected=("/dhikr" "/qibla" "/profile" "/wallet")
|
|
for path in "${protected[@]}"; do
|
|
local content
|
|
content=$(fetch "${BASE_URL}${path}")
|
|
if echo "$content" | grep -qiE "(login|sign in|Welcome to Falah|redirect|auth)"; then
|
|
pass "$path — redirects to auth when unauthenticated"
|
|
else
|
|
warn "$path — no auth wall detected (may expose content)"
|
|
fi
|
|
done
|
|
|
|
# —— Invalid auth token ——
|
|
sub "Edge: Invalid auth token"
|
|
local bad_resp
|
|
bad_resp=$(fetch -H "Authorization: Bearer invalid_token_here" "${BASE_URL}/api/prayer")
|
|
if echo "$bad_resp" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d.get('timings') else 1)" 2>/dev/null; then
|
|
pass "Prayer API — accessible with invalid token (public data)"
|
|
else
|
|
warn "Prayer API — rejects invalid token (may be protected)"
|
|
fi
|
|
|
|
# —— Rapid consecutive requests ——
|
|
sub "Edge: Rapid requests (10x in parallel)"
|
|
local ok=0 fail_conn=0
|
|
for i in $(seq 1 10); do
|
|
local code
|
|
code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 \
|
|
"${BASE_URL}/api/prayer?city=Kuala+Lumpur&country=MY" 2>/dev/null)
|
|
[ "$code" = "200" ] && ok=$((ok+1)) || fail_conn=$((fail_conn+1))
|
|
done
|
|
if [ "$ok" -eq 10 ]; then
|
|
pass "Rapid requests — 10/10 returned 200"
|
|
else
|
|
warn "Rapid requests — ${ok}/10 returned 200, ${fail_conn} failed"
|
|
fi
|
|
|
|
# —— Missing query params on API routes ——
|
|
sub "Edge: Missing params on various APIs"
|
|
for endpoint in "api/prayer" "api/qibla"; do
|
|
local resp
|
|
resp=$(fetch "${BASE_URL}/${endpoint}")
|
|
if echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d.get('error') else 1)" 2>/dev/null; then
|
|
pass "${endpoint} — returns error when params missing"
|
|
else
|
|
warn "${endpoint} — no error for missing params"
|
|
fi
|
|
done
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# LAYER 6: Mobile Viewport & Touch Targets
|
|
#─────────────────────────────────────────────────────────────────
|
|
layer_mobile() {
|
|
section "LAYER 6: Mobile — Responsive, Touch Targets, Text Size"
|
|
|
|
local pages=("/" "/prayer" "/dhikr" "/qibla" "/halal-monitor" "/nur")
|
|
|
|
for path in "${pages[@]}"; do
|
|
local content
|
|
content=$(fetch "${BASE_URL}${path}")
|
|
|
|
# Viewport meta
|
|
if echo "$content" | grep -qi 'name=.viewport.'; then
|
|
pass "$path — viewport meta present"
|
|
else
|
|
fail "$path — MISSING viewport meta"
|
|
fi
|
|
|
|
# Touch-friendly: min-h >= 44px
|
|
if echo "$content" | grep -qP 'min-h(?:eight)?[=:].*?(44|4[5-9]|[5-9][0-9])'; then
|
|
pass "$path — has ≥44px touch targets"
|
|
else
|
|
warn "$path — no ≥44px touch target classes detected"
|
|
fi
|
|
|
|
# Text below 12px
|
|
if echo "$content" | grep -qP 'text-\[1[0-1]px\]'; then
|
|
warn "$path — has sub-12px text (text-[10-11px])"
|
|
else
|
|
pass "$path — no sub-12px text detected"
|
|
fi
|
|
done
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# Report
|
|
#─────────────────────────────────────────────────────────────────
|
|
print_report() {
|
|
local total=$((PASS + FAIL + WARN))
|
|
local grade="A"
|
|
[ "$FAIL" -gt 0 ] && grade="B"
|
|
[ "$FAIL" -gt 3 ] && grade="C"
|
|
[ "$FAIL" -gt 10 ] && grade="D"
|
|
|
|
echo ""
|
|
echo -e "${CYAN}════════════════════════════════════════${NC}"
|
|
echo -e "${CYAN} QA TEST RESULTS — Grade: ${grade}${NC}"
|
|
echo -e "${CYAN}════════════════════════════════════════${NC}"
|
|
echo ""
|
|
echo -e " ${GREEN}Pass:${NC} $PASS ${RED}Fail:${NC} $FAIL ${YELLOW}Warn:${NC} $WARN Total: $total"
|
|
echo ""
|
|
|
|
local report_file="qa-report-$(date +%Y%m%d_%H%M%S).md"
|
|
{
|
|
echo "# QA Test Report"
|
|
echo "**Date:** $(date)"
|
|
echo "**Base URL:** $BASE_URL"
|
|
echo "**Layer:** $LAYER_FILTER"
|
|
echo ""
|
|
echo "## Summary"
|
|
echo "| ✅ Pass | ❌ Fail | ⚠️ Warn | Total | Grade |"
|
|
echo "|:------:|:------:|:------:|:----:|:-----:|"
|
|
echo "| $PASS | $FAIL | $WARN | $total | **$grade** |"
|
|
echo ""
|
|
echo "### Re-run with CI exit"
|
|
echo '```bash'
|
|
echo "./tests/qa-test-suite.sh --ci-exit"
|
|
echo '```'
|
|
} > "$report_file"
|
|
echo " Report saved to: $report_file"
|
|
|
|
if $CI_EXIT && [ "$FAIL" -gt 0 ]; then
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
#─────────────────────────────────────────────────────────────────
|
|
# Main
|
|
#─────────────────────────────────────────────────────────────────
|
|
echo "╔══════════════════════════════════════════════════════╗"
|
|
echo "║ Falah Mobile — Comprehensive QA Test Suite ║"
|
|
echo "║ $(date) ║"
|
|
echo "╚══════════════════════════════════════════════════════╝"
|
|
echo " Layer: $LAYER_FILTER URL: $BASE_URL"
|
|
|
|
should_run "system" && layer_system
|
|
should_run "api" && layer_api
|
|
should_run "render" && layer_render
|
|
should_run "flow" && layer_flow
|
|
should_run "edge" && layer_edge
|
|
should_run "mobile" && layer_mobile
|
|
|
|
print_report
|