Error Feedback Service + friendly error handling across all pages
- Error Feedback Service: POST /api/feedback saves to JSONL, GET with admin token - ErrorFeedback component: warm amber tones, friendly messages, 'Report Issue' button - Replaced ALL red-themed error displays across 15+ pages with ErrorFeedback - Kind classification: network, auth, location, upgrade, default messages - QA test suite v2: 8-layer testing (system, API, render, scenario, edge, mobile, perf, security) - Browser-based visual QA with Playwright (Layer 9)
This commit is contained in:
root
@@ -0,0 +1,836 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Falah Mobile — Comprehensive QA Test Suite v2
|
||||
=============================================
|
||||
Multi-layer, contract-driven, user-scenario-based testing.
|
||||
|
||||
Usage:
|
||||
python3 tests/qa-test-suite.py # Full suite
|
||||
python3 tests/qa-test-suite.py --layer api # Single layer
|
||||
python3 tests/qa-test-suite.py --ci-exit # Exit 1 on failures
|
||||
python3 tests/qa-test-suite.py --url https://staging.falahos.my/mobile
|
||||
|
||||
Layers (run in order; each builds on the previous):
|
||||
1. SYSTEM — HTTP status, route existence, redirect chain
|
||||
2. API — Response shape contracts (strict schema validation)
|
||||
3. RENDER — Parse HTML; assert no error strings, expected content present
|
||||
4. SCENARIO — Real user journeys (first-time, returning, premium, etc.)
|
||||
5. EDGE — Missing params, invalid tokens, rapid fire, offline
|
||||
6. MOBILE — Viewport, touch targets ≥44px, text ≥12px, tap targets
|
||||
7. PERF — Response times, page weight, Time to First Byte
|
||||
8. SECURITY — Headers, no secrets in HTML, HSTS, CSP
|
||||
"""
|
||||
|
||||
import argparse, json, os, re, sys, time, urllib.request, urllib.error
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from html.parser import HTMLParser
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Config
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile")
|
||||
TIMEOUT = 20
|
||||
PASS, FAIL, WARN = 0, 0, 0
|
||||
REPORT_LINES = []
|
||||
CI_EXIT = False
|
||||
|
||||
G = "\033[0;32m"; R = "\033[0;31m"; Y = "\033[1;33m"
|
||||
C = "\033[0;36m"; N = "\033[0m"
|
||||
|
||||
def ok(msg): global PASS; PASS += 1; print(f" {G}✓{N} {msg}"); REPORT_LINES.append(("PASS", msg))
|
||||
def fail(msg): global FAIL; FAIL += 1; print(f" {R}✗{N} {msg}"); REPORT_LINES.append(("FAIL", msg))
|
||||
def warn(msg): global WARN; WARN += 1; print(f" {Y}⚠{N} {msg}"); REPORT_LINES.append(("WARN", msg))
|
||||
|
||||
def section(title):
|
||||
print(f"\n{C}════════════════════════════════════════════════════════════{N}")
|
||||
print(f"{C} {title}{N}")
|
||||
print(f"{C}════════════════════════════════════════════════════════════{N}")
|
||||
|
||||
def sub(title):
|
||||
print(f"\n{C}── {title} ──{N}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# HTTP helper
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def req(path: str, method="GET", headers=None, body=None, timeout=TIMEOUT) -> dict:
|
||||
"""Make HTTP request, return {status, headers, text, elapsed}."""
|
||||
url = f"{BASE_URL.rstrip('/')}/{path.lstrip('/')}"
|
||||
hdrs = {"User-Agent": "Falah-QA/2.0", "Accept": "*/*"}
|
||||
if headers:
|
||||
hdrs.update(headers)
|
||||
data = body.encode() if body else None
|
||||
t0 = time.time()
|
||||
try:
|
||||
req = urllib.request.Request(url, data=data, headers=hdrs, method=method)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
return {
|
||||
"status": resp.status,
|
||||
"headers": dict(resp.headers),
|
||||
"text": text,
|
||||
"elapsed": time.time() - t0,
|
||||
}
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
return {
|
||||
"status": e.code,
|
||||
"headers": dict(e.headers),
|
||||
"text": raw.decode("utf-8", errors="replace"),
|
||||
"elapsed": time.time() - t0,
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": 0, "headers": {}, "text": str(e), "elapsed": time.time() - t0}
|
||||
|
||||
def json_req(path, method="GET", headers=None, body=None) -> dict:
|
||||
r = req(path, method, headers, body)
|
||||
try:
|
||||
r["json"] = json.loads(r["text"])
|
||||
except:
|
||||
r["json"] = None
|
||||
return r
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# HTML Content Parser (lightweight)
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
class ContentSniffer(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.texts = []
|
||||
self.scripts = []
|
||||
self.in_script = False
|
||||
self.meta_viewport = False
|
||||
self.has_44px = False
|
||||
self.has_small_text = False
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
ad = dict(attrs)
|
||||
if tag == "meta" and ad.get("name", "").lower() == "viewport":
|
||||
self.meta_viewport = True
|
||||
if tag == "script":
|
||||
self.in_script = True
|
||||
# Check inline styles for min-height ≥44px
|
||||
style = ad.get("style", "")
|
||||
cls = ad.get("class", "")
|
||||
for s in [style, cls]:
|
||||
if re.search(r'min-h(?:eight)?[=:].*?(44|4[5-9]|[5-9]\d)', s):
|
||||
self.has_44px = True
|
||||
if re.search(r'text-\[1[0-1]px\]', cls):
|
||||
self.has_small_text = True
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == "script":
|
||||
self.in_script = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_script:
|
||||
self.scripts.append(data)
|
||||
else:
|
||||
self.texts.append(data)
|
||||
|
||||
def sniff_html(html: str) -> ContentSniffer:
|
||||
s = ContentSniffer()
|
||||
try:
|
||||
s.feed(html)
|
||||
except:
|
||||
pass
|
||||
return s
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 1: SYSTEM
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_system():
|
||||
section("LAYER 1: SYSTEM — Route Existence & HTTP Status")
|
||||
|
||||
routes = [
|
||||
("/", "Landing page", {200, 308}),
|
||||
("/prayer", "Prayer times", {200}),
|
||||
("/dhikr", "Dhikr counter", {200}),
|
||||
("/qibla", "Qibla finder", {200}),
|
||||
("/halal-monitor", "Halal monitor", {200}),
|
||||
("/nur", "Nur AI", {200}),
|
||||
("/forum", "Forum", {200}),
|
||||
("/souq", "Souq", {200}),
|
||||
("/groups", "Groups", {200}),
|
||||
("/api/health", "Health API endpoint", {200}),
|
||||
("/api/prayer", "Prayer API endpoint", {200}),
|
||||
]
|
||||
|
||||
for path, label, expected in routes:
|
||||
r = req(path)
|
||||
code = r["status"]
|
||||
if code == 0:
|
||||
fail(f"{label} ({path}) — CONNECTION FAILED: {r['text'][:80]}")
|
||||
elif code in expected:
|
||||
ok(f"{label} ({path}) → {code} in {r['elapsed']*1000:.0f}ms")
|
||||
else:
|
||||
fail(f"{label} ({path}) → {code} (expected {expected})")
|
||||
|
||||
# Redirect chain check: /mobile/ should redirect to /mobile
|
||||
sub("Redirect integrity")
|
||||
r = req("/")
|
||||
if r["status"] == 308:
|
||||
loc = r["headers"].get("Location", "")
|
||||
ok(f"Landing redirects (308 → {loc})")
|
||||
elif r["status"] == 200:
|
||||
ok("Landing returns 200 directly")
|
||||
else:
|
||||
warn(f"Landing returned {r['status']}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 2: API Response Contracts
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_api():
|
||||
section("LAYER 2: API — Response Shape Contracts")
|
||||
|
||||
# ── Prayer API ──
|
||||
sub("Contract: Prayer API")
|
||||
r = json_req("api/prayer?city=Kuala+Lumpur&country=MY")
|
||||
if r["status"] != 200:
|
||||
fail(f"Prayer API → {r['status']}")
|
||||
return
|
||||
|
||||
j = r["json"]
|
||||
if not j:
|
||||
fail("Prayer API — not JSON")
|
||||
return
|
||||
|
||||
# Shape contract
|
||||
contract = {
|
||||
"timings": dict,
|
||||
"date": str,
|
||||
"hijri": str,
|
||||
"city": str,
|
||||
"country": str,
|
||||
"timezone": str,
|
||||
}
|
||||
violations = []
|
||||
for field, ftype in contract.items():
|
||||
if field not in j:
|
||||
violations.append(f"missing '{field}'")
|
||||
elif not isinstance(j[field], ftype):
|
||||
violations.append(f"'{field}' type={type(j[field]).__name__} (expected {ftype.__name__})")
|
||||
|
||||
if violations:
|
||||
fail(f"Prayer API contract violations: {'; '.join(violations)}")
|
||||
else:
|
||||
ok("Prayer API — full contract satisfied")
|
||||
|
||||
# Value-level checks
|
||||
if isinstance(j.get("timings"), dict):
|
||||
required_times = {"Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"}
|
||||
missing_times = required_times - set(j["timings"].keys())
|
||||
if missing_times:
|
||||
fail(f"Prayer API — missing prayer times: {missing_times}")
|
||||
else:
|
||||
ok(f"Prayer API — all 5 prayer times present")
|
||||
|
||||
# Time format check: "HH:MM (TZ)"
|
||||
import re as _re
|
||||
for p in required_times:
|
||||
t = j["timings"].get(p, "")
|
||||
if not _re.match(r'\d{2}:\d{2}\s*\([A-Z]+\)', t):
|
||||
warn(f"Prayer API — '{p}' time format unexpected: '{t}'")
|
||||
else:
|
||||
fail("Prayer API — timings not a dict")
|
||||
|
||||
# Date format checks
|
||||
if isinstance(j.get("date"), str):
|
||||
ok(f"Prayer API — date: {j['date']}")
|
||||
if isinstance(j.get("hijri"), str):
|
||||
ok(f"Prayer API — hijri: {j['hijri']}")
|
||||
|
||||
# ── Prayer API: empty params ──
|
||||
sub("Contract: Prayer API (empty params)")
|
||||
r2 = json_req("api/prayer?city=&country=")
|
||||
if r2["json"] and r2["json"].get("error"):
|
||||
ok("Prayer API — returns error for empty params")
|
||||
elif r2["json"] and r2["json"].get("timings"):
|
||||
warn("Prayer API — no error for empty params, returned default data")
|
||||
else:
|
||||
warn(f"Prayer API (empty) — unexpected: {str(r2['json'])[:80]}")
|
||||
|
||||
# ── Health API ──
|
||||
sub("Contract: Health API")
|
||||
r3 = json_req("api/health")
|
||||
if r3["json"] and r3["json"].get("status") == "ok":
|
||||
ok("Health API — status=ok")
|
||||
else:
|
||||
fail(f"Health API — unexpected: {str(r3['json'])[:80]}")
|
||||
|
||||
# ── Ummah ID Health ──
|
||||
sub("Contract: Ummah ID Health")
|
||||
r4 = json_req("../auth/health") # Traefik route
|
||||
if r4["json"] and r4["json"].get("status") == "ok":
|
||||
ok("Ummah ID Health — status=ok")
|
||||
elif r4["status"] != 0:
|
||||
warn(f"Ummah ID Health → {r4['status']}")
|
||||
else:
|
||||
ok("Ummah ID Health — available (expected for SSO route)")
|
||||
# This may 404 from mobile base path; that's fine
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 3: RENDER VALIDATION
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
ERROR_PATTERNS = [
|
||||
"Invalid response",
|
||||
"Internal Server Error",
|
||||
"Application error",
|
||||
"Cannot read properties of",
|
||||
"undefined is not",
|
||||
"TypeError",
|
||||
"Failed to fetch",
|
||||
"not found",
|
||||
"500 Internal",
|
||||
"JSON Parse error",
|
||||
"Unexpected token",
|
||||
"Cannot find module",
|
||||
"Minified React error",
|
||||
]
|
||||
|
||||
def layer_render():
|
||||
section("LAYER 3: RENDER — HTML Content Validation")
|
||||
|
||||
pages = [
|
||||
("/", "Landing", ["Falah", "Assalamualaikum", "Daily Verse", "Quick Actions"]),
|
||||
("/prayer", "Prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"]),
|
||||
("/dhikr", "Dhikr", ["Dhikr", "Digital Tasbih", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]),
|
||||
("/qibla", "Qibla", ["Qibla Finder", "Kaaba", "bearing"]),
|
||||
("/halal-monitor", "Halal Monitor", ["Halal Monitor", "Restaurants"]),
|
||||
("/nur", "Nur", ["Nur"]),
|
||||
("/forum", "Forum", ["Forum"]),
|
||||
("/souq", "Souq", ["Souq"]),
|
||||
("/groups", "Groups", ["Groups"]),
|
||||
]
|
||||
|
||||
for path, label, expected_strings in pages:
|
||||
r = req(path)
|
||||
html = r["text"]
|
||||
snif = sniff_html(html)
|
||||
content_text = " ".join(snif.texts)
|
||||
|
||||
# 1. Content length check
|
||||
if len(html) < 1000:
|
||||
warn(f"{label} ({path}) — suspiciously small ({len(html)} bytes)")
|
||||
continue
|
||||
|
||||
# 2. Scan for error patterns
|
||||
found_errors = []
|
||||
for pat in ERROR_PATTERNS:
|
||||
if re.search(pat, html, re.IGNORECASE):
|
||||
found_errors.append(pat)
|
||||
|
||||
if found_errors:
|
||||
fail(f"{label} ({path}) — ERRORS in HTML: {', '.join(found_errors)}")
|
||||
# Show context around the first error
|
||||
for pat in found_errors:
|
||||
idx = html.lower().find(pat.lower())
|
||||
if idx >= 0:
|
||||
ctx = html[max(0, idx-40):idx+len(pat)+60].replace('\n', ' ')
|
||||
print(f" Context: ...{ctx}...")
|
||||
else:
|
||||
ok(f"{label} ({path}) — clean (no error patterns)")
|
||||
|
||||
# 3. Check expected content
|
||||
for es in expected_strings:
|
||||
if es.lower() in content_text.lower():
|
||||
ok(f"{label} ({path}) — contains \"{es}\"")
|
||||
else:
|
||||
fail(f"{label} ({path}) — MISSING expected text: \"{es}\"")
|
||||
|
||||
# ── Full-page cross-check: scan EVERY page for ALL error patterns ──
|
||||
sub("Cross-page: global error pattern scan")
|
||||
all_paths = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"]
|
||||
for pat in ERROR_PATTERNS:
|
||||
hits = []
|
||||
for p in all_paths:
|
||||
html = req(p)["text"]
|
||||
if re.search(pat, html, re.IGNORECASE):
|
||||
hits.append(p)
|
||||
if hits:
|
||||
fail(f"Pattern \"{pat}\" found on: {', '.join(hits)}")
|
||||
else:
|
||||
ok(f"Pattern \"{pat}\" — absent from all pages")
|
||||
|
||||
# ── Invisible text/content (empty divs, loading spinners stuck) ──
|
||||
sub("Content quality: loading states, empty containers")
|
||||
for path in ["/prayer", "/dhikr", "/qibla"]:
|
||||
html = req(path)["text"]
|
||||
# Count loading spinners
|
||||
spinners = len(re.findall(r'(animate-spin|loading|Loader|spinner)', html, re.IGNORECASE))
|
||||
if spinners > 5:
|
||||
warn(f"{path} — {spinners} loading spinners detected (might be stuck)")
|
||||
else:
|
||||
ok(f"{path} — {spinners} loading indicators (acceptable)")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 4: USER SCENARIOS
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_scenario():
|
||||
section("LAYER 4: SCENARIO — Real User Journeys")
|
||||
|
||||
# ── Scenario A: First-time visitor (unauthenticated) ──
|
||||
sub("Scenario A: First-time visitor (unauthenticated)")
|
||||
paths = ["/", "/prayer", "/qibla"]
|
||||
for path in paths:
|
||||
r = req(path)
|
||||
if r["status"] == 200:
|
||||
html = r["text"]
|
||||
# Should see some content, not just a blank auth wall
|
||||
if "login" in html.lower() or "welcome to falah" in html.lower():
|
||||
ok(f"Unauthenticated {path} — shows auth wall (expected)")
|
||||
else:
|
||||
ok(f"Unauthenticated {path} — shows content")
|
||||
else:
|
||||
warn(f"Unauthenticated {path} → {r['status']}")
|
||||
|
||||
# ── Scenario B: Login flow ──
|
||||
sub("Scenario B: Login flow")
|
||||
login_res = json_req("api/auth/login", "POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
body=json.dumps({"email": "demo@falahos.my", "password": "password123"}))
|
||||
token = None
|
||||
if login_res["json"]:
|
||||
token = login_res["json"].get("token")
|
||||
if token:
|
||||
ok(f"Login — obtained token ({token[:16]}...{token[-4:]})")
|
||||
else:
|
||||
fail(f"Login — no token in response: {str(login_res['json'])[:100]}")
|
||||
else:
|
||||
fail(f"Login — HTTP {login_res['status']}: {login_res['text'][:100]}")
|
||||
|
||||
if not token:
|
||||
warn("Skipping authenticated scenarios")
|
||||
return
|
||||
|
||||
auth_hdrs = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
# ── Scenario C: Authenticated session ──
|
||||
sub("Scenario C: Authenticated user browsing")
|
||||
for path, expected in [
|
||||
("/", ["Assalamualaikum", "Falah"]),
|
||||
("/prayer", ["Prayer Times", "Fajr"]),
|
||||
("/dhikr", ["Dhikr", "Counter"]),
|
||||
("/qibla", ["Qibla", "Kaaba"]),
|
||||
]:
|
||||
r = req(path, headers=auth_hdrs)
|
||||
html = r["text"]
|
||||
content = " ".join(sniff_html(html).texts)
|
||||
missing = [e for e in expected if e.lower() not in content.lower()]
|
||||
if missing:
|
||||
fail(f"Authenticated {path} — missing: {missing}")
|
||||
elif r["status"] != 200:
|
||||
warn(f"Authenticated {path} → {r['status']}")
|
||||
else:
|
||||
ok(f"Authenticated {path} — all expected content present")
|
||||
|
||||
# ── Scenario D: Log a Dhikr session ──
|
||||
sub("Scenario D: Log a Dhikr session")
|
||||
dhikr_res = json_req("api/dhikr", "POST",
|
||||
headers={"Content-Type": "application/json", **auth_hdrs},
|
||||
body=json.dumps({"type": "subhanallah", "count": 33, "target": 33}))
|
||||
if dhikr_res["json"]:
|
||||
if dhikr_res["json"].get("saved") or dhikr_res["json"].get("id") or dhikr_res["json"].get("success"):
|
||||
ok("Dhikr session logged successfully")
|
||||
else:
|
||||
ok(f"Dhikr session — response: {str(dhikr_res['json'])[:80]}")
|
||||
else:
|
||||
warn(f"Dhikr session — HTTP {dhikr_res['status']}")
|
||||
|
||||
# ── Scenario E: Get Dhikr stats ──
|
||||
sub("Scenario E: Retrieve Dhikr stats")
|
||||
stats_res = json_req("api/dhikr/stats", headers=auth_hdrs)
|
||||
if stats_res["json"]:
|
||||
ok(f"Dhikr stats — received data: {str(stats_res['json'])[:100]}")
|
||||
else:
|
||||
warn(f"Dhikr stats — HTTP {stats_res['status']}")
|
||||
|
||||
# ── Scenario F: Qibla API ──
|
||||
sub("Scenario F: Qibla bearing calculation")
|
||||
qibla_res = json_req("api/qibla?lat=3.139&lng=101.6869", headers=auth_hdrs)
|
||||
j = qibla_res["json"]
|
||||
if j and isinstance(j.get("bearing"), (int, float)) and isinstance(j.get("distance"), (int, float)):
|
||||
bearing = j["bearing"]
|
||||
distance = j["distance"]
|
||||
if 280 < bearing < 320:
|
||||
ok(f"Qibla bearing from KL → Mecca: {bearing:.1f}° (correct range)")
|
||||
else:
|
||||
fail(f"Qibla bearing out of range: {bearing}° (expected ~298°)")
|
||||
if 6000 < distance < 8000:
|
||||
ok(f"Qibla distance KL → Mecca: {distance:.0f} km (correct range)")
|
||||
else:
|
||||
fail(f"Qibla distance out of range: {distance} km (expected ~7100)")
|
||||
else:
|
||||
fail(f"Qibla API — unexpected: {str(j)[:100]}")
|
||||
|
||||
# ── Scenario G: Full journey — access all spiritual tools in sequence ──
|
||||
sub("Scenario G: Full spiritual tools journey")
|
||||
journey_pages = ["/prayer", "/dhikr", "/qibla"]
|
||||
all_ok = True
|
||||
for page in journey_pages:
|
||||
r = req(page, headers=auth_hdrs)
|
||||
if r["status"] == 200:
|
||||
# Check no errors
|
||||
if any(re.search(p, r["text"], re.IGNORECASE) for p in ERROR_PATTERNS):
|
||||
fail(f"{page} — errors present (see Layer 3)")
|
||||
all_ok = False
|
||||
else:
|
||||
ok(f"{page} — accessible and clean")
|
||||
else:
|
||||
fail(f"{page} → {r['status']}")
|
||||
all_ok = False
|
||||
if all_ok:
|
||||
ok("Full journey (Prayer → Dhikr → Qibla) — ALL pages clean")
|
||||
|
||||
# ── Scenario H: Location change ──
|
||||
sub("Scenario H: Prayer times for different city")
|
||||
for city, country, label in [
|
||||
("Makkah", "SA", "Mecca"),
|
||||
("Jakarta", "ID", "Jakarta"),
|
||||
("London", "GB", "London"),
|
||||
]:
|
||||
r = json_req(f"api/prayer?city={city}&country={country}")
|
||||
if r["json"] and r["json"].get("timings"):
|
||||
ok(f"Prayer times for {label} — loaded")
|
||||
else:
|
||||
fail(f"Prayer times for {label} — failed: {str(r['json'])[:80]}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 5: EDGE CASES
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_edge():
|
||||
section("LAYER 5: EDGE — Error Handling, Resilience")
|
||||
|
||||
# ── Invalid auth token ──
|
||||
sub("Edge: Invalid auth token")
|
||||
bad = "Bearer eyJ...INVALID...SIG"
|
||||
for path in ["api/prayer", "api/dhikr/stats"]:
|
||||
r = json_req(path, headers={"Authorization": bad})
|
||||
# Public endpoints should still work; protected ones should 401
|
||||
if r["status"] in (200, 401):
|
||||
ok(f"{path} with bad token → {r['status']} (graceful)")
|
||||
else:
|
||||
warn(f"{path} with bad token → {r['status']} (unexpected)")
|
||||
|
||||
# ── Missing required params ──
|
||||
sub("Edge: Missing query parameters")
|
||||
for path in ["api/prayer", "api/qibla"]:
|
||||
r = json_req(path)
|
||||
if r["json"] and r["json"].get("error"):
|
||||
ok(f"{path} — returns error when params missing")
|
||||
elif r["status"] in (400, 422, 500):
|
||||
ok(f"{path} → {r['status']} (error for missing params)")
|
||||
elif r["status"] == 200:
|
||||
warn(f"{path} — 200 with missing params (may silently default)")
|
||||
else:
|
||||
warn(f"{path} → {r['status']} (unexpected)")
|
||||
|
||||
# ── Special characters in params ──
|
||||
sub("Edge: Special characters in city names")
|
||||
specials = [
|
||||
("New York", "US", "space in name"),
|
||||
("São Paulo", "BR", "accented chars"),
|
||||
("Köln", "DE", "umlaut"),
|
||||
("Kuala Lumpur", "MY", "normal ref"),
|
||||
]
|
||||
for city, country, label in specials:
|
||||
r = json_req(f"api/prayer?city={urllib.parse.quote(city)}&country={country}")
|
||||
if r["json"] and r["json"].get("timings"):
|
||||
ok(f"Prayer API — handles '{label}': {city}")
|
||||
else:
|
||||
warn(f"Prayer API — '{label}' failed: {str(r['json'])[:80]}")
|
||||
|
||||
# ── Rapid-fire / concurrency ──
|
||||
sub("Edge: Rapid requests (burst test)")
|
||||
import threading
|
||||
results = []
|
||||
def fire(n):
|
||||
r = req("api/prayer?city=Kuala+Lumpur&country=MY")
|
||||
results.append((n, r["status"], r["elapsed"]))
|
||||
|
||||
threads = [threading.Thread(target=fire, args=(i,)) for i in range(20)]
|
||||
t0 = time.time()
|
||||
for t in threads: t.start()
|
||||
for t in threads: t.join()
|
||||
elapsed = time.time() - t0
|
||||
|
||||
statuses = [s for _, s, _ in results]
|
||||
ok_count = sum(1 for s in statuses if s == 200)
|
||||
fail_count = sum(1 for s in statuses if s != 200)
|
||||
times = [e for _, _, e in results]
|
||||
avg_ms = sum(times) / len(times) * 1000
|
||||
|
||||
if ok_count == 20:
|
||||
ok(f"Burst test — 20/20 returned 200 (avg {avg_ms:.0f}ms, total {elapsed:.1f}s)")
|
||||
else:
|
||||
warn(f"Burst test — {ok_count}/20 returned 200, {fail_count} failed (avg {avg_ms:.0f}ms)")
|
||||
|
||||
# ── Negative: non-existent route ──
|
||||
sub("Edge: Non-existent routes return 404")
|
||||
for path in ["/this-does-not-exist", "/api/nonexistent", "/mobile/undefined-route"]:
|
||||
r = req(path)
|
||||
if r["status"] == 404:
|
||||
ok(f"{path} → 404 (correct)")
|
||||
else:
|
||||
warn(f"{path} → {r['status']} (expected 404)")
|
||||
|
||||
# ── Timeout / large payload ──
|
||||
sub("Edge: Response time sanity")
|
||||
for path in ["/", "/prayer", "/dhikr", "/api/prayer?city=Kuala+Lumpur&country=MY"]:
|
||||
r = req(path, timeout=10)
|
||||
ms = r["elapsed"] * 1000
|
||||
if ms > 3000:
|
||||
warn(f"{path} — slow ({ms:.0f}ms)")
|
||||
elif ms > 10000:
|
||||
fail(f"{path} — timeout risk ({ms:.0f}ms)")
|
||||
else:
|
||||
ok(f"{path} — {ms:.0f}ms")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 6: MOBILE UX
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_mobile():
|
||||
section("LAYER 6: MOBILE UX — Touch, Text, Responsive Design")
|
||||
|
||||
pages = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"]
|
||||
|
||||
# ── Viewport meta ──
|
||||
sub("Mobile: Viewport meta tag")
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
if 'name="viewport"' in html or "name='viewport'" in html:
|
||||
ok(f"{path} — viewport meta present")
|
||||
else:
|
||||
fail(f"{path} — MISSING viewport meta tag")
|
||||
|
||||
# ── Touch targets (min-height ≥44px) ──
|
||||
sub("Mobile: Touch targets ≥44px")
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
# Check for 44px patterns
|
||||
has_44 = bool(re.search(r'min-h[^}]*?(?:44|4[5-9]|[5-9]\d)', html, re.IGNORECASE))
|
||||
has_min_h = 'min-h' in html.lower()
|
||||
if has_44:
|
||||
ok(f"{path} — ≥44px touch targets detected")
|
||||
elif has_min_h:
|
||||
warn(f"{path} — has min-h classes but none ≥44px detected")
|
||||
else:
|
||||
warn(f"{path} — no explicit min-h classes found (check CSS approach)")
|
||||
|
||||
# ── Text size ≥12px ──
|
||||
sub("Mobile: No sub-12px text")
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
sub_12 = re.findall(r'text-\[(1[0-1]px)\]', html)
|
||||
if sub_12:
|
||||
sizes = list(set(sub_12))
|
||||
warn(f"{path} — sub-12px text found: {sizes}")
|
||||
else:
|
||||
ok(f"{path} — no sub-12px text")
|
||||
|
||||
# ── Bottom navigation check ──
|
||||
sub("Mobile: Bottom navigation present")
|
||||
for path in ["/", "/prayer", "/dhikr", "/qibla"]:
|
||||
html = req(path)["text"]
|
||||
nav_links = ["Nur", "Souq", "Prayer", "Forum", "Wallet", "Profile"]
|
||||
found = [n for n in nav_links if n.lower() in html.lower()]
|
||||
if len(found) >= 5:
|
||||
ok(f"{path} — bottom nav with {len(found)}/6 links")
|
||||
else:
|
||||
warn(f"{path} — bottom nav incomplete ({len(found)}/6)")
|
||||
|
||||
# ── Touch spacing: tap targets not touching ──
|
||||
sub("Mobile: Basic tap target spacing")
|
||||
for path in ["/prayer", "/dhikr"]:
|
||||
html = req(path)["text"]
|
||||
buttons = re.findall(r'<button[^>]*>', html)
|
||||
if len(buttons) > 0:
|
||||
ok(f"{path} — {len(buttons)} tap targets found")
|
||||
else:
|
||||
warn(f"{path} — no <button> elements found (might use divs)")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 7: PERFORMANCE
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_perf():
|
||||
section("LAYER 7: PERFORMANCE — Response Times & Payload Size")
|
||||
|
||||
pages = [
|
||||
("/", "Landing"),
|
||||
("/prayer", "Prayer"),
|
||||
("/dhikr", "Dhikr"),
|
||||
("/qibla", "Qibla"),
|
||||
("/halal-monitor", "Halal Monitor"),
|
||||
("/api/prayer?city=Kuala+Lumpur&country=MY", "Prayer API (cached)"),
|
||||
]
|
||||
|
||||
for path, label in pages:
|
||||
# First request (cold)
|
||||
cold = req(path)
|
||||
cold_ms = cold["elapsed"] * 1000
|
||||
|
||||
# Second request (warm)
|
||||
warm = req(path)
|
||||
warm_ms = warm["elapsed"] * 1000
|
||||
|
||||
size_kb = len(cold["text"]) / 1024
|
||||
|
||||
status = ok
|
||||
notes = []
|
||||
if cold_ms > 3000:
|
||||
status = warn
|
||||
notes.append(f"cold={cold_ms:.0f}ms")
|
||||
if warm_ms > 1500:
|
||||
status = warn
|
||||
notes.append(f"warm={warm_ms:.0f}ms")
|
||||
if size_kb > 500:
|
||||
status = warn
|
||||
notes.append(f"size={size_kb:.0f}KB")
|
||||
if not notes:
|
||||
notes.append(f"cold={cold_ms:.0f}ms warm={warm_ms:.0f}ms")
|
||||
status(f"{label} ({path}) — {'; '.join(notes)}, {size_kb:.0f}KB")
|
||||
|
||||
sub("Performance: API response times")
|
||||
for endpoint, label in [
|
||||
("api/prayer?city=Kuala+Lumpur&country=MY", "Prayer API"),
|
||||
("api/health", "Health API"),
|
||||
]:
|
||||
r = req(endpoint)
|
||||
ms = r["elapsed"] * 1000
|
||||
if ms > 2000:
|
||||
warn(f"{label} — {ms:.0f}ms (slow)")
|
||||
else:
|
||||
ok(f"{label} — {ms:.0f}ms")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 8: SECURITY BASICS
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_security():
|
||||
section("LAYER 8: SECURITY — Headers, Secrets in HTML")
|
||||
|
||||
# ── Security headers ──
|
||||
sub("Security: Response headers")
|
||||
r = req("/")
|
||||
hdrs = r["headers"]
|
||||
checks = [
|
||||
("Content-Type", lambda v: "text/html" in v if v else False),
|
||||
("X-Content-Type-Options", lambda v: v == "nosniff" if v else False),
|
||||
]
|
||||
for hname, check in checks:
|
||||
val = hdrs.get(hname)
|
||||
if val and check(val):
|
||||
ok(f"{hname}: {val}")
|
||||
elif val:
|
||||
warn(f"{hname}: {val} (unexpected value)")
|
||||
else:
|
||||
warn(f"{hname}: MISSING")
|
||||
|
||||
# ── Secrets in HTML ──
|
||||
sub("Security: No secrets leaked in HTML")
|
||||
secret_patterns = [
|
||||
(r'eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+', 'JWT token'),
|
||||
(r'ghp_[A-Za-z0-9]{36}', 'GitHub PAT'),
|
||||
(r'sk-[A-Za-z0-9]{32,}', 'OpenAI key'),
|
||||
(r'-----BEGIN (RSA |EC )?PRIVATE KEY-----', 'Private key'),
|
||||
(r'flh-[a-z0-9]{3,}', 'FLH secret pattern'),
|
||||
]
|
||||
pages = ["/", "/prayer", "/dhikr", "/qibla"]
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
for pat, desc in secret_patterns:
|
||||
matches = re.findall(pat, html)
|
||||
if matches:
|
||||
fail(f"{path} — possible {desc} leak: {matches[0][:20]}...")
|
||||
else:
|
||||
ok(f"{path} — no {desc} leak")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# MAIN
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def main():
|
||||
global BASE_URL, CI_EXIT
|
||||
parser = argparse.ArgumentParser(description="Falah Mobile QA Test Suite")
|
||||
parser.add_argument("--layer", choices=["system", "api", "render", "scenario", "edge", "mobile", "perf", "security", "all"], default="all")
|
||||
parser.add_argument("--ci-exit", action="store_true")
|
||||
parser.add_argument("--url", default=BASE_URL)
|
||||
args = parser.parse_args()
|
||||
BASE_URL = args.url.rstrip("/")
|
||||
CI_EXIT = args.ci_exit
|
||||
|
||||
print(f"{'='*60}")
|
||||
print(f" Falah Mobile — QA Test Suite v2")
|
||||
print(f" URL: {BASE_URL}")
|
||||
print(f" Layer: {args.layer}")
|
||||
print(f" Time: {time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
import urllib.parse # for special chars test
|
||||
|
||||
layers = {
|
||||
"system": layer_system,
|
||||
"api": layer_api,
|
||||
"render": layer_render,
|
||||
"scenario": layer_scenario,
|
||||
"edge": layer_edge,
|
||||
"mobile": layer_mobile,
|
||||
"perf": layer_perf,
|
||||
"security": layer_security,
|
||||
}
|
||||
|
||||
if args.layer == "all":
|
||||
for name, fn in layers.items():
|
||||
try:
|
||||
fn()
|
||||
except Exception as e:
|
||||
print(f" {R}✗{N} Layer '{name}' crashed: {e}")
|
||||
import traceback; traceback.print_exc()
|
||||
else:
|
||||
fn = layers.get(args.layer)
|
||||
if fn:
|
||||
fn()
|
||||
else:
|
||||
print(f"Unknown layer: {args.layer}")
|
||||
sys.exit(1)
|
||||
|
||||
total = PASS + FAIL + WARN
|
||||
grade = "A"
|
||||
if FAIL > 0: grade = "B"
|
||||
if FAIL > 3: grade = "C"
|
||||
if FAIL > 10: grade = "D"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" QA RESULTS — Grade: {grade}")
|
||||
print(f"{'='*60}")
|
||||
print(f" {G}Pass:{N} {PASS} {R}Fail:{N} {FAIL} {Y}Warn:{N} {WARN} Total: {total}")
|
||||
|
||||
# Generate report
|
||||
report = f"""# QA Test Report
|
||||
**Date:** {time.strftime('%Y-%m-%d %H:%M:%S')}
|
||||
**URL:** {BASE_URL}
|
||||
**Layer:** {args.layer}
|
||||
|
||||
## Summary
|
||||
| ✅ Pass | ❌ Fail | ⚠️ Warn | Total | Grade |
|
||||
|:------:|:------:|:------:|:----:|:-----:|
|
||||
| {PASS} | {FAIL} | {WARN} | {total} | **{grade}** |
|
||||
|
||||
## Details
|
||||
| Result | Message |
|
||||
|:------|:--------|
|
||||
"""
|
||||
for result, msg in REPORT_LINES:
|
||||
icon = {"PASS": "✅", "FAIL": "❌", "WARN": "⚠️"}.get(result, "•")
|
||||
report += f"| {icon} | {msg} |\n"
|
||||
|
||||
report += f"\n## Re-run\n```bash\npython3 tests/qa-test-suite.py{' --ci-exit' if CI_EXIT else ''}\n```"
|
||||
report_file = f"qa-report-{time.strftime('%Y%m%d_%H%M%S')}.md"
|
||||
with open(report_file, "w") as f:
|
||||
f.write(report)
|
||||
print(f" Report: {report_file}")
|
||||
|
||||
if CI_EXIT and FAIL > 0:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user