root
14d7127e41
- Error Feedback Service: POST /api/feedback saves to JSONL, GET with admin token - ErrorFeedback component: warm amber tones, friendly messages, 'Report Issue' button - Replaced ALL red-themed error displays across 15+ pages with ErrorFeedback - Kind classification: network, auth, location, upgrade, default messages - QA test suite v2: 8-layer testing (system, API, render, scenario, edge, mobile, perf, security) - Browser-based visual QA with Playwright (Layer 9)
837 lines
34 KiB
Python
837 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Falah Mobile — Comprehensive QA Test Suite v2
|
|
=============================================
|
|
Multi-layer, contract-driven, user-scenario-based testing.
|
|
|
|
Usage:
|
|
python3 tests/qa-test-suite.py # Full suite
|
|
python3 tests/qa-test-suite.py --layer api # Single layer
|
|
python3 tests/qa-test-suite.py --ci-exit # Exit 1 on failures
|
|
python3 tests/qa-test-suite.py --url https://staging.falahos.my/mobile
|
|
|
|
Layers (run in order; each builds on the previous):
|
|
1. SYSTEM — HTTP status, route existence, redirect chain
|
|
2. API — Response shape contracts (strict schema validation)
|
|
3. RENDER — Parse HTML; assert no error strings, expected content present
|
|
4. SCENARIO — Real user journeys (first-time, returning, premium, etc.)
|
|
5. EDGE — Missing params, invalid tokens, rapid fire, offline
|
|
6. MOBILE — Viewport, touch targets ≥44px, text ≥12px, tap targets
|
|
7. PERF — Response times, page weight, Time to First Byte
|
|
8. SECURITY — Headers, no secrets in HTML, HSTS, CSP
|
|
"""
|
|
|
|
import argparse, json, os, re, sys, time, urllib.request, urllib.error
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional
|
|
from html.parser import HTMLParser
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# Config
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile")
|
|
TIMEOUT = 20
|
|
PASS, FAIL, WARN = 0, 0, 0
|
|
REPORT_LINES = []
|
|
CI_EXIT = False
|
|
|
|
G = "\033[0;32m"; R = "\033[0;31m"; Y = "\033[1;33m"
|
|
C = "\033[0;36m"; N = "\033[0m"
|
|
|
|
def ok(msg): global PASS; PASS += 1; print(f" {G}✓{N} {msg}"); REPORT_LINES.append(("PASS", msg))
|
|
def fail(msg): global FAIL; FAIL += 1; print(f" {R}✗{N} {msg}"); REPORT_LINES.append(("FAIL", msg))
|
|
def warn(msg): global WARN; WARN += 1; print(f" {Y}⚠{N} {msg}"); REPORT_LINES.append(("WARN", msg))
|
|
|
|
def section(title):
|
|
print(f"\n{C}════════════════════════════════════════════════════════════{N}")
|
|
print(f"{C} {title}{N}")
|
|
print(f"{C}════════════════════════════════════════════════════════════{N}")
|
|
|
|
def sub(title):
|
|
print(f"\n{C}── {title} ──{N}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# HTTP helper
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def req(path: str, method="GET", headers=None, body=None, timeout=TIMEOUT) -> dict:
|
|
"""Make HTTP request, return {status, headers, text, elapsed}."""
|
|
url = f"{BASE_URL.rstrip('/')}/{path.lstrip('/')}"
|
|
hdrs = {"User-Agent": "Falah-QA/2.0", "Accept": "*/*"}
|
|
if headers:
|
|
hdrs.update(headers)
|
|
data = body.encode() if body else None
|
|
t0 = time.time()
|
|
try:
|
|
req = urllib.request.Request(url, data=data, headers=hdrs, method=method)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
raw = resp.read()
|
|
text = raw.decode("utf-8", errors="replace")
|
|
return {
|
|
"status": resp.status,
|
|
"headers": dict(resp.headers),
|
|
"text": text,
|
|
"elapsed": time.time() - t0,
|
|
}
|
|
except urllib.error.HTTPError as e:
|
|
raw = e.read()
|
|
return {
|
|
"status": e.code,
|
|
"headers": dict(e.headers),
|
|
"text": raw.decode("utf-8", errors="replace"),
|
|
"elapsed": time.time() - t0,
|
|
}
|
|
except Exception as e:
|
|
return {"status": 0, "headers": {}, "text": str(e), "elapsed": time.time() - t0}
|
|
|
|
def json_req(path, method="GET", headers=None, body=None) -> dict:
|
|
r = req(path, method, headers, body)
|
|
try:
|
|
r["json"] = json.loads(r["text"])
|
|
except:
|
|
r["json"] = None
|
|
return r
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# HTML Content Parser (lightweight)
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
class ContentSniffer(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.texts = []
|
|
self.scripts = []
|
|
self.in_script = False
|
|
self.meta_viewport = False
|
|
self.has_44px = False
|
|
self.has_small_text = False
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
ad = dict(attrs)
|
|
if tag == "meta" and ad.get("name", "").lower() == "viewport":
|
|
self.meta_viewport = True
|
|
if tag == "script":
|
|
self.in_script = True
|
|
# Check inline styles for min-height ≥44px
|
|
style = ad.get("style", "")
|
|
cls = ad.get("class", "")
|
|
for s in [style, cls]:
|
|
if re.search(r'min-h(?:eight)?[=:].*?(44|4[5-9]|[5-9]\d)', s):
|
|
self.has_44px = True
|
|
if re.search(r'text-\[1[0-1]px\]', cls):
|
|
self.has_small_text = True
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag == "script":
|
|
self.in_script = False
|
|
|
|
def handle_data(self, data):
|
|
if self.in_script:
|
|
self.scripts.append(data)
|
|
else:
|
|
self.texts.append(data)
|
|
|
|
def sniff_html(html: str) -> ContentSniffer:
|
|
s = ContentSniffer()
|
|
try:
|
|
s.feed(html)
|
|
except:
|
|
pass
|
|
return s
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 1: SYSTEM
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_system():
|
|
section("LAYER 1: SYSTEM — Route Existence & HTTP Status")
|
|
|
|
routes = [
|
|
("/", "Landing page", {200, 308}),
|
|
("/prayer", "Prayer times", {200}),
|
|
("/dhikr", "Dhikr counter", {200}),
|
|
("/qibla", "Qibla finder", {200}),
|
|
("/halal-monitor", "Halal monitor", {200}),
|
|
("/nur", "Nur AI", {200}),
|
|
("/forum", "Forum", {200}),
|
|
("/souq", "Souq", {200}),
|
|
("/groups", "Groups", {200}),
|
|
("/api/health", "Health API endpoint", {200}),
|
|
("/api/prayer", "Prayer API endpoint", {200}),
|
|
]
|
|
|
|
for path, label, expected in routes:
|
|
r = req(path)
|
|
code = r["status"]
|
|
if code == 0:
|
|
fail(f"{label} ({path}) — CONNECTION FAILED: {r['text'][:80]}")
|
|
elif code in expected:
|
|
ok(f"{label} ({path}) → {code} in {r['elapsed']*1000:.0f}ms")
|
|
else:
|
|
fail(f"{label} ({path}) → {code} (expected {expected})")
|
|
|
|
# Redirect chain check: /mobile/ should redirect to /mobile
|
|
sub("Redirect integrity")
|
|
r = req("/")
|
|
if r["status"] == 308:
|
|
loc = r["headers"].get("Location", "")
|
|
ok(f"Landing redirects (308 → {loc})")
|
|
elif r["status"] == 200:
|
|
ok("Landing returns 200 directly")
|
|
else:
|
|
warn(f"Landing returned {r['status']}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 2: API Response Contracts
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_api():
|
|
section("LAYER 2: API — Response Shape Contracts")
|
|
|
|
# ── Prayer API ──
|
|
sub("Contract: Prayer API")
|
|
r = json_req("api/prayer?city=Kuala+Lumpur&country=MY")
|
|
if r["status"] != 200:
|
|
fail(f"Prayer API → {r['status']}")
|
|
return
|
|
|
|
j = r["json"]
|
|
if not j:
|
|
fail("Prayer API — not JSON")
|
|
return
|
|
|
|
# Shape contract
|
|
contract = {
|
|
"timings": dict,
|
|
"date": str,
|
|
"hijri": str,
|
|
"city": str,
|
|
"country": str,
|
|
"timezone": str,
|
|
}
|
|
violations = []
|
|
for field, ftype in contract.items():
|
|
if field not in j:
|
|
violations.append(f"missing '{field}'")
|
|
elif not isinstance(j[field], ftype):
|
|
violations.append(f"'{field}' type={type(j[field]).__name__} (expected {ftype.__name__})")
|
|
|
|
if violations:
|
|
fail(f"Prayer API contract violations: {'; '.join(violations)}")
|
|
else:
|
|
ok("Prayer API — full contract satisfied")
|
|
|
|
# Value-level checks
|
|
if isinstance(j.get("timings"), dict):
|
|
required_times = {"Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"}
|
|
missing_times = required_times - set(j["timings"].keys())
|
|
if missing_times:
|
|
fail(f"Prayer API — missing prayer times: {missing_times}")
|
|
else:
|
|
ok(f"Prayer API — all 5 prayer times present")
|
|
|
|
# Time format check: "HH:MM (TZ)"
|
|
import re as _re
|
|
for p in required_times:
|
|
t = j["timings"].get(p, "")
|
|
if not _re.match(r'\d{2}:\d{2}\s*\([A-Z]+\)', t):
|
|
warn(f"Prayer API — '{p}' time format unexpected: '{t}'")
|
|
else:
|
|
fail("Prayer API — timings not a dict")
|
|
|
|
# Date format checks
|
|
if isinstance(j.get("date"), str):
|
|
ok(f"Prayer API — date: {j['date']}")
|
|
if isinstance(j.get("hijri"), str):
|
|
ok(f"Prayer API — hijri: {j['hijri']}")
|
|
|
|
# ── Prayer API: empty params ──
|
|
sub("Contract: Prayer API (empty params)")
|
|
r2 = json_req("api/prayer?city=&country=")
|
|
if r2["json"] and r2["json"].get("error"):
|
|
ok("Prayer API — returns error for empty params")
|
|
elif r2["json"] and r2["json"].get("timings"):
|
|
warn("Prayer API — no error for empty params, returned default data")
|
|
else:
|
|
warn(f"Prayer API (empty) — unexpected: {str(r2['json'])[:80]}")
|
|
|
|
# ── Health API ──
|
|
sub("Contract: Health API")
|
|
r3 = json_req("api/health")
|
|
if r3["json"] and r3["json"].get("status") == "ok":
|
|
ok("Health API — status=ok")
|
|
else:
|
|
fail(f"Health API — unexpected: {str(r3['json'])[:80]}")
|
|
|
|
# ── Ummah ID Health ──
|
|
sub("Contract: Ummah ID Health")
|
|
r4 = json_req("../auth/health") # Traefik route
|
|
if r4["json"] and r4["json"].get("status") == "ok":
|
|
ok("Ummah ID Health — status=ok")
|
|
elif r4["status"] != 0:
|
|
warn(f"Ummah ID Health → {r4['status']}")
|
|
else:
|
|
ok("Ummah ID Health — available (expected for SSO route)")
|
|
# This may 404 from mobile base path; that's fine
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 3: RENDER VALIDATION
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
ERROR_PATTERNS = [
|
|
"Invalid response",
|
|
"Internal Server Error",
|
|
"Application error",
|
|
"Cannot read properties of",
|
|
"undefined is not",
|
|
"TypeError",
|
|
"Failed to fetch",
|
|
"not found",
|
|
"500 Internal",
|
|
"JSON Parse error",
|
|
"Unexpected token",
|
|
"Cannot find module",
|
|
"Minified React error",
|
|
]
|
|
|
|
def layer_render():
|
|
section("LAYER 3: RENDER — HTML Content Validation")
|
|
|
|
pages = [
|
|
("/", "Landing", ["Falah", "Assalamualaikum", "Daily Verse", "Quick Actions"]),
|
|
("/prayer", "Prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"]),
|
|
("/dhikr", "Dhikr", ["Dhikr", "Digital Tasbih", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]),
|
|
("/qibla", "Qibla", ["Qibla Finder", "Kaaba", "bearing"]),
|
|
("/halal-monitor", "Halal Monitor", ["Halal Monitor", "Restaurants"]),
|
|
("/nur", "Nur", ["Nur"]),
|
|
("/forum", "Forum", ["Forum"]),
|
|
("/souq", "Souq", ["Souq"]),
|
|
("/groups", "Groups", ["Groups"]),
|
|
]
|
|
|
|
for path, label, expected_strings in pages:
|
|
r = req(path)
|
|
html = r["text"]
|
|
snif = sniff_html(html)
|
|
content_text = " ".join(snif.texts)
|
|
|
|
# 1. Content length check
|
|
if len(html) < 1000:
|
|
warn(f"{label} ({path}) — suspiciously small ({len(html)} bytes)")
|
|
continue
|
|
|
|
# 2. Scan for error patterns
|
|
found_errors = []
|
|
for pat in ERROR_PATTERNS:
|
|
if re.search(pat, html, re.IGNORECASE):
|
|
found_errors.append(pat)
|
|
|
|
if found_errors:
|
|
fail(f"{label} ({path}) — ERRORS in HTML: {', '.join(found_errors)}")
|
|
# Show context around the first error
|
|
for pat in found_errors:
|
|
idx = html.lower().find(pat.lower())
|
|
if idx >= 0:
|
|
ctx = html[max(0, idx-40):idx+len(pat)+60].replace('\n', ' ')
|
|
print(f" Context: ...{ctx}...")
|
|
else:
|
|
ok(f"{label} ({path}) — clean (no error patterns)")
|
|
|
|
# 3. Check expected content
|
|
for es in expected_strings:
|
|
if es.lower() in content_text.lower():
|
|
ok(f"{label} ({path}) — contains \"{es}\"")
|
|
else:
|
|
fail(f"{label} ({path}) — MISSING expected text: \"{es}\"")
|
|
|
|
# ── Full-page cross-check: scan EVERY page for ALL error patterns ──
|
|
sub("Cross-page: global error pattern scan")
|
|
all_paths = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"]
|
|
for pat in ERROR_PATTERNS:
|
|
hits = []
|
|
for p in all_paths:
|
|
html = req(p)["text"]
|
|
if re.search(pat, html, re.IGNORECASE):
|
|
hits.append(p)
|
|
if hits:
|
|
fail(f"Pattern \"{pat}\" found on: {', '.join(hits)}")
|
|
else:
|
|
ok(f"Pattern \"{pat}\" — absent from all pages")
|
|
|
|
# ── Invisible text/content (empty divs, loading spinners stuck) ──
|
|
sub("Content quality: loading states, empty containers")
|
|
for path in ["/prayer", "/dhikr", "/qibla"]:
|
|
html = req(path)["text"]
|
|
# Count loading spinners
|
|
spinners = len(re.findall(r'(animate-spin|loading|Loader|spinner)', html, re.IGNORECASE))
|
|
if spinners > 5:
|
|
warn(f"{path} — {spinners} loading spinners detected (might be stuck)")
|
|
else:
|
|
ok(f"{path} — {spinners} loading indicators (acceptable)")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 4: USER SCENARIOS
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_scenario():
|
|
section("LAYER 4: SCENARIO — Real User Journeys")
|
|
|
|
# ── Scenario A: First-time visitor (unauthenticated) ──
|
|
sub("Scenario A: First-time visitor (unauthenticated)")
|
|
paths = ["/", "/prayer", "/qibla"]
|
|
for path in paths:
|
|
r = req(path)
|
|
if r["status"] == 200:
|
|
html = r["text"]
|
|
# Should see some content, not just a blank auth wall
|
|
if "login" in html.lower() or "welcome to falah" in html.lower():
|
|
ok(f"Unauthenticated {path} — shows auth wall (expected)")
|
|
else:
|
|
ok(f"Unauthenticated {path} — shows content")
|
|
else:
|
|
warn(f"Unauthenticated {path} → {r['status']}")
|
|
|
|
# ── Scenario B: Login flow ──
|
|
sub("Scenario B: Login flow")
|
|
login_res = json_req("api/auth/login", "POST",
|
|
headers={"Content-Type": "application/json"},
|
|
body=json.dumps({"email": "demo@falahos.my", "password": "password123"}))
|
|
token = None
|
|
if login_res["json"]:
|
|
token = login_res["json"].get("token")
|
|
if token:
|
|
ok(f"Login — obtained token ({token[:16]}...{token[-4:]})")
|
|
else:
|
|
fail(f"Login — no token in response: {str(login_res['json'])[:100]}")
|
|
else:
|
|
fail(f"Login — HTTP {login_res['status']}: {login_res['text'][:100]}")
|
|
|
|
if not token:
|
|
warn("Skipping authenticated scenarios")
|
|
return
|
|
|
|
auth_hdrs = {"Authorization": f"Bearer {token}"}
|
|
|
|
# ── Scenario C: Authenticated session ──
|
|
sub("Scenario C: Authenticated user browsing")
|
|
for path, expected in [
|
|
("/", ["Assalamualaikum", "Falah"]),
|
|
("/prayer", ["Prayer Times", "Fajr"]),
|
|
("/dhikr", ["Dhikr", "Counter"]),
|
|
("/qibla", ["Qibla", "Kaaba"]),
|
|
]:
|
|
r = req(path, headers=auth_hdrs)
|
|
html = r["text"]
|
|
content = " ".join(sniff_html(html).texts)
|
|
missing = [e for e in expected if e.lower() not in content.lower()]
|
|
if missing:
|
|
fail(f"Authenticated {path} — missing: {missing}")
|
|
elif r["status"] != 200:
|
|
warn(f"Authenticated {path} → {r['status']}")
|
|
else:
|
|
ok(f"Authenticated {path} — all expected content present")
|
|
|
|
# ── Scenario D: Log a Dhikr session ──
|
|
sub("Scenario D: Log a Dhikr session")
|
|
dhikr_res = json_req("api/dhikr", "POST",
|
|
headers={"Content-Type": "application/json", **auth_hdrs},
|
|
body=json.dumps({"type": "subhanallah", "count": 33, "target": 33}))
|
|
if dhikr_res["json"]:
|
|
if dhikr_res["json"].get("saved") or dhikr_res["json"].get("id") or dhikr_res["json"].get("success"):
|
|
ok("Dhikr session logged successfully")
|
|
else:
|
|
ok(f"Dhikr session — response: {str(dhikr_res['json'])[:80]}")
|
|
else:
|
|
warn(f"Dhikr session — HTTP {dhikr_res['status']}")
|
|
|
|
# ── Scenario E: Get Dhikr stats ──
|
|
sub("Scenario E: Retrieve Dhikr stats")
|
|
stats_res = json_req("api/dhikr/stats", headers=auth_hdrs)
|
|
if stats_res["json"]:
|
|
ok(f"Dhikr stats — received data: {str(stats_res['json'])[:100]}")
|
|
else:
|
|
warn(f"Dhikr stats — HTTP {stats_res['status']}")
|
|
|
|
# ── Scenario F: Qibla API ──
|
|
sub("Scenario F: Qibla bearing calculation")
|
|
qibla_res = json_req("api/qibla?lat=3.139&lng=101.6869", headers=auth_hdrs)
|
|
j = qibla_res["json"]
|
|
if j and isinstance(j.get("bearing"), (int, float)) and isinstance(j.get("distance"), (int, float)):
|
|
bearing = j["bearing"]
|
|
distance = j["distance"]
|
|
if 280 < bearing < 320:
|
|
ok(f"Qibla bearing from KL → Mecca: {bearing:.1f}° (correct range)")
|
|
else:
|
|
fail(f"Qibla bearing out of range: {bearing}° (expected ~298°)")
|
|
if 6000 < distance < 8000:
|
|
ok(f"Qibla distance KL → Mecca: {distance:.0f} km (correct range)")
|
|
else:
|
|
fail(f"Qibla distance out of range: {distance} km (expected ~7100)")
|
|
else:
|
|
fail(f"Qibla API — unexpected: {str(j)[:100]}")
|
|
|
|
# ── Scenario G: Full journey — access all spiritual tools in sequence ──
|
|
sub("Scenario G: Full spiritual tools journey")
|
|
journey_pages = ["/prayer", "/dhikr", "/qibla"]
|
|
all_ok = True
|
|
for page in journey_pages:
|
|
r = req(page, headers=auth_hdrs)
|
|
if r["status"] == 200:
|
|
# Check no errors
|
|
if any(re.search(p, r["text"], re.IGNORECASE) for p in ERROR_PATTERNS):
|
|
fail(f"{page} — errors present (see Layer 3)")
|
|
all_ok = False
|
|
else:
|
|
ok(f"{page} — accessible and clean")
|
|
else:
|
|
fail(f"{page} → {r['status']}")
|
|
all_ok = False
|
|
if all_ok:
|
|
ok("Full journey (Prayer → Dhikr → Qibla) — ALL pages clean")
|
|
|
|
# ── Scenario H: Location change ──
|
|
sub("Scenario H: Prayer times for different city")
|
|
for city, country, label in [
|
|
("Makkah", "SA", "Mecca"),
|
|
("Jakarta", "ID", "Jakarta"),
|
|
("London", "GB", "London"),
|
|
]:
|
|
r = json_req(f"api/prayer?city={city}&country={country}")
|
|
if r["json"] and r["json"].get("timings"):
|
|
ok(f"Prayer times for {label} — loaded")
|
|
else:
|
|
fail(f"Prayer times for {label} — failed: {str(r['json'])[:80]}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 5: EDGE CASES
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_edge():
|
|
section("LAYER 5: EDGE — Error Handling, Resilience")
|
|
|
|
# ── Invalid auth token ──
|
|
sub("Edge: Invalid auth token")
|
|
bad = "Bearer eyJ...INVALID...SIG"
|
|
for path in ["api/prayer", "api/dhikr/stats"]:
|
|
r = json_req(path, headers={"Authorization": bad})
|
|
# Public endpoints should still work; protected ones should 401
|
|
if r["status"] in (200, 401):
|
|
ok(f"{path} with bad token → {r['status']} (graceful)")
|
|
else:
|
|
warn(f"{path} with bad token → {r['status']} (unexpected)")
|
|
|
|
# ── Missing required params ──
|
|
sub("Edge: Missing query parameters")
|
|
for path in ["api/prayer", "api/qibla"]:
|
|
r = json_req(path)
|
|
if r["json"] and r["json"].get("error"):
|
|
ok(f"{path} — returns error when params missing")
|
|
elif r["status"] in (400, 422, 500):
|
|
ok(f"{path} → {r['status']} (error for missing params)")
|
|
elif r["status"] == 200:
|
|
warn(f"{path} — 200 with missing params (may silently default)")
|
|
else:
|
|
warn(f"{path} → {r['status']} (unexpected)")
|
|
|
|
# ── Special characters in params ──
|
|
sub("Edge: Special characters in city names")
|
|
specials = [
|
|
("New York", "US", "space in name"),
|
|
("São Paulo", "BR", "accented chars"),
|
|
("Köln", "DE", "umlaut"),
|
|
("Kuala Lumpur", "MY", "normal ref"),
|
|
]
|
|
for city, country, label in specials:
|
|
r = json_req(f"api/prayer?city={urllib.parse.quote(city)}&country={country}")
|
|
if r["json"] and r["json"].get("timings"):
|
|
ok(f"Prayer API — handles '{label}': {city}")
|
|
else:
|
|
warn(f"Prayer API — '{label}' failed: {str(r['json'])[:80]}")
|
|
|
|
# ── Rapid-fire / concurrency ──
|
|
sub("Edge: Rapid requests (burst test)")
|
|
import threading
|
|
results = []
|
|
def fire(n):
|
|
r = req("api/prayer?city=Kuala+Lumpur&country=MY")
|
|
results.append((n, r["status"], r["elapsed"]))
|
|
|
|
threads = [threading.Thread(target=fire, args=(i,)) for i in range(20)]
|
|
t0 = time.time()
|
|
for t in threads: t.start()
|
|
for t in threads: t.join()
|
|
elapsed = time.time() - t0
|
|
|
|
statuses = [s for _, s, _ in results]
|
|
ok_count = sum(1 for s in statuses if s == 200)
|
|
fail_count = sum(1 for s in statuses if s != 200)
|
|
times = [e for _, _, e in results]
|
|
avg_ms = sum(times) / len(times) * 1000
|
|
|
|
if ok_count == 20:
|
|
ok(f"Burst test — 20/20 returned 200 (avg {avg_ms:.0f}ms, total {elapsed:.1f}s)")
|
|
else:
|
|
warn(f"Burst test — {ok_count}/20 returned 200, {fail_count} failed (avg {avg_ms:.0f}ms)")
|
|
|
|
# ── Negative: non-existent route ──
|
|
sub("Edge: Non-existent routes return 404")
|
|
for path in ["/this-does-not-exist", "/api/nonexistent", "/mobile/undefined-route"]:
|
|
r = req(path)
|
|
if r["status"] == 404:
|
|
ok(f"{path} → 404 (correct)")
|
|
else:
|
|
warn(f"{path} → {r['status']} (expected 404)")
|
|
|
|
# ── Timeout / large payload ──
|
|
sub("Edge: Response time sanity")
|
|
for path in ["/", "/prayer", "/dhikr", "/api/prayer?city=Kuala+Lumpur&country=MY"]:
|
|
r = req(path, timeout=10)
|
|
ms = r["elapsed"] * 1000
|
|
if ms > 3000:
|
|
warn(f"{path} — slow ({ms:.0f}ms)")
|
|
elif ms > 10000:
|
|
fail(f"{path} — timeout risk ({ms:.0f}ms)")
|
|
else:
|
|
ok(f"{path} — {ms:.0f}ms")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 6: MOBILE UX
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_mobile():
|
|
section("LAYER 6: MOBILE UX — Touch, Text, Responsive Design")
|
|
|
|
pages = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"]
|
|
|
|
# ── Viewport meta ──
|
|
sub("Mobile: Viewport meta tag")
|
|
for path in pages:
|
|
html = req(path)["text"]
|
|
if 'name="viewport"' in html or "name='viewport'" in html:
|
|
ok(f"{path} — viewport meta present")
|
|
else:
|
|
fail(f"{path} — MISSING viewport meta tag")
|
|
|
|
# ── Touch targets (min-height ≥44px) ──
|
|
sub("Mobile: Touch targets ≥44px")
|
|
for path in pages:
|
|
html = req(path)["text"]
|
|
# Check for 44px patterns
|
|
has_44 = bool(re.search(r'min-h[^}]*?(?:44|4[5-9]|[5-9]\d)', html, re.IGNORECASE))
|
|
has_min_h = 'min-h' in html.lower()
|
|
if has_44:
|
|
ok(f"{path} — ≥44px touch targets detected")
|
|
elif has_min_h:
|
|
warn(f"{path} — has min-h classes but none ≥44px detected")
|
|
else:
|
|
warn(f"{path} — no explicit min-h classes found (check CSS approach)")
|
|
|
|
# ── Text size ≥12px ──
|
|
sub("Mobile: No sub-12px text")
|
|
for path in pages:
|
|
html = req(path)["text"]
|
|
sub_12 = re.findall(r'text-\[(1[0-1]px)\]', html)
|
|
if sub_12:
|
|
sizes = list(set(sub_12))
|
|
warn(f"{path} — sub-12px text found: {sizes}")
|
|
else:
|
|
ok(f"{path} — no sub-12px text")
|
|
|
|
# ── Bottom navigation check ──
|
|
sub("Mobile: Bottom navigation present")
|
|
for path in ["/", "/prayer", "/dhikr", "/qibla"]:
|
|
html = req(path)["text"]
|
|
nav_links = ["Nur", "Souq", "Prayer", "Forum", "Wallet", "Profile"]
|
|
found = [n for n in nav_links if n.lower() in html.lower()]
|
|
if len(found) >= 5:
|
|
ok(f"{path} — bottom nav with {len(found)}/6 links")
|
|
else:
|
|
warn(f"{path} — bottom nav incomplete ({len(found)}/6)")
|
|
|
|
# ── Touch spacing: tap targets not touching ──
|
|
sub("Mobile: Basic tap target spacing")
|
|
for path in ["/prayer", "/dhikr"]:
|
|
html = req(path)["text"]
|
|
buttons = re.findall(r'<button[^>]*>', html)
|
|
if len(buttons) > 0:
|
|
ok(f"{path} — {len(buttons)} tap targets found")
|
|
else:
|
|
warn(f"{path} — no <button> elements found (might use divs)")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 7: PERFORMANCE
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_perf():
|
|
section("LAYER 7: PERFORMANCE — Response Times & Payload Size")
|
|
|
|
pages = [
|
|
("/", "Landing"),
|
|
("/prayer", "Prayer"),
|
|
("/dhikr", "Dhikr"),
|
|
("/qibla", "Qibla"),
|
|
("/halal-monitor", "Halal Monitor"),
|
|
("/api/prayer?city=Kuala+Lumpur&country=MY", "Prayer API (cached)"),
|
|
]
|
|
|
|
for path, label in pages:
|
|
# First request (cold)
|
|
cold = req(path)
|
|
cold_ms = cold["elapsed"] * 1000
|
|
|
|
# Second request (warm)
|
|
warm = req(path)
|
|
warm_ms = warm["elapsed"] * 1000
|
|
|
|
size_kb = len(cold["text"]) / 1024
|
|
|
|
status = ok
|
|
notes = []
|
|
if cold_ms > 3000:
|
|
status = warn
|
|
notes.append(f"cold={cold_ms:.0f}ms")
|
|
if warm_ms > 1500:
|
|
status = warn
|
|
notes.append(f"warm={warm_ms:.0f}ms")
|
|
if size_kb > 500:
|
|
status = warn
|
|
notes.append(f"size={size_kb:.0f}KB")
|
|
if not notes:
|
|
notes.append(f"cold={cold_ms:.0f}ms warm={warm_ms:.0f}ms")
|
|
status(f"{label} ({path}) — {'; '.join(notes)}, {size_kb:.0f}KB")
|
|
|
|
sub("Performance: API response times")
|
|
for endpoint, label in [
|
|
("api/prayer?city=Kuala+Lumpur&country=MY", "Prayer API"),
|
|
("api/health", "Health API"),
|
|
]:
|
|
r = req(endpoint)
|
|
ms = r["elapsed"] * 1000
|
|
if ms > 2000:
|
|
warn(f"{label} — {ms:.0f}ms (slow)")
|
|
else:
|
|
ok(f"{label} — {ms:.0f}ms")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# LAYER 8: SECURITY BASICS
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def layer_security():
|
|
section("LAYER 8: SECURITY — Headers, Secrets in HTML")
|
|
|
|
# ── Security headers ──
|
|
sub("Security: Response headers")
|
|
r = req("/")
|
|
hdrs = r["headers"]
|
|
checks = [
|
|
("Content-Type", lambda v: "text/html" in v if v else False),
|
|
("X-Content-Type-Options", lambda v: v == "nosniff" if v else False),
|
|
]
|
|
for hname, check in checks:
|
|
val = hdrs.get(hname)
|
|
if val and check(val):
|
|
ok(f"{hname}: {val}")
|
|
elif val:
|
|
warn(f"{hname}: {val} (unexpected value)")
|
|
else:
|
|
warn(f"{hname}: MISSING")
|
|
|
|
# ── Secrets in HTML ──
|
|
sub("Security: No secrets leaked in HTML")
|
|
secret_patterns = [
|
|
(r'eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+', 'JWT token'),
|
|
(r'ghp_[A-Za-z0-9]{36}', 'GitHub PAT'),
|
|
(r'sk-[A-Za-z0-9]{32,}', 'OpenAI key'),
|
|
(r'-----BEGIN (RSA |EC )?PRIVATE KEY-----', 'Private key'),
|
|
(r'flh-[a-z0-9]{3,}', 'FLH secret pattern'),
|
|
]
|
|
pages = ["/", "/prayer", "/dhikr", "/qibla"]
|
|
for path in pages:
|
|
html = req(path)["text"]
|
|
for pat, desc in secret_patterns:
|
|
matches = re.findall(pat, html)
|
|
if matches:
|
|
fail(f"{path} — possible {desc} leak: {matches[0][:20]}...")
|
|
else:
|
|
ok(f"{path} — no {desc} leak")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# MAIN
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
def main():
|
|
global BASE_URL, CI_EXIT
|
|
parser = argparse.ArgumentParser(description="Falah Mobile QA Test Suite")
|
|
parser.add_argument("--layer", choices=["system", "api", "render", "scenario", "edge", "mobile", "perf", "security", "all"], default="all")
|
|
parser.add_argument("--ci-exit", action="store_true")
|
|
parser.add_argument("--url", default=BASE_URL)
|
|
args = parser.parse_args()
|
|
BASE_URL = args.url.rstrip("/")
|
|
CI_EXIT = args.ci_exit
|
|
|
|
print(f"{'='*60}")
|
|
print(f" Falah Mobile — QA Test Suite v2")
|
|
print(f" URL: {BASE_URL}")
|
|
print(f" Layer: {args.layer}")
|
|
print(f" Time: {time.strftime('%Y-%m-%d %H:%M:%S')}")
|
|
print(f"{'='*60}")
|
|
|
|
import urllib.parse # for special chars test
|
|
|
|
layers = {
|
|
"system": layer_system,
|
|
"api": layer_api,
|
|
"render": layer_render,
|
|
"scenario": layer_scenario,
|
|
"edge": layer_edge,
|
|
"mobile": layer_mobile,
|
|
"perf": layer_perf,
|
|
"security": layer_security,
|
|
}
|
|
|
|
if args.layer == "all":
|
|
for name, fn in layers.items():
|
|
try:
|
|
fn()
|
|
except Exception as e:
|
|
print(f" {R}✗{N} Layer '{name}' crashed: {e}")
|
|
import traceback; traceback.print_exc()
|
|
else:
|
|
fn = layers.get(args.layer)
|
|
if fn:
|
|
fn()
|
|
else:
|
|
print(f"Unknown layer: {args.layer}")
|
|
sys.exit(1)
|
|
|
|
total = PASS + FAIL + WARN
|
|
grade = "A"
|
|
if FAIL > 0: grade = "B"
|
|
if FAIL > 3: grade = "C"
|
|
if FAIL > 10: grade = "D"
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f" QA RESULTS — Grade: {grade}")
|
|
print(f"{'='*60}")
|
|
print(f" {G}Pass:{N} {PASS} {R}Fail:{N} {FAIL} {Y}Warn:{N} {WARN} Total: {total}")
|
|
|
|
# Generate report
|
|
report = f"""# QA Test Report
|
|
**Date:** {time.strftime('%Y-%m-%d %H:%M:%S')}
|
|
**URL:** {BASE_URL}
|
|
**Layer:** {args.layer}
|
|
|
|
## Summary
|
|
| ✅ Pass | ❌ Fail | ⚠️ Warn | Total | Grade |
|
|
|:------:|:------:|:------:|:----:|:-----:|
|
|
| {PASS} | {FAIL} | {WARN} | {total} | **{grade}** |
|
|
|
|
## Details
|
|
| Result | Message |
|
|
|:------|:--------|
|
|
"""
|
|
for result, msg in REPORT_LINES:
|
|
icon = {"PASS": "✅", "FAIL": "❌", "WARN": "⚠️"}.get(result, "•")
|
|
report += f"| {icon} | {msg} |\n"
|
|
|
|
report += f"\n## Re-run\n```bash\npython3 tests/qa-test-suite.py{' --ci-exit' if CI_EXIT else ''}\n```"
|
|
report_file = f"qa-report-{time.strftime('%Y%m%d_%H%M%S')}.md"
|
|
with open(report_file, "w") as f:
|
|
f.write(report)
|
|
print(f" Report: {report_file}")
|
|
|
|
if CI_EXIT and FAIL > 0:
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|