#!/usr/bin/env python3 """ Falah Mobile — Comprehensive QA Test Suite v2 ============================================= Multi-layer, contract-driven, user-scenario-based testing. Usage: python3 tests/qa-test-suite.py # Full suite python3 tests/qa-test-suite.py --layer api # Single layer python3 tests/qa-test-suite.py --ci-exit # Exit 1 on failures python3 tests/qa-test-suite.py --url https://staging.falahos.my/mobile Layers (run in order; each builds on the previous): 1. SYSTEM — HTTP status, route existence, redirect chain 2. API — Response shape contracts (strict schema validation) 3. RENDER — Parse HTML; assert no error strings, expected content present 4. SCENARIO — Real user journeys (first-time, returning, premium, etc.) 5. EDGE — Missing params, invalid tokens, rapid fire, offline 6. MOBILE — Viewport, touch targets ≥44px, text ≥12px, tap targets 7. PERF — Response times, page weight, Time to First Byte 8. SECURITY — Headers, no secrets in HTML, HSTS, CSP """ import argparse, json, os, re, sys, time, urllib.request, urllib.error from dataclasses import dataclass, field from typing import Optional from html.parser import HTMLParser # ────────────────────────────────────────────────────────────────────── # Config # ────────────────────────────────────────────────────────────────────── BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile") TIMEOUT = 20 PASS, FAIL, WARN = 0, 0, 0 REPORT_LINES = [] CI_EXIT = False G = "\033[0;32m"; R = "\033[0;31m"; Y = "\033[1;33m" C = "\033[0;36m"; N = "\033[0m" def ok(msg): global PASS; PASS += 1; print(f" {G}✓{N} {msg}"); REPORT_LINES.append(("PASS", msg)) def fail(msg): global FAIL; FAIL += 1; print(f" {R}✗{N} {msg}"); REPORT_LINES.append(("FAIL", msg)) def warn(msg): global WARN; WARN += 1; print(f" {Y}⚠{N} {msg}"); REPORT_LINES.append(("WARN", msg)) def section(title): print(f"\n{C}════════════════════════════════════════════════════════════{N}") print(f"{C} {title}{N}") print(f"{C}════════════════════════════════════════════════════════════{N}") def sub(title): print(f"\n{C}── {title} ──{N}") # ────────────────────────────────────────────────────────────────────── # HTTP helper # ────────────────────────────────────────────────────────────────────── def req(path: str, method="GET", headers=None, body=None, timeout=TIMEOUT) -> dict: """Make HTTP request, return {status, headers, text, elapsed}.""" url = f"{BASE_URL.rstrip('/')}/{path.lstrip('/')}" hdrs = {"User-Agent": "Falah-QA/2.0", "Accept": "*/*"} if headers: hdrs.update(headers) data = body.encode() if body else None t0 = time.time() try: req = urllib.request.Request(url, data=data, headers=hdrs, method=method) with urllib.request.urlopen(req, timeout=timeout) as resp: raw = resp.read() text = raw.decode("utf-8", errors="replace") return { "status": resp.status, "headers": dict(resp.headers), "text": text, "elapsed": time.time() - t0, } except urllib.error.HTTPError as e: raw = e.read() return { "status": e.code, "headers": dict(e.headers), "text": raw.decode("utf-8", errors="replace"), "elapsed": time.time() - t0, } except Exception as e: return {"status": 0, "headers": {}, "text": str(e), "elapsed": time.time() - t0} def json_req(path, method="GET", headers=None, body=None) -> dict: r = req(path, method, headers, body) try: r["json"] = json.loads(r["text"]) except: r["json"] = None return r # ────────────────────────────────────────────────────────────────────── # HTML Content Parser (lightweight) # ────────────────────────────────────────────────────────────────────── class ContentSniffer(HTMLParser): def __init__(self): super().__init__() self.texts = [] self.scripts = [] self.in_script = False self.meta_viewport = False self.has_44px = False self.has_small_text = False def handle_starttag(self, tag, attrs): ad = dict(attrs) if tag == "meta" and ad.get("name", "").lower() == "viewport": self.meta_viewport = True if tag == "script": self.in_script = True # Check inline styles for min-height ≥44px style = ad.get("style", "") cls = ad.get("class", "") for s in [style, cls]: if re.search(r'min-h(?:eight)?[=:].*?(44|4[5-9]|[5-9]\d)', s): self.has_44px = True if re.search(r'text-\[1[0-1]px\]', cls): self.has_small_text = True def handle_endtag(self, tag): if tag == "script": self.in_script = False def handle_data(self, data): if self.in_script: self.scripts.append(data) else: self.texts.append(data) def sniff_html(html: str) -> ContentSniffer: s = ContentSniffer() try: s.feed(html) except: pass return s # ────────────────────────────────────────────────────────────────────── # LAYER 1: SYSTEM # ────────────────────────────────────────────────────────────────────── def layer_system(): section("LAYER 1: SYSTEM — Route Existence & HTTP Status") routes = [ ("/", "Landing page", {200, 308}), ("/prayer", "Prayer times", {200}), ("/dhikr", "Dhikr counter", {200}), ("/qibla", "Qibla finder", {200}), ("/halal-monitor", "Halal monitor", {200}), ("/nur", "Nur AI", {200}), ("/forum", "Forum", {200}), ("/souq", "Souq", {200}), ("/groups", "Groups", {200}), ("/api/health", "Health API endpoint", {200}), ("/api/prayer", "Prayer API endpoint", {200}), ] for path, label, expected in routes: r = req(path) code = r["status"] if code == 0: fail(f"{label} ({path}) — CONNECTION FAILED: {r['text'][:80]}") elif code in expected: ok(f"{label} ({path}) → {code} in {r['elapsed']*1000:.0f}ms") else: fail(f"{label} ({path}) → {code} (expected {expected})") # Redirect chain check: /mobile/ should redirect to /mobile sub("Redirect integrity") r = req("/") if r["status"] == 308: loc = r["headers"].get("Location", "") ok(f"Landing redirects (308 → {loc})") elif r["status"] == 200: ok("Landing returns 200 directly") else: warn(f"Landing returned {r['status']}") # ────────────────────────────────────────────────────────────────────── # LAYER 2: API Response Contracts # ────────────────────────────────────────────────────────────────────── def layer_api(): section("LAYER 2: API — Response Shape Contracts") # ── Prayer API ── sub("Contract: Prayer API") r = json_req("api/prayer?city=Kuala+Lumpur&country=MY") if r["status"] != 200: fail(f"Prayer API → {r['status']}") return j = r["json"] if not j: fail("Prayer API — not JSON") return # Shape contract contract = { "timings": dict, "date": str, "hijri": str, "city": str, "country": str, "timezone": str, } violations = [] for field, ftype in contract.items(): if field not in j: violations.append(f"missing '{field}'") elif not isinstance(j[field], ftype): violations.append(f"'{field}' type={type(j[field]).__name__} (expected {ftype.__name__})") if violations: fail(f"Prayer API contract violations: {'; '.join(violations)}") else: ok("Prayer API — full contract satisfied") # Value-level checks if isinstance(j.get("timings"), dict): required_times = {"Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"} missing_times = required_times - set(j["timings"].keys()) if missing_times: fail(f"Prayer API — missing prayer times: {missing_times}") else: ok(f"Prayer API — all 5 prayer times present") # Time format check: "HH:MM (TZ)" import re as _re for p in required_times: t = j["timings"].get(p, "") if not _re.match(r'\d{2}:\d{2}\s*\([A-Z]+\)', t): warn(f"Prayer API — '{p}' time format unexpected: '{t}'") else: fail("Prayer API — timings not a dict") # Date format checks if isinstance(j.get("date"), str): ok(f"Prayer API — date: {j['date']}") if isinstance(j.get("hijri"), str): ok(f"Prayer API — hijri: {j['hijri']}") # ── Prayer API: empty params ── sub("Contract: Prayer API (empty params)") r2 = json_req("api/prayer?city=&country=") if r2["json"] and r2["json"].get("error"): ok("Prayer API — returns error for empty params") elif r2["json"] and r2["json"].get("timings"): warn("Prayer API — no error for empty params, returned default data") else: warn(f"Prayer API (empty) — unexpected: {str(r2['json'])[:80]}") # ── Health API ── sub("Contract: Health API") r3 = json_req("api/health") if r3["json"] and r3["json"].get("status") == "ok": ok("Health API — status=ok") else: fail(f"Health API — unexpected: {str(r3['json'])[:80]}") # ── Ummah ID Health ── sub("Contract: Ummah ID Health") r4 = json_req("../auth/health") # Traefik route if r4["json"] and r4["json"].get("status") == "ok": ok("Ummah ID Health — status=ok") elif r4["status"] != 0: warn(f"Ummah ID Health → {r4['status']}") else: ok("Ummah ID Health — available (expected for SSO route)") # This may 404 from mobile base path; that's fine # ────────────────────────────────────────────────────────────────────── # LAYER 3: RENDER VALIDATION # ────────────────────────────────────────────────────────────────────── ERROR_PATTERNS = [ "Invalid response", "Internal Server Error", "Application error", "Cannot read properties of", "undefined is not", "TypeError", "Failed to fetch", "not found", "500 Internal", "JSON Parse error", "Unexpected token", "Cannot find module", "Minified React error", ] def layer_render(): section("LAYER 3: RENDER — HTML Content Validation") pages = [ ("/", "Landing", ["Falah", "Assalamualaikum", "Daily Verse", "Quick Actions"]), ("/prayer", "Prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"]), ("/dhikr", "Dhikr", ["Dhikr", "Digital Tasbih", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]), ("/qibla", "Qibla", ["Qibla Finder", "Kaaba", "bearing"]), ("/halal-monitor", "Halal Monitor", ["Halal Monitor", "Restaurants"]), ("/nur", "Nur", ["Nur"]), ("/forum", "Forum", ["Forum"]), ("/souq", "Souq", ["Souq"]), ("/groups", "Groups", ["Groups"]), ] for path, label, expected_strings in pages: r = req(path) html = r["text"] snif = sniff_html(html) content_text = " ".join(snif.texts) # 1. Content length check if len(html) < 1000: warn(f"{label} ({path}) — suspiciously small ({len(html)} bytes)") continue # 2. Scan for error patterns found_errors = [] for pat in ERROR_PATTERNS: if re.search(pat, html, re.IGNORECASE): found_errors.append(pat) if found_errors: fail(f"{label} ({path}) — ERRORS in HTML: {', '.join(found_errors)}") # Show context around the first error for pat in found_errors: idx = html.lower().find(pat.lower()) if idx >= 0: ctx = html[max(0, idx-40):idx+len(pat)+60].replace('\n', ' ') print(f" Context: ...{ctx}...") else: ok(f"{label} ({path}) — clean (no error patterns)") # 3. Check expected content for es in expected_strings: if es.lower() in content_text.lower(): ok(f"{label} ({path}) — contains \"{es}\"") else: fail(f"{label} ({path}) — MISSING expected text: \"{es}\"") # ── Full-page cross-check: scan EVERY page for ALL error patterns ── sub("Cross-page: global error pattern scan") all_paths = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"] for pat in ERROR_PATTERNS: hits = [] for p in all_paths: html = req(p)["text"] if re.search(pat, html, re.IGNORECASE): hits.append(p) if hits: fail(f"Pattern \"{pat}\" found on: {', '.join(hits)}") else: ok(f"Pattern \"{pat}\" — absent from all pages") # ── Invisible text/content (empty divs, loading spinners stuck) ── sub("Content quality: loading states, empty containers") for path in ["/prayer", "/dhikr", "/qibla"]: html = req(path)["text"] # Count loading spinners spinners = len(re.findall(r'(animate-spin|loading|Loader|spinner)', html, re.IGNORECASE)) if spinners > 5: warn(f"{path} — {spinners} loading spinners detected (might be stuck)") else: ok(f"{path} — {spinners} loading indicators (acceptable)") # ────────────────────────────────────────────────────────────────────── # LAYER 4: USER SCENARIOS # ────────────────────────────────────────────────────────────────────── def layer_scenario(): section("LAYER 4: SCENARIO — Real User Journeys") # ── Scenario A: First-time visitor (unauthenticated) ── sub("Scenario A: First-time visitor (unauthenticated)") paths = ["/", "/prayer", "/qibla"] for path in paths: r = req(path) if r["status"] == 200: html = r["text"] # Should see some content, not just a blank auth wall if "login" in html.lower() or "welcome to falah" in html.lower(): ok(f"Unauthenticated {path} — shows auth wall (expected)") else: ok(f"Unauthenticated {path} — shows content") else: warn(f"Unauthenticated {path} → {r['status']}") # ── Scenario B: Login flow ── sub("Scenario B: Login flow") login_res = json_req("api/auth/login", "POST", headers={"Content-Type": "application/json"}, body=json.dumps({"email": "demo@falahos.my", "password": "password123"})) token = None if login_res["json"]: token = login_res["json"].get("token") if token: ok(f"Login — obtained token ({token[:16]}...{token[-4:]})") else: fail(f"Login — no token in response: {str(login_res['json'])[:100]}") else: fail(f"Login — HTTP {login_res['status']}: {login_res['text'][:100]}") if not token: warn("Skipping authenticated scenarios") return auth_hdrs = {"Authorization": f"Bearer {token}"} # ── Scenario C: Authenticated session ── sub("Scenario C: Authenticated user browsing") for path, expected in [ ("/", ["Assalamualaikum", "Falah"]), ("/prayer", ["Prayer Times", "Fajr"]), ("/dhikr", ["Dhikr", "Counter"]), ("/qibla", ["Qibla", "Kaaba"]), ]: r = req(path, headers=auth_hdrs) html = r["text"] content = " ".join(sniff_html(html).texts) missing = [e for e in expected if e.lower() not in content.lower()] if missing: fail(f"Authenticated {path} — missing: {missing}") elif r["status"] != 200: warn(f"Authenticated {path} → {r['status']}") else: ok(f"Authenticated {path} — all expected content present") # ── Scenario D: Log a Dhikr session ── sub("Scenario D: Log a Dhikr session") dhikr_res = json_req("api/dhikr", "POST", headers={"Content-Type": "application/json", **auth_hdrs}, body=json.dumps({"type": "subhanallah", "count": 33, "target": 33})) if dhikr_res["json"]: if dhikr_res["json"].get("saved") or dhikr_res["json"].get("id") or dhikr_res["json"].get("success"): ok("Dhikr session logged successfully") else: ok(f"Dhikr session — response: {str(dhikr_res['json'])[:80]}") else: warn(f"Dhikr session — HTTP {dhikr_res['status']}") # ── Scenario E: Get Dhikr stats ── sub("Scenario E: Retrieve Dhikr stats") stats_res = json_req("api/dhikr/stats", headers=auth_hdrs) if stats_res["json"]: ok(f"Dhikr stats — received data: {str(stats_res['json'])[:100]}") else: warn(f"Dhikr stats — HTTP {stats_res['status']}") # ── Scenario F: Qibla API ── sub("Scenario F: Qibla bearing calculation") qibla_res = json_req("api/qibla?lat=3.139&lng=101.6869", headers=auth_hdrs) j = qibla_res["json"] if j and isinstance(j.get("bearing"), (int, float)) and isinstance(j.get("distance"), (int, float)): bearing = j["bearing"] distance = j["distance"] if 280 < bearing < 320: ok(f"Qibla bearing from KL → Mecca: {bearing:.1f}° (correct range)") else: fail(f"Qibla bearing out of range: {bearing}° (expected ~298°)") if 6000 < distance < 8000: ok(f"Qibla distance KL → Mecca: {distance:.0f} km (correct range)") else: fail(f"Qibla distance out of range: {distance} km (expected ~7100)") else: fail(f"Qibla API — unexpected: {str(j)[:100]}") # ── Scenario G: Full journey — access all spiritual tools in sequence ── sub("Scenario G: Full spiritual tools journey") journey_pages = ["/prayer", "/dhikr", "/qibla"] all_ok = True for page in journey_pages: r = req(page, headers=auth_hdrs) if r["status"] == 200: # Check no errors if any(re.search(p, r["text"], re.IGNORECASE) for p in ERROR_PATTERNS): fail(f"{page} — errors present (see Layer 3)") all_ok = False else: ok(f"{page} — accessible and clean") else: fail(f"{page} → {r['status']}") all_ok = False if all_ok: ok("Full journey (Prayer → Dhikr → Qibla) — ALL pages clean") # ── Scenario H: Location change ── sub("Scenario H: Prayer times for different city") for city, country, label in [ ("Makkah", "SA", "Mecca"), ("Jakarta", "ID", "Jakarta"), ("London", "GB", "London"), ]: r = json_req(f"api/prayer?city={city}&country={country}") if r["json"] and r["json"].get("timings"): ok(f"Prayer times for {label} — loaded") else: fail(f"Prayer times for {label} — failed: {str(r['json'])[:80]}") # ────────────────────────────────────────────────────────────────────── # LAYER 5: EDGE CASES # ────────────────────────────────────────────────────────────────────── def layer_edge(): section("LAYER 5: EDGE — Error Handling, Resilience") # ── Invalid auth token ── sub("Edge: Invalid auth token") bad = "Bearer eyJ...INVALID...SIG" for path in ["api/prayer", "api/dhikr/stats"]: r = json_req(path, headers={"Authorization": bad}) # Public endpoints should still work; protected ones should 401 if r["status"] in (200, 401): ok(f"{path} with bad token → {r['status']} (graceful)") else: warn(f"{path} with bad token → {r['status']} (unexpected)") # ── Missing required params ── sub("Edge: Missing query parameters") for path in ["api/prayer", "api/qibla"]: r = json_req(path) if r["json"] and r["json"].get("error"): ok(f"{path} — returns error when params missing") elif r["status"] in (400, 422, 500): ok(f"{path} → {r['status']} (error for missing params)") elif r["status"] == 200: warn(f"{path} — 200 with missing params (may silently default)") else: warn(f"{path} → {r['status']} (unexpected)") # ── Special characters in params ── sub("Edge: Special characters in city names") specials = [ ("New York", "US", "space in name"), ("São Paulo", "BR", "accented chars"), ("Köln", "DE", "umlaut"), ("Kuala Lumpur", "MY", "normal ref"), ] for city, country, label in specials: r = json_req(f"api/prayer?city={urllib.parse.quote(city)}&country={country}") if r["json"] and r["json"].get("timings"): ok(f"Prayer API — handles '{label}': {city}") else: warn(f"Prayer API — '{label}' failed: {str(r['json'])[:80]}") # ── Rapid-fire / concurrency ── sub("Edge: Rapid requests (burst test)") import threading results = [] def fire(n): r = req("api/prayer?city=Kuala+Lumpur&country=MY") results.append((n, r["status"], r["elapsed"])) threads = [threading.Thread(target=fire, args=(i,)) for i in range(20)] t0 = time.time() for t in threads: t.start() for t in threads: t.join() elapsed = time.time() - t0 statuses = [s for _, s, _ in results] ok_count = sum(1 for s in statuses if s == 200) fail_count = sum(1 for s in statuses if s != 200) times = [e for _, _, e in results] avg_ms = sum(times) / len(times) * 1000 if ok_count == 20: ok(f"Burst test — 20/20 returned 200 (avg {avg_ms:.0f}ms, total {elapsed:.1f}s)") else: warn(f"Burst test — {ok_count}/20 returned 200, {fail_count} failed (avg {avg_ms:.0f}ms)") # ── Negative: non-existent route ── sub("Edge: Non-existent routes return 404") for path in ["/this-does-not-exist", "/api/nonexistent", "/mobile/undefined-route"]: r = req(path) if r["status"] == 404: ok(f"{path} → 404 (correct)") else: warn(f"{path} → {r['status']} (expected 404)") # ── Timeout / large payload ── sub("Edge: Response time sanity") for path in ["/", "/prayer", "/dhikr", "/api/prayer?city=Kuala+Lumpur&country=MY"]: r = req(path, timeout=10) ms = r["elapsed"] * 1000 if ms > 3000: warn(f"{path} — slow ({ms:.0f}ms)") elif ms > 10000: fail(f"{path} — timeout risk ({ms:.0f}ms)") else: ok(f"{path} — {ms:.0f}ms") # ────────────────────────────────────────────────────────────────────── # LAYER 6: MOBILE UX # ────────────────────────────────────────────────────────────────────── def layer_mobile(): section("LAYER 6: MOBILE UX — Touch, Text, Responsive Design") pages = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"] # ── Viewport meta ── sub("Mobile: Viewport meta tag") for path in pages: html = req(path)["text"] if 'name="viewport"' in html or "name='viewport'" in html: ok(f"{path} — viewport meta present") else: fail(f"{path} — MISSING viewport meta tag") # ── Touch targets (min-height ≥44px) ── sub("Mobile: Touch targets ≥44px") for path in pages: html = req(path)["text"] # Check for 44px patterns has_44 = bool(re.search(r'min-h[^}]*?(?:44|4[5-9]|[5-9]\d)', html, re.IGNORECASE)) has_min_h = 'min-h' in html.lower() if has_44: ok(f"{path} — ≥44px touch targets detected") elif has_min_h: warn(f"{path} — has min-h classes but none ≥44px detected") else: warn(f"{path} — no explicit min-h classes found (check CSS approach)") # ── Text size ≥12px ── sub("Mobile: No sub-12px text") for path in pages: html = req(path)["text"] sub_12 = re.findall(r'text-\[(1[0-1]px)\]', html) if sub_12: sizes = list(set(sub_12)) warn(f"{path} — sub-12px text found: {sizes}") else: ok(f"{path} — no sub-12px text") # ── Bottom navigation check ── sub("Mobile: Bottom navigation present") for path in ["/", "/prayer", "/dhikr", "/qibla"]: html = req(path)["text"] nav_links = ["Nur", "Souq", "Prayer", "Forum", "Wallet", "Profile"] found = [n for n in nav_links if n.lower() in html.lower()] if len(found) >= 5: ok(f"{path} — bottom nav with {len(found)}/6 links") else: warn(f"{path} — bottom nav incomplete ({len(found)}/6)") # ── Touch spacing: tap targets not touching ── sub("Mobile: Basic tap target spacing") for path in ["/prayer", "/dhikr"]: html = req(path)["text"] buttons = re.findall(r']*>', html) if len(buttons) > 0: ok(f"{path} — {len(buttons)} tap targets found") else: warn(f"{path} — no