Error Feedback Service + friendly error handling across all pages
- Error Feedback Service: POST /api/feedback saves to JSONL, GET with admin token - ErrorFeedback component: warm amber tones, friendly messages, 'Report Issue' button - Replaced ALL red-themed error displays across 15+ pages with ErrorFeedback - Kind classification: network, auth, location, upgrade, default messages - QA test suite v2: 8-layer testing (system, API, render, scenario, edge, mobile, perf, security) - Browser-based visual QA with Playwright (Layer 9)
This commit is contained in:
root
@@ -0,0 +1,27 @@
|
||||
# Falah Mobile — QA Test Strategy
|
||||
|
||||
## Layers of Testing
|
||||
|
||||
```
|
||||
Layer 1: SYSTEM ── HTTP status, route existence, redirects
|
||||
Layer 2: API ── Response shape, data types, error codes
|
||||
Layer 3: RENDER ── Page content validates (no error messages in output)
|
||||
Layer 4: FLOW ── Multi-step user journeys end-to-end
|
||||
Layer 5: EDGE ── Permissions denied, no GPS, offline, first-time
|
||||
Layer 6: MOBILE ── Viewport sizes, touch targets (≥44px), scroll
|
||||
Layer 7: REGRESSION ── Compare old behavior vs new
|
||||
```
|
||||
|
||||
## Key Principle
|
||||
**A 200 status code does NOT mean the page works.** Every page must be tested for:
|
||||
- Absence of error messages in rendered HTML
|
||||
- Expected content elements present
|
||||
- API calls return correct data shapes
|
||||
- Visual states (loading → success → empty → error) all tested
|
||||
|
||||
## Test Automation Priority
|
||||
1. API response shape validation (catch mismatch like PrayerApiResponse)
|
||||
2. Page content scanning for known error strings
|
||||
3. Authenticated flow tests
|
||||
4. Permission denied scenarios
|
||||
5. Accessibility baseline
|
||||
@@ -0,0 +1,836 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Falah Mobile — Comprehensive QA Test Suite v2
|
||||
=============================================
|
||||
Multi-layer, contract-driven, user-scenario-based testing.
|
||||
|
||||
Usage:
|
||||
python3 tests/qa-test-suite.py # Full suite
|
||||
python3 tests/qa-test-suite.py --layer api # Single layer
|
||||
python3 tests/qa-test-suite.py --ci-exit # Exit 1 on failures
|
||||
python3 tests/qa-test-suite.py --url https://staging.falahos.my/mobile
|
||||
|
||||
Layers (run in order; each builds on the previous):
|
||||
1. SYSTEM — HTTP status, route existence, redirect chain
|
||||
2. API — Response shape contracts (strict schema validation)
|
||||
3. RENDER — Parse HTML; assert no error strings, expected content present
|
||||
4. SCENARIO — Real user journeys (first-time, returning, premium, etc.)
|
||||
5. EDGE — Missing params, invalid tokens, rapid fire, offline
|
||||
6. MOBILE — Viewport, touch targets ≥44px, text ≥12px, tap targets
|
||||
7. PERF — Response times, page weight, Time to First Byte
|
||||
8. SECURITY — Headers, no secrets in HTML, HSTS, CSP
|
||||
"""
|
||||
|
||||
import argparse, json, os, re, sys, time, urllib.request, urllib.error
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
from html.parser import HTMLParser
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# Config
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile")
|
||||
TIMEOUT = 20
|
||||
PASS, FAIL, WARN = 0, 0, 0
|
||||
REPORT_LINES = []
|
||||
CI_EXIT = False
|
||||
|
||||
G = "\033[0;32m"; R = "\033[0;31m"; Y = "\033[1;33m"
|
||||
C = "\033[0;36m"; N = "\033[0m"
|
||||
|
||||
def ok(msg): global PASS; PASS += 1; print(f" {G}✓{N} {msg}"); REPORT_LINES.append(("PASS", msg))
|
||||
def fail(msg): global FAIL; FAIL += 1; print(f" {R}✗{N} {msg}"); REPORT_LINES.append(("FAIL", msg))
|
||||
def warn(msg): global WARN; WARN += 1; print(f" {Y}⚠{N} {msg}"); REPORT_LINES.append(("WARN", msg))
|
||||
|
||||
def section(title):
|
||||
print(f"\n{C}════════════════════════════════════════════════════════════{N}")
|
||||
print(f"{C} {title}{N}")
|
||||
print(f"{C}════════════════════════════════════════════════════════════{N}")
|
||||
|
||||
def sub(title):
|
||||
print(f"\n{C}── {title} ──{N}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# HTTP helper
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def req(path: str, method="GET", headers=None, body=None, timeout=TIMEOUT) -> dict:
|
||||
"""Make HTTP request, return {status, headers, text, elapsed}."""
|
||||
url = f"{BASE_URL.rstrip('/')}/{path.lstrip('/')}"
|
||||
hdrs = {"User-Agent": "Falah-QA/2.0", "Accept": "*/*"}
|
||||
if headers:
|
||||
hdrs.update(headers)
|
||||
data = body.encode() if body else None
|
||||
t0 = time.time()
|
||||
try:
|
||||
req = urllib.request.Request(url, data=data, headers=hdrs, method=method)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
return {
|
||||
"status": resp.status,
|
||||
"headers": dict(resp.headers),
|
||||
"text": text,
|
||||
"elapsed": time.time() - t0,
|
||||
}
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
return {
|
||||
"status": e.code,
|
||||
"headers": dict(e.headers),
|
||||
"text": raw.decode("utf-8", errors="replace"),
|
||||
"elapsed": time.time() - t0,
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": 0, "headers": {}, "text": str(e), "elapsed": time.time() - t0}
|
||||
|
||||
def json_req(path, method="GET", headers=None, body=None) -> dict:
|
||||
r = req(path, method, headers, body)
|
||||
try:
|
||||
r["json"] = json.loads(r["text"])
|
||||
except:
|
||||
r["json"] = None
|
||||
return r
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# HTML Content Parser (lightweight)
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
class ContentSniffer(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.texts = []
|
||||
self.scripts = []
|
||||
self.in_script = False
|
||||
self.meta_viewport = False
|
||||
self.has_44px = False
|
||||
self.has_small_text = False
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
ad = dict(attrs)
|
||||
if tag == "meta" and ad.get("name", "").lower() == "viewport":
|
||||
self.meta_viewport = True
|
||||
if tag == "script":
|
||||
self.in_script = True
|
||||
# Check inline styles for min-height ≥44px
|
||||
style = ad.get("style", "")
|
||||
cls = ad.get("class", "")
|
||||
for s in [style, cls]:
|
||||
if re.search(r'min-h(?:eight)?[=:].*?(44|4[5-9]|[5-9]\d)', s):
|
||||
self.has_44px = True
|
||||
if re.search(r'text-\[1[0-1]px\]', cls):
|
||||
self.has_small_text = True
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == "script":
|
||||
self.in_script = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_script:
|
||||
self.scripts.append(data)
|
||||
else:
|
||||
self.texts.append(data)
|
||||
|
||||
def sniff_html(html: str) -> ContentSniffer:
|
||||
s = ContentSniffer()
|
||||
try:
|
||||
s.feed(html)
|
||||
except:
|
||||
pass
|
||||
return s
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 1: SYSTEM
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_system():
|
||||
section("LAYER 1: SYSTEM — Route Existence & HTTP Status")
|
||||
|
||||
routes = [
|
||||
("/", "Landing page", {200, 308}),
|
||||
("/prayer", "Prayer times", {200}),
|
||||
("/dhikr", "Dhikr counter", {200}),
|
||||
("/qibla", "Qibla finder", {200}),
|
||||
("/halal-monitor", "Halal monitor", {200}),
|
||||
("/nur", "Nur AI", {200}),
|
||||
("/forum", "Forum", {200}),
|
||||
("/souq", "Souq", {200}),
|
||||
("/groups", "Groups", {200}),
|
||||
("/api/health", "Health API endpoint", {200}),
|
||||
("/api/prayer", "Prayer API endpoint", {200}),
|
||||
]
|
||||
|
||||
for path, label, expected in routes:
|
||||
r = req(path)
|
||||
code = r["status"]
|
||||
if code == 0:
|
||||
fail(f"{label} ({path}) — CONNECTION FAILED: {r['text'][:80]}")
|
||||
elif code in expected:
|
||||
ok(f"{label} ({path}) → {code} in {r['elapsed']*1000:.0f}ms")
|
||||
else:
|
||||
fail(f"{label} ({path}) → {code} (expected {expected})")
|
||||
|
||||
# Redirect chain check: /mobile/ should redirect to /mobile
|
||||
sub("Redirect integrity")
|
||||
r = req("/")
|
||||
if r["status"] == 308:
|
||||
loc = r["headers"].get("Location", "")
|
||||
ok(f"Landing redirects (308 → {loc})")
|
||||
elif r["status"] == 200:
|
||||
ok("Landing returns 200 directly")
|
||||
else:
|
||||
warn(f"Landing returned {r['status']}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 2: API Response Contracts
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_api():
|
||||
section("LAYER 2: API — Response Shape Contracts")
|
||||
|
||||
# ── Prayer API ──
|
||||
sub("Contract: Prayer API")
|
||||
r = json_req("api/prayer?city=Kuala+Lumpur&country=MY")
|
||||
if r["status"] != 200:
|
||||
fail(f"Prayer API → {r['status']}")
|
||||
return
|
||||
|
||||
j = r["json"]
|
||||
if not j:
|
||||
fail("Prayer API — not JSON")
|
||||
return
|
||||
|
||||
# Shape contract
|
||||
contract = {
|
||||
"timings": dict,
|
||||
"date": str,
|
||||
"hijri": str,
|
||||
"city": str,
|
||||
"country": str,
|
||||
"timezone": str,
|
||||
}
|
||||
violations = []
|
||||
for field, ftype in contract.items():
|
||||
if field not in j:
|
||||
violations.append(f"missing '{field}'")
|
||||
elif not isinstance(j[field], ftype):
|
||||
violations.append(f"'{field}' type={type(j[field]).__name__} (expected {ftype.__name__})")
|
||||
|
||||
if violations:
|
||||
fail(f"Prayer API contract violations: {'; '.join(violations)}")
|
||||
else:
|
||||
ok("Prayer API — full contract satisfied")
|
||||
|
||||
# Value-level checks
|
||||
if isinstance(j.get("timings"), dict):
|
||||
required_times = {"Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"}
|
||||
missing_times = required_times - set(j["timings"].keys())
|
||||
if missing_times:
|
||||
fail(f"Prayer API — missing prayer times: {missing_times}")
|
||||
else:
|
||||
ok(f"Prayer API — all 5 prayer times present")
|
||||
|
||||
# Time format check: "HH:MM (TZ)"
|
||||
import re as _re
|
||||
for p in required_times:
|
||||
t = j["timings"].get(p, "")
|
||||
if not _re.match(r'\d{2}:\d{2}\s*\([A-Z]+\)', t):
|
||||
warn(f"Prayer API — '{p}' time format unexpected: '{t}'")
|
||||
else:
|
||||
fail("Prayer API — timings not a dict")
|
||||
|
||||
# Date format checks
|
||||
if isinstance(j.get("date"), str):
|
||||
ok(f"Prayer API — date: {j['date']}")
|
||||
if isinstance(j.get("hijri"), str):
|
||||
ok(f"Prayer API — hijri: {j['hijri']}")
|
||||
|
||||
# ── Prayer API: empty params ──
|
||||
sub("Contract: Prayer API (empty params)")
|
||||
r2 = json_req("api/prayer?city=&country=")
|
||||
if r2["json"] and r2["json"].get("error"):
|
||||
ok("Prayer API — returns error for empty params")
|
||||
elif r2["json"] and r2["json"].get("timings"):
|
||||
warn("Prayer API — no error for empty params, returned default data")
|
||||
else:
|
||||
warn(f"Prayer API (empty) — unexpected: {str(r2['json'])[:80]}")
|
||||
|
||||
# ── Health API ──
|
||||
sub("Contract: Health API")
|
||||
r3 = json_req("api/health")
|
||||
if r3["json"] and r3["json"].get("status") == "ok":
|
||||
ok("Health API — status=ok")
|
||||
else:
|
||||
fail(f"Health API — unexpected: {str(r3['json'])[:80]}")
|
||||
|
||||
# ── Ummah ID Health ──
|
||||
sub("Contract: Ummah ID Health")
|
||||
r4 = json_req("../auth/health") # Traefik route
|
||||
if r4["json"] and r4["json"].get("status") == "ok":
|
||||
ok("Ummah ID Health — status=ok")
|
||||
elif r4["status"] != 0:
|
||||
warn(f"Ummah ID Health → {r4['status']}")
|
||||
else:
|
||||
ok("Ummah ID Health — available (expected for SSO route)")
|
||||
# This may 404 from mobile base path; that's fine
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 3: RENDER VALIDATION
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
ERROR_PATTERNS = [
|
||||
"Invalid response",
|
||||
"Internal Server Error",
|
||||
"Application error",
|
||||
"Cannot read properties of",
|
||||
"undefined is not",
|
||||
"TypeError",
|
||||
"Failed to fetch",
|
||||
"not found",
|
||||
"500 Internal",
|
||||
"JSON Parse error",
|
||||
"Unexpected token",
|
||||
"Cannot find module",
|
||||
"Minified React error",
|
||||
]
|
||||
|
||||
def layer_render():
|
||||
section("LAYER 3: RENDER — HTML Content Validation")
|
||||
|
||||
pages = [
|
||||
("/", "Landing", ["Falah", "Assalamualaikum", "Daily Verse", "Quick Actions"]),
|
||||
("/prayer", "Prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha"]),
|
||||
("/dhikr", "Dhikr", ["Dhikr", "Digital Tasbih", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]),
|
||||
("/qibla", "Qibla", ["Qibla Finder", "Kaaba", "bearing"]),
|
||||
("/halal-monitor", "Halal Monitor", ["Halal Monitor", "Restaurants"]),
|
||||
("/nur", "Nur", ["Nur"]),
|
||||
("/forum", "Forum", ["Forum"]),
|
||||
("/souq", "Souq", ["Souq"]),
|
||||
("/groups", "Groups", ["Groups"]),
|
||||
]
|
||||
|
||||
for path, label, expected_strings in pages:
|
||||
r = req(path)
|
||||
html = r["text"]
|
||||
snif = sniff_html(html)
|
||||
content_text = " ".join(snif.texts)
|
||||
|
||||
# 1. Content length check
|
||||
if len(html) < 1000:
|
||||
warn(f"{label} ({path}) — suspiciously small ({len(html)} bytes)")
|
||||
continue
|
||||
|
||||
# 2. Scan for error patterns
|
||||
found_errors = []
|
||||
for pat in ERROR_PATTERNS:
|
||||
if re.search(pat, html, re.IGNORECASE):
|
||||
found_errors.append(pat)
|
||||
|
||||
if found_errors:
|
||||
fail(f"{label} ({path}) — ERRORS in HTML: {', '.join(found_errors)}")
|
||||
# Show context around the first error
|
||||
for pat in found_errors:
|
||||
idx = html.lower().find(pat.lower())
|
||||
if idx >= 0:
|
||||
ctx = html[max(0, idx-40):idx+len(pat)+60].replace('\n', ' ')
|
||||
print(f" Context: ...{ctx}...")
|
||||
else:
|
||||
ok(f"{label} ({path}) — clean (no error patterns)")
|
||||
|
||||
# 3. Check expected content
|
||||
for es in expected_strings:
|
||||
if es.lower() in content_text.lower():
|
||||
ok(f"{label} ({path}) — contains \"{es}\"")
|
||||
else:
|
||||
fail(f"{label} ({path}) — MISSING expected text: \"{es}\"")
|
||||
|
||||
# ── Full-page cross-check: scan EVERY page for ALL error patterns ──
|
||||
sub("Cross-page: global error pattern scan")
|
||||
all_paths = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"]
|
||||
for pat in ERROR_PATTERNS:
|
||||
hits = []
|
||||
for p in all_paths:
|
||||
html = req(p)["text"]
|
||||
if re.search(pat, html, re.IGNORECASE):
|
||||
hits.append(p)
|
||||
if hits:
|
||||
fail(f"Pattern \"{pat}\" found on: {', '.join(hits)}")
|
||||
else:
|
||||
ok(f"Pattern \"{pat}\" — absent from all pages")
|
||||
|
||||
# ── Invisible text/content (empty divs, loading spinners stuck) ──
|
||||
sub("Content quality: loading states, empty containers")
|
||||
for path in ["/prayer", "/dhikr", "/qibla"]:
|
||||
html = req(path)["text"]
|
||||
# Count loading spinners
|
||||
spinners = len(re.findall(r'(animate-spin|loading|Loader|spinner)', html, re.IGNORECASE))
|
||||
if spinners > 5:
|
||||
warn(f"{path} — {spinners} loading spinners detected (might be stuck)")
|
||||
else:
|
||||
ok(f"{path} — {spinners} loading indicators (acceptable)")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 4: USER SCENARIOS
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_scenario():
|
||||
section("LAYER 4: SCENARIO — Real User Journeys")
|
||||
|
||||
# ── Scenario A: First-time visitor (unauthenticated) ──
|
||||
sub("Scenario A: First-time visitor (unauthenticated)")
|
||||
paths = ["/", "/prayer", "/qibla"]
|
||||
for path in paths:
|
||||
r = req(path)
|
||||
if r["status"] == 200:
|
||||
html = r["text"]
|
||||
# Should see some content, not just a blank auth wall
|
||||
if "login" in html.lower() or "welcome to falah" in html.lower():
|
||||
ok(f"Unauthenticated {path} — shows auth wall (expected)")
|
||||
else:
|
||||
ok(f"Unauthenticated {path} — shows content")
|
||||
else:
|
||||
warn(f"Unauthenticated {path} → {r['status']}")
|
||||
|
||||
# ── Scenario B: Login flow ──
|
||||
sub("Scenario B: Login flow")
|
||||
login_res = json_req("api/auth/login", "POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
body=json.dumps({"email": "demo@falahos.my", "password": "password123"}))
|
||||
token = None
|
||||
if login_res["json"]:
|
||||
token = login_res["json"].get("token")
|
||||
if token:
|
||||
ok(f"Login — obtained token ({token[:16]}...{token[-4:]})")
|
||||
else:
|
||||
fail(f"Login — no token in response: {str(login_res['json'])[:100]}")
|
||||
else:
|
||||
fail(f"Login — HTTP {login_res['status']}: {login_res['text'][:100]}")
|
||||
|
||||
if not token:
|
||||
warn("Skipping authenticated scenarios")
|
||||
return
|
||||
|
||||
auth_hdrs = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
# ── Scenario C: Authenticated session ──
|
||||
sub("Scenario C: Authenticated user browsing")
|
||||
for path, expected in [
|
||||
("/", ["Assalamualaikum", "Falah"]),
|
||||
("/prayer", ["Prayer Times", "Fajr"]),
|
||||
("/dhikr", ["Dhikr", "Counter"]),
|
||||
("/qibla", ["Qibla", "Kaaba"]),
|
||||
]:
|
||||
r = req(path, headers=auth_hdrs)
|
||||
html = r["text"]
|
||||
content = " ".join(sniff_html(html).texts)
|
||||
missing = [e for e in expected if e.lower() not in content.lower()]
|
||||
if missing:
|
||||
fail(f"Authenticated {path} — missing: {missing}")
|
||||
elif r["status"] != 200:
|
||||
warn(f"Authenticated {path} → {r['status']}")
|
||||
else:
|
||||
ok(f"Authenticated {path} — all expected content present")
|
||||
|
||||
# ── Scenario D: Log a Dhikr session ──
|
||||
sub("Scenario D: Log a Dhikr session")
|
||||
dhikr_res = json_req("api/dhikr", "POST",
|
||||
headers={"Content-Type": "application/json", **auth_hdrs},
|
||||
body=json.dumps({"type": "subhanallah", "count": 33, "target": 33}))
|
||||
if dhikr_res["json"]:
|
||||
if dhikr_res["json"].get("saved") or dhikr_res["json"].get("id") or dhikr_res["json"].get("success"):
|
||||
ok("Dhikr session logged successfully")
|
||||
else:
|
||||
ok(f"Dhikr session — response: {str(dhikr_res['json'])[:80]}")
|
||||
else:
|
||||
warn(f"Dhikr session — HTTP {dhikr_res['status']}")
|
||||
|
||||
# ── Scenario E: Get Dhikr stats ──
|
||||
sub("Scenario E: Retrieve Dhikr stats")
|
||||
stats_res = json_req("api/dhikr/stats", headers=auth_hdrs)
|
||||
if stats_res["json"]:
|
||||
ok(f"Dhikr stats — received data: {str(stats_res['json'])[:100]}")
|
||||
else:
|
||||
warn(f"Dhikr stats — HTTP {stats_res['status']}")
|
||||
|
||||
# ── Scenario F: Qibla API ──
|
||||
sub("Scenario F: Qibla bearing calculation")
|
||||
qibla_res = json_req("api/qibla?lat=3.139&lng=101.6869", headers=auth_hdrs)
|
||||
j = qibla_res["json"]
|
||||
if j and isinstance(j.get("bearing"), (int, float)) and isinstance(j.get("distance"), (int, float)):
|
||||
bearing = j["bearing"]
|
||||
distance = j["distance"]
|
||||
if 280 < bearing < 320:
|
||||
ok(f"Qibla bearing from KL → Mecca: {bearing:.1f}° (correct range)")
|
||||
else:
|
||||
fail(f"Qibla bearing out of range: {bearing}° (expected ~298°)")
|
||||
if 6000 < distance < 8000:
|
||||
ok(f"Qibla distance KL → Mecca: {distance:.0f} km (correct range)")
|
||||
else:
|
||||
fail(f"Qibla distance out of range: {distance} km (expected ~7100)")
|
||||
else:
|
||||
fail(f"Qibla API — unexpected: {str(j)[:100]}")
|
||||
|
||||
# ── Scenario G: Full journey — access all spiritual tools in sequence ──
|
||||
sub("Scenario G: Full spiritual tools journey")
|
||||
journey_pages = ["/prayer", "/dhikr", "/qibla"]
|
||||
all_ok = True
|
||||
for page in journey_pages:
|
||||
r = req(page, headers=auth_hdrs)
|
||||
if r["status"] == 200:
|
||||
# Check no errors
|
||||
if any(re.search(p, r["text"], re.IGNORECASE) for p in ERROR_PATTERNS):
|
||||
fail(f"{page} — errors present (see Layer 3)")
|
||||
all_ok = False
|
||||
else:
|
||||
ok(f"{page} — accessible and clean")
|
||||
else:
|
||||
fail(f"{page} → {r['status']}")
|
||||
all_ok = False
|
||||
if all_ok:
|
||||
ok("Full journey (Prayer → Dhikr → Qibla) — ALL pages clean")
|
||||
|
||||
# ── Scenario H: Location change ──
|
||||
sub("Scenario H: Prayer times for different city")
|
||||
for city, country, label in [
|
||||
("Makkah", "SA", "Mecca"),
|
||||
("Jakarta", "ID", "Jakarta"),
|
||||
("London", "GB", "London"),
|
||||
]:
|
||||
r = json_req(f"api/prayer?city={city}&country={country}")
|
||||
if r["json"] and r["json"].get("timings"):
|
||||
ok(f"Prayer times for {label} — loaded")
|
||||
else:
|
||||
fail(f"Prayer times for {label} — failed: {str(r['json'])[:80]}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 5: EDGE CASES
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_edge():
|
||||
section("LAYER 5: EDGE — Error Handling, Resilience")
|
||||
|
||||
# ── Invalid auth token ──
|
||||
sub("Edge: Invalid auth token")
|
||||
bad = "Bearer eyJ...INVALID...SIG"
|
||||
for path in ["api/prayer", "api/dhikr/stats"]:
|
||||
r = json_req(path, headers={"Authorization": bad})
|
||||
# Public endpoints should still work; protected ones should 401
|
||||
if r["status"] in (200, 401):
|
||||
ok(f"{path} with bad token → {r['status']} (graceful)")
|
||||
else:
|
||||
warn(f"{path} with bad token → {r['status']} (unexpected)")
|
||||
|
||||
# ── Missing required params ──
|
||||
sub("Edge: Missing query parameters")
|
||||
for path in ["api/prayer", "api/qibla"]:
|
||||
r = json_req(path)
|
||||
if r["json"] and r["json"].get("error"):
|
||||
ok(f"{path} — returns error when params missing")
|
||||
elif r["status"] in (400, 422, 500):
|
||||
ok(f"{path} → {r['status']} (error for missing params)")
|
||||
elif r["status"] == 200:
|
||||
warn(f"{path} — 200 with missing params (may silently default)")
|
||||
else:
|
||||
warn(f"{path} → {r['status']} (unexpected)")
|
||||
|
||||
# ── Special characters in params ──
|
||||
sub("Edge: Special characters in city names")
|
||||
specials = [
|
||||
("New York", "US", "space in name"),
|
||||
("São Paulo", "BR", "accented chars"),
|
||||
("Köln", "DE", "umlaut"),
|
||||
("Kuala Lumpur", "MY", "normal ref"),
|
||||
]
|
||||
for city, country, label in specials:
|
||||
r = json_req(f"api/prayer?city={urllib.parse.quote(city)}&country={country}")
|
||||
if r["json"] and r["json"].get("timings"):
|
||||
ok(f"Prayer API — handles '{label}': {city}")
|
||||
else:
|
||||
warn(f"Prayer API — '{label}' failed: {str(r['json'])[:80]}")
|
||||
|
||||
# ── Rapid-fire / concurrency ──
|
||||
sub("Edge: Rapid requests (burst test)")
|
||||
import threading
|
||||
results = []
|
||||
def fire(n):
|
||||
r = req("api/prayer?city=Kuala+Lumpur&country=MY")
|
||||
results.append((n, r["status"], r["elapsed"]))
|
||||
|
||||
threads = [threading.Thread(target=fire, args=(i,)) for i in range(20)]
|
||||
t0 = time.time()
|
||||
for t in threads: t.start()
|
||||
for t in threads: t.join()
|
||||
elapsed = time.time() - t0
|
||||
|
||||
statuses = [s for _, s, _ in results]
|
||||
ok_count = sum(1 for s in statuses if s == 200)
|
||||
fail_count = sum(1 for s in statuses if s != 200)
|
||||
times = [e for _, _, e in results]
|
||||
avg_ms = sum(times) / len(times) * 1000
|
||||
|
||||
if ok_count == 20:
|
||||
ok(f"Burst test — 20/20 returned 200 (avg {avg_ms:.0f}ms, total {elapsed:.1f}s)")
|
||||
else:
|
||||
warn(f"Burst test — {ok_count}/20 returned 200, {fail_count} failed (avg {avg_ms:.0f}ms)")
|
||||
|
||||
# ── Negative: non-existent route ──
|
||||
sub("Edge: Non-existent routes return 404")
|
||||
for path in ["/this-does-not-exist", "/api/nonexistent", "/mobile/undefined-route"]:
|
||||
r = req(path)
|
||||
if r["status"] == 404:
|
||||
ok(f"{path} → 404 (correct)")
|
||||
else:
|
||||
warn(f"{path} → {r['status']} (expected 404)")
|
||||
|
||||
# ── Timeout / large payload ──
|
||||
sub("Edge: Response time sanity")
|
||||
for path in ["/", "/prayer", "/dhikr", "/api/prayer?city=Kuala+Lumpur&country=MY"]:
|
||||
r = req(path, timeout=10)
|
||||
ms = r["elapsed"] * 1000
|
||||
if ms > 3000:
|
||||
warn(f"{path} — slow ({ms:.0f}ms)")
|
||||
elif ms > 10000:
|
||||
fail(f"{path} — timeout risk ({ms:.0f}ms)")
|
||||
else:
|
||||
ok(f"{path} — {ms:.0f}ms")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 6: MOBILE UX
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_mobile():
|
||||
section("LAYER 6: MOBILE UX — Touch, Text, Responsive Design")
|
||||
|
||||
pages = ["/", "/prayer", "/dhikr", "/qibla", "/halal-monitor", "/nur", "/forum", "/souq", "/groups"]
|
||||
|
||||
# ── Viewport meta ──
|
||||
sub("Mobile: Viewport meta tag")
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
if 'name="viewport"' in html or "name='viewport'" in html:
|
||||
ok(f"{path} — viewport meta present")
|
||||
else:
|
||||
fail(f"{path} — MISSING viewport meta tag")
|
||||
|
||||
# ── Touch targets (min-height ≥44px) ──
|
||||
sub("Mobile: Touch targets ≥44px")
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
# Check for 44px patterns
|
||||
has_44 = bool(re.search(r'min-h[^}]*?(?:44|4[5-9]|[5-9]\d)', html, re.IGNORECASE))
|
||||
has_min_h = 'min-h' in html.lower()
|
||||
if has_44:
|
||||
ok(f"{path} — ≥44px touch targets detected")
|
||||
elif has_min_h:
|
||||
warn(f"{path} — has min-h classes but none ≥44px detected")
|
||||
else:
|
||||
warn(f"{path} — no explicit min-h classes found (check CSS approach)")
|
||||
|
||||
# ── Text size ≥12px ──
|
||||
sub("Mobile: No sub-12px text")
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
sub_12 = re.findall(r'text-\[(1[0-1]px)\]', html)
|
||||
if sub_12:
|
||||
sizes = list(set(sub_12))
|
||||
warn(f"{path} — sub-12px text found: {sizes}")
|
||||
else:
|
||||
ok(f"{path} — no sub-12px text")
|
||||
|
||||
# ── Bottom navigation check ──
|
||||
sub("Mobile: Bottom navigation present")
|
||||
for path in ["/", "/prayer", "/dhikr", "/qibla"]:
|
||||
html = req(path)["text"]
|
||||
nav_links = ["Nur", "Souq", "Prayer", "Forum", "Wallet", "Profile"]
|
||||
found = [n for n in nav_links if n.lower() in html.lower()]
|
||||
if len(found) >= 5:
|
||||
ok(f"{path} — bottom nav with {len(found)}/6 links")
|
||||
else:
|
||||
warn(f"{path} — bottom nav incomplete ({len(found)}/6)")
|
||||
|
||||
# ── Touch spacing: tap targets not touching ──
|
||||
sub("Mobile: Basic tap target spacing")
|
||||
for path in ["/prayer", "/dhikr"]:
|
||||
html = req(path)["text"]
|
||||
buttons = re.findall(r'<button[^>]*>', html)
|
||||
if len(buttons) > 0:
|
||||
ok(f"{path} — {len(buttons)} tap targets found")
|
||||
else:
|
||||
warn(f"{path} — no <button> elements found (might use divs)")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 7: PERFORMANCE
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_perf():
|
||||
section("LAYER 7: PERFORMANCE — Response Times & Payload Size")
|
||||
|
||||
pages = [
|
||||
("/", "Landing"),
|
||||
("/prayer", "Prayer"),
|
||||
("/dhikr", "Dhikr"),
|
||||
("/qibla", "Qibla"),
|
||||
("/halal-monitor", "Halal Monitor"),
|
||||
("/api/prayer?city=Kuala+Lumpur&country=MY", "Prayer API (cached)"),
|
||||
]
|
||||
|
||||
for path, label in pages:
|
||||
# First request (cold)
|
||||
cold = req(path)
|
||||
cold_ms = cold["elapsed"] * 1000
|
||||
|
||||
# Second request (warm)
|
||||
warm = req(path)
|
||||
warm_ms = warm["elapsed"] * 1000
|
||||
|
||||
size_kb = len(cold["text"]) / 1024
|
||||
|
||||
status = ok
|
||||
notes = []
|
||||
if cold_ms > 3000:
|
||||
status = warn
|
||||
notes.append(f"cold={cold_ms:.0f}ms")
|
||||
if warm_ms > 1500:
|
||||
status = warn
|
||||
notes.append(f"warm={warm_ms:.0f}ms")
|
||||
if size_kb > 500:
|
||||
status = warn
|
||||
notes.append(f"size={size_kb:.0f}KB")
|
||||
if not notes:
|
||||
notes.append(f"cold={cold_ms:.0f}ms warm={warm_ms:.0f}ms")
|
||||
status(f"{label} ({path}) — {'; '.join(notes)}, {size_kb:.0f}KB")
|
||||
|
||||
sub("Performance: API response times")
|
||||
for endpoint, label in [
|
||||
("api/prayer?city=Kuala+Lumpur&country=MY", "Prayer API"),
|
||||
("api/health", "Health API"),
|
||||
]:
|
||||
r = req(endpoint)
|
||||
ms = r["elapsed"] * 1000
|
||||
if ms > 2000:
|
||||
warn(f"{label} — {ms:.0f}ms (slow)")
|
||||
else:
|
||||
ok(f"{label} — {ms:.0f}ms")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# LAYER 8: SECURITY BASICS
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def layer_security():
|
||||
section("LAYER 8: SECURITY — Headers, Secrets in HTML")
|
||||
|
||||
# ── Security headers ──
|
||||
sub("Security: Response headers")
|
||||
r = req("/")
|
||||
hdrs = r["headers"]
|
||||
checks = [
|
||||
("Content-Type", lambda v: "text/html" in v if v else False),
|
||||
("X-Content-Type-Options", lambda v: v == "nosniff" if v else False),
|
||||
]
|
||||
for hname, check in checks:
|
||||
val = hdrs.get(hname)
|
||||
if val and check(val):
|
||||
ok(f"{hname}: {val}")
|
||||
elif val:
|
||||
warn(f"{hname}: {val} (unexpected value)")
|
||||
else:
|
||||
warn(f"{hname}: MISSING")
|
||||
|
||||
# ── Secrets in HTML ──
|
||||
sub("Security: No secrets leaked in HTML")
|
||||
secret_patterns = [
|
||||
(r'eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+', 'JWT token'),
|
||||
(r'ghp_[A-Za-z0-9]{36}', 'GitHub PAT'),
|
||||
(r'sk-[A-Za-z0-9]{32,}', 'OpenAI key'),
|
||||
(r'-----BEGIN (RSA |EC )?PRIVATE KEY-----', 'Private key'),
|
||||
(r'flh-[a-z0-9]{3,}', 'FLH secret pattern'),
|
||||
]
|
||||
pages = ["/", "/prayer", "/dhikr", "/qibla"]
|
||||
for path in pages:
|
||||
html = req(path)["text"]
|
||||
for pat, desc in secret_patterns:
|
||||
matches = re.findall(pat, html)
|
||||
if matches:
|
||||
fail(f"{path} — possible {desc} leak: {matches[0][:20]}...")
|
||||
else:
|
||||
ok(f"{path} — no {desc} leak")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# MAIN
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def main():
|
||||
global BASE_URL, CI_EXIT
|
||||
parser = argparse.ArgumentParser(description="Falah Mobile QA Test Suite")
|
||||
parser.add_argument("--layer", choices=["system", "api", "render", "scenario", "edge", "mobile", "perf", "security", "all"], default="all")
|
||||
parser.add_argument("--ci-exit", action="store_true")
|
||||
parser.add_argument("--url", default=BASE_URL)
|
||||
args = parser.parse_args()
|
||||
BASE_URL = args.url.rstrip("/")
|
||||
CI_EXIT = args.ci_exit
|
||||
|
||||
print(f"{'='*60}")
|
||||
print(f" Falah Mobile — QA Test Suite v2")
|
||||
print(f" URL: {BASE_URL}")
|
||||
print(f" Layer: {args.layer}")
|
||||
print(f" Time: {time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
import urllib.parse # for special chars test
|
||||
|
||||
layers = {
|
||||
"system": layer_system,
|
||||
"api": layer_api,
|
||||
"render": layer_render,
|
||||
"scenario": layer_scenario,
|
||||
"edge": layer_edge,
|
||||
"mobile": layer_mobile,
|
||||
"perf": layer_perf,
|
||||
"security": layer_security,
|
||||
}
|
||||
|
||||
if args.layer == "all":
|
||||
for name, fn in layers.items():
|
||||
try:
|
||||
fn()
|
||||
except Exception as e:
|
||||
print(f" {R}✗{N} Layer '{name}' crashed: {e}")
|
||||
import traceback; traceback.print_exc()
|
||||
else:
|
||||
fn = layers.get(args.layer)
|
||||
if fn:
|
||||
fn()
|
||||
else:
|
||||
print(f"Unknown layer: {args.layer}")
|
||||
sys.exit(1)
|
||||
|
||||
total = PASS + FAIL + WARN
|
||||
grade = "A"
|
||||
if FAIL > 0: grade = "B"
|
||||
if FAIL > 3: grade = "C"
|
||||
if FAIL > 10: grade = "D"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" QA RESULTS — Grade: {grade}")
|
||||
print(f"{'='*60}")
|
||||
print(f" {G}Pass:{N} {PASS} {R}Fail:{N} {FAIL} {Y}Warn:{N} {WARN} Total: {total}")
|
||||
|
||||
# Generate report
|
||||
report = f"""# QA Test Report
|
||||
**Date:** {time.strftime('%Y-%m-%d %H:%M:%S')}
|
||||
**URL:** {BASE_URL}
|
||||
**Layer:** {args.layer}
|
||||
|
||||
## Summary
|
||||
| ✅ Pass | ❌ Fail | ⚠️ Warn | Total | Grade |
|
||||
|:------:|:------:|:------:|:----:|:-----:|
|
||||
| {PASS} | {FAIL} | {WARN} | {total} | **{grade}** |
|
||||
|
||||
## Details
|
||||
| Result | Message |
|
||||
|:------|:--------|
|
||||
"""
|
||||
for result, msg in REPORT_LINES:
|
||||
icon = {"PASS": "✅", "FAIL": "❌", "WARN": "⚠️"}.get(result, "•")
|
||||
report += f"| {icon} | {msg} |\n"
|
||||
|
||||
report += f"\n## Re-run\n```bash\npython3 tests/qa-test-suite.py{' --ci-exit' if CI_EXIT else ''}\n```"
|
||||
report_file = f"qa-report-{time.strftime('%Y%m%d_%H%M%S')}.md"
|
||||
with open(report_file, "w") as f:
|
||||
f.write(report)
|
||||
print(f" Report: {report_file}")
|
||||
|
||||
if CI_EXIT and FAIL > 0:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,448 @@
|
||||
#!/bin/bash
|
||||
#=============================================================================
|
||||
# Falah Mobile — Comprehensive QA Test Suite
|
||||
#
|
||||
# Multi-layer testing: System → API → Render → User Flows → Edge → Mobile
|
||||
#
|
||||
# Usage:
|
||||
# ./tests/qa-test-suite.sh # Full run
|
||||
# ./tests/qa-test-suite.sh --layer system # Layer 1 only
|
||||
# ./tests/qa-test-suite.sh --ci-exit # Exit 1 on failures
|
||||
#
|
||||
# Layers:
|
||||
# 1. SYSTEM — HTTP status, route existence
|
||||
# 2. API — Response shape, data type validation
|
||||
# 3. RENDER — Page content validates (no error messages in HTML)
|
||||
# 4. FLOW — Multi-step user journeys end-to-end
|
||||
# 5. EDGE — Permission denied, missing params, rate limiting
|
||||
# 6. MOBILE — Viewport, touch targets (≥44px), text size ≥12px
|
||||
#=============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${BASE_URL:-https://falahos.my/mobile}"
|
||||
TIMEOUT=15
|
||||
PASS=0; FAIL=0; WARN=0
|
||||
LAYER_FILTER="all"
|
||||
CI_EXIT=false
|
||||
REPORT_FILE=""
|
||||
|
||||
# Parse args
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--layer=*) LAYER_FILTER="${1#*=}" ;;
|
||||
--ci-exit) CI_EXIT=true ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'; NC='\033[0m'
|
||||
|
||||
pass() { PASS=$((PASS+1)); echo -e " ${GREEN}✓${NC} $1"; }
|
||||
fail() { FAIL=$((FAIL+1)); echo -e " ${RED}✗${NC} $1"; }
|
||||
warn() { WARN=$((WARN+1)); echo -e " ${YELLOW}⚠${NC} $1"; }
|
||||
|
||||
section() { echo ""; echo -e "${CYAN}════════════════════════════════════════${NC}"; echo -e "${CYAN} $1${NC}"; echo -e "${CYAN}════════════════════════════════════════${NC}"; }
|
||||
sub() { echo -e "\n${CYAN}── $1 ──${NC}"; }
|
||||
|
||||
should_run() { [ "$LAYER_FILTER" = "all" ] || [ "$LAYER_FILTER" = "$1" ]; }
|
||||
|
||||
fetch() { curl -s --connect-timeout "$TIMEOUT" "$@" 2>/dev/null || echo ""; }
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# LAYER 1: System — Route Existence & HTTP Status
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
layer_system() {
|
||||
section "LAYER 1: System — Route Existence & HTTP Status"
|
||||
|
||||
local routes=(
|
||||
"/" "Landing"
|
||||
"/prayer" "Prayer"
|
||||
"/dhikr" "Dhikr"
|
||||
"/qibla" "Qibla"
|
||||
"/halal-monitor" "Halal Monitor"
|
||||
"/nur" "Nur AI"
|
||||
"/forum" "Forum"
|
||||
"/souq" "Souq"
|
||||
"/groups" "Groups"
|
||||
"/api/health" "Health API"
|
||||
)
|
||||
|
||||
for ((i=0; i<${#routes[@]}; i+=2)); do
|
||||
local path="${routes[$i]}"
|
||||
local label="${routes[$((i+1))]}"
|
||||
local code
|
||||
code=$(fetch -o /dev/null -w '%{http_code}' "${BASE_URL}${path}")
|
||||
case "$code" in
|
||||
200|308) pass "$label ($path → $code)" ;;
|
||||
000) fail "$label ($path → CONNECTION FAILED)" ;;
|
||||
404) fail "$label ($path → 404 NOT FOUND)" ;;
|
||||
500) fail "$label ($path → 500 SERVER ERROR)" ;;
|
||||
*) fail "$label ($path → $code)" ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# LAYER 2: API Response Shape Validation
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
layer_api() {
|
||||
section "LAYER 2: API — Response Shape & Data Type Validation"
|
||||
|
||||
# —— Prayer API ——
|
||||
sub "Prayer API — timings, date, hijri, city, country, timezone"
|
||||
local resp
|
||||
resp=$(fetch "${BASE_URL}/api/prayer?city=Kuala+Lumpur&country=MY")
|
||||
if echo "$resp" | python3 -c "
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
checks = {
|
||||
'timings (dict)': 'timings' in d and isinstance(d['timings'], dict),
|
||||
'has Fajr time': 'Fajr' in d.get('timings', {}),
|
||||
'date (string)': isinstance(d.get('date'), str) and len(d['date']) > 5,
|
||||
'hijri (string)': isinstance(d.get('hijri'), str) and len(d['hijri']) > 5,
|
||||
'city (string)': isinstance(d.get('city'), str) and len(d['city']) > 0,
|
||||
}
|
||||
all_ok = all(checks.values())
|
||||
for name, ok in checks.items():
|
||||
print(f' {\"✓\" if ok else \"✗\"} {name}')
|
||||
sys.exit(0 if all_ok else 1)
|
||||
" 2>&1; then
|
||||
pass "Prayer API — shape valid"
|
||||
else
|
||||
fail "Prayer API — INVALID shape"
|
||||
echo " Raw: $(echo "$resp" | head -c 200)"
|
||||
fi
|
||||
|
||||
# —— Prayer API with empty params ——
|
||||
sub "Prayer API — empty/missing params (should handle gracefully)"
|
||||
resp=$(fetch "${BASE_URL}/api/prayer?city=&country=")
|
||||
local has_err
|
||||
has_err=*** "$resp" python3 -c "import sys,json; d=json.load(sys.stdin); print('error' in d)" 2>/dev/null || echo "true"
|
||||
if [ "$has_err" = "True" ]; then
|
||||
pass "Prayer API — returns error for empty params"
|
||||
else
|
||||
warn "Prayer API — no error for empty params (may silently default)"
|
||||
fi
|
||||
|
||||
# —— Health API ——
|
||||
sub "Health API — { status, uptime, timestamp }"
|
||||
resp=$(fetch "${BASE_URL}/api/health")
|
||||
if echo "$resp" | python3 -c "
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
assert d.get('status') == 'ok', f'status={d.get(\"status\")}'
|
||||
sys.exit(0)
|
||||
" 2>&1; then
|
||||
pass "Health API — returns ok"
|
||||
else
|
||||
fail "Health API — unexpected response: $(echo "$resp" | head -c 100)"
|
||||
fi
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# LAYER 3: Render Validation (content-level)
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
layer_render() {
|
||||
section "LAYER 3: Render — Page Content Validation"
|
||||
echo " (Checking rendered HTML for error states, not just HTTP codes)"
|
||||
|
||||
local pages=(
|
||||
"/prayer" "Prayer Times" "Invalid response from prayer API"
|
||||
"/dhikr" "Dhikr" "Invalid response"
|
||||
"/qibla" "Qibla" "Invalid response"
|
||||
"/halal-monitor" "Halal Monitor" "Invalid response"
|
||||
"/nur" "Nur" "Invalid response"
|
||||
"/forum" "Forum" "Invalid response"
|
||||
"/souq" "Souq" "Invalid response"
|
||||
"/groups" "Groups" "Invalid response"
|
||||
"/" "Falah" "Invalid response"
|
||||
)
|
||||
|
||||
for ((i=0; i<${#pages[@]}; i+=3)); do
|
||||
local path="${pages[$i]}"
|
||||
local title="${pages[$((i+1))]}"
|
||||
local error_pat="${pages[$((i+2))]}"
|
||||
local content
|
||||
content=$(fetch "${BASE_URL}${path}")
|
||||
|
||||
# Content length
|
||||
local clen
|
||||
clen=*** "$content" wc -c)
|
||||
if [ "$clen" -lt 1000 ]; then
|
||||
warn "$path — very short content (${clen}b), may be blank/error"
|
||||
fi
|
||||
|
||||
# Error strings in rendered output
|
||||
if echo "$content" | grep -qiF "$error_pat"; then
|
||||
fail "$path — CONTAINS ERROR \"$error_pat\" in rendered HTML"
|
||||
else
|
||||
pass "$path — clean of \"$error_pat\""
|
||||
fi
|
||||
|
||||
# Expected title
|
||||
if echo "$content" | grep -qiF "$title"; then
|
||||
pass "$path — contains title \"$title\""
|
||||
else
|
||||
fail "$path — MISSING expected title \"$title\""
|
||||
fi
|
||||
done
|
||||
|
||||
# Scan ALL pages for common error patterns
|
||||
sub "Global scan — common error patterns across all pages"
|
||||
local patterns=(
|
||||
"Invalid response"
|
||||
"Internal Server Error"
|
||||
"Application error"
|
||||
"Cannot read properties"
|
||||
"undefined is not"
|
||||
"TypeError"
|
||||
"failed to fetch"
|
||||
)
|
||||
for pat in "${patterns[@]}"; do
|
||||
local hit=0
|
||||
for path in /prayer /dhikr /qibla /halal-monitor /nur /forum /souq /groups /; do
|
||||
if fetch "${BASE_URL}${path}" | grep -qiF "$pat"; then
|
||||
hit=$((hit+1))
|
||||
fail "${path} → pattern: \"$pat\""
|
||||
fi
|
||||
done
|
||||
[ "$hit" -eq 0 ] && pass "No page contains: \"$pat\""
|
||||
done
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# LAYER 4: User Flow Testing
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
layer_flow() {
|
||||
section "LAYER 4: Flow — Multi-Step User Journeys"
|
||||
|
||||
# Try to get auth token via login API
|
||||
sub "Flow: Login"
|
||||
local login_resp token
|
||||
login_resp=$(fetch -X POST -H "Content-Type: application/json" \
|
||||
-d '{"email":"demo@falahos.my","password":"password123"}' \
|
||||
"${BASE_URL}/api/auth/login")
|
||||
token=*** "$login_resp" python3 -c "import sys,json; print(json.load(sys.stdin).get('token','NO_TOKEN'))" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$token" ] || [ "$token" = "NO_TOKEN" ]; then
|
||||
fail "Login — could not obtain auth token"
|
||||
warn "Skipping authenticated flows"
|
||||
return
|
||||
fi
|
||||
pass "Login — obtained auth token (${token:0:12}...)"
|
||||
|
||||
# —— Flow: Prayer page authenticated ——
|
||||
sub "Flow: Prayer page (authenticated)"
|
||||
local prayer_page
|
||||
prayer_page=$(fetch -H "Authorization: Bearer ${token}" "${BASE_URL}/prayer")
|
||||
if echo "$prayer_page" | grep -qF "Invalid response from prayer API"; then
|
||||
fail "Prayer page — STILL shows API error when authenticated"
|
||||
elif echo "$prayer_page" | grep -qF "Prayer Times"; then
|
||||
pass "Prayer page — renders correctly when authenticated"
|
||||
else
|
||||
warn "Prayer page — unexpected content when authenticated"
|
||||
fi
|
||||
|
||||
# —— Flow: Log a Dhikr session via API ——
|
||||
sub "Flow: Dhikr counter — log session"
|
||||
local dhikr_resp
|
||||
dhikr_resp=$(fetch -X POST -H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer ${token}" \
|
||||
-d '{"type":"subhanallah","count":33,"target":33}' \
|
||||
"${BASE_URL}/api/dhikr")
|
||||
if echo "$dhikr_resp" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d.get('saved') or d.get('id') or d.get('success'), 'no success'; sys.exit(0)" 2>/dev/null; then
|
||||
pass "Dhikr session — logged successfully"
|
||||
else
|
||||
warn "Dhikr session — response: $(echo "$dhikr_resp" | head -c 80)"
|
||||
fi
|
||||
|
||||
# —— Flow: Qibla bearing calculation ——
|
||||
sub "Flow: Qibla — bearing from Kuala Lumpur"
|
||||
local qibla_resp
|
||||
qibla_resp=$(fetch -H "Authorization: Bearer ${token}" \
|
||||
"${BASE_URL}/api/qibla?lat=3.139&lng=101.6869")
|
||||
if echo "$qibla_resp" | python3 -c "
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
b = d.get('bearing', 0)
|
||||
dist = d.get('distance', 0)
|
||||
assert 280 < b < 320, f'bearing {b} out of range for KL→Mecca'
|
||||
assert 6000 < dist < 8000, f'distance {dist} out of range'
|
||||
print(f' Bearing: {b}°, Distance: {dist} km')
|
||||
sys.exit(0)
|
||||
" 2>&1; then
|
||||
pass "Qibla API — bearing correct for KL→Mecca"
|
||||
else
|
||||
fail "Qibla API — unexpected: $(echo "$qibla_resp" | head -c 100)"
|
||||
fi
|
||||
|
||||
# —— Flow: Full journey (Prayer → Dhikr → Qibla) ——
|
||||
sub "Flow: Full journey — all spiritual tools accessible"
|
||||
local all_ok=0
|
||||
for page in prayer dhikr qibla; do
|
||||
local pg
|
||||
pg=$(curl -s -o /dev/null -w '%{http_code}' \
|
||||
-H "Authorization: Bearer ${token}" \
|
||||
--connect-timeout 10 "${BASE_URL}/${page}" 2>/dev/null)
|
||||
[ "$pg" = "200" ] && all_ok=$((all_ok+1))
|
||||
done
|
||||
if [ "$all_ok" -eq 3 ]; then
|
||||
pass "Full journey — Prayer + Dhikr + Qibla all accessible (HTTP 200)"
|
||||
else
|
||||
warn "Full journey — only $all_ok/3 pages returned 200"
|
||||
fi
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# LAYER 5: Edge Cases
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
layer_edge() {
|
||||
section "LAYER 5: Edge — Error States, Permissions, Resilience"
|
||||
|
||||
# —— Unauthenticated access to protected pages ——
|
||||
sub "Edge: Protected pages redirect unauthenticated users"
|
||||
local protected=("/dhikr" "/qibla" "/profile" "/wallet")
|
||||
for path in "${protected[@]}"; do
|
||||
local content
|
||||
content=$(fetch "${BASE_URL}${path}")
|
||||
if echo "$content" | grep -qiE "(login|sign in|Welcome to Falah|redirect|auth)"; then
|
||||
pass "$path — redirects to auth when unauthenticated"
|
||||
else
|
||||
warn "$path — no auth wall detected (may expose content)"
|
||||
fi
|
||||
done
|
||||
|
||||
# —— Invalid auth token ——
|
||||
sub "Edge: Invalid auth token"
|
||||
local bad_resp
|
||||
bad_resp=$(fetch -H "Authorization: Bearer invalid_token_here" "${BASE_URL}/api/prayer")
|
||||
if echo "$bad_resp" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d.get('timings') else 1)" 2>/dev/null; then
|
||||
pass "Prayer API — accessible with invalid token (public data)"
|
||||
else
|
||||
warn "Prayer API — rejects invalid token (may be protected)"
|
||||
fi
|
||||
|
||||
# —— Rapid consecutive requests ——
|
||||
sub "Edge: Rapid requests (10x in parallel)"
|
||||
local ok=0 fail_conn=0
|
||||
for i in $(seq 1 10); do
|
||||
local code
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 \
|
||||
"${BASE_URL}/api/prayer?city=Kuala+Lumpur&country=MY" 2>/dev/null)
|
||||
[ "$code" = "200" ] && ok=$((ok+1)) || fail_conn=$((fail_conn+1))
|
||||
done
|
||||
if [ "$ok" -eq 10 ]; then
|
||||
pass "Rapid requests — 10/10 returned 200"
|
||||
else
|
||||
warn "Rapid requests — ${ok}/10 returned 200, ${fail_conn} failed"
|
||||
fi
|
||||
|
||||
# —— Missing query params on API routes ——
|
||||
sub "Edge: Missing params on various APIs"
|
||||
for endpoint in "api/prayer" "api/qibla"; do
|
||||
local resp
|
||||
resp=$(fetch "${BASE_URL}/${endpoint}")
|
||||
if echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); sys.exit(0 if d.get('error') else 1)" 2>/dev/null; then
|
||||
pass "${endpoint} — returns error when params missing"
|
||||
else
|
||||
warn "${endpoint} — no error for missing params"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# LAYER 6: Mobile Viewport & Touch Targets
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
layer_mobile() {
|
||||
section "LAYER 6: Mobile — Responsive, Touch Targets, Text Size"
|
||||
|
||||
local pages=("/" "/prayer" "/dhikr" "/qibla" "/halal-monitor" "/nur")
|
||||
|
||||
for path in "${pages[@]}"; do
|
||||
local content
|
||||
content=$(fetch "${BASE_URL}${path}")
|
||||
|
||||
# Viewport meta
|
||||
if echo "$content" | grep -qi 'name=.viewport.'; then
|
||||
pass "$path — viewport meta present"
|
||||
else
|
||||
fail "$path — MISSING viewport meta"
|
||||
fi
|
||||
|
||||
# Touch-friendly: min-h >= 44px
|
||||
if echo "$content" | grep -qP 'min-h(?:eight)?[=:].*?(44|4[5-9]|[5-9][0-9])'; then
|
||||
pass "$path — has ≥44px touch targets"
|
||||
else
|
||||
warn "$path — no ≥44px touch target classes detected"
|
||||
fi
|
||||
|
||||
# Text below 12px
|
||||
if echo "$content" | grep -qP 'text-\[1[0-1]px\]'; then
|
||||
warn "$path — has sub-12px text (text-[10-11px])"
|
||||
else
|
||||
pass "$path — no sub-12px text detected"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# Report
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
print_report() {
|
||||
local total=$((PASS + FAIL + WARN))
|
||||
local grade="A"
|
||||
[ "$FAIL" -gt 0 ] && grade="B"
|
||||
[ "$FAIL" -gt 3 ] && grade="C"
|
||||
[ "$FAIL" -gt 10 ] && grade="D"
|
||||
|
||||
echo ""
|
||||
echo -e "${CYAN}════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN} QA TEST RESULTS — Grade: ${grade}${NC}"
|
||||
echo -e "${CYAN}════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo -e " ${GREEN}Pass:${NC} $PASS ${RED}Fail:${NC} $FAIL ${YELLOW}Warn:${NC} $WARN Total: $total"
|
||||
echo ""
|
||||
|
||||
local report_file="qa-report-$(date +%Y%m%d_%H%M%S).md"
|
||||
{
|
||||
echo "# QA Test Report"
|
||||
echo "**Date:** $(date)"
|
||||
echo "**Base URL:** $BASE_URL"
|
||||
echo "**Layer:** $LAYER_FILTER"
|
||||
echo ""
|
||||
echo "## Summary"
|
||||
echo "| ✅ Pass | ❌ Fail | ⚠️ Warn | Total | Grade |"
|
||||
echo "|:------:|:------:|:------:|:----:|:-----:|"
|
||||
echo "| $PASS | $FAIL | $WARN | $total | **$grade** |"
|
||||
echo ""
|
||||
echo "### Re-run with CI exit"
|
||||
echo '```bash'
|
||||
echo "./tests/qa-test-suite.sh --ci-exit"
|
||||
echo '```'
|
||||
} > "$report_file"
|
||||
echo " Report saved to: $report_file"
|
||||
|
||||
if $CI_EXIT && [ "$FAIL" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
# Main
|
||||
#─────────────────────────────────────────────────────────────────
|
||||
echo "╔══════════════════════════════════════════════════════╗"
|
||||
echo "║ Falah Mobile — Comprehensive QA Test Suite ║"
|
||||
echo "║ $(date) ║"
|
||||
echo "╚══════════════════════════════════════════════════════╝"
|
||||
echo " Layer: $LAYER_FILTER URL: $BASE_URL"
|
||||
|
||||
should_run "system" && layer_system
|
||||
should_run "api" && layer_api
|
||||
should_run "render" && layer_render
|
||||
should_run "flow" && layer_flow
|
||||
should_run "edge" && layer_edge
|
||||
should_run "mobile" && layer_mobile
|
||||
|
||||
print_report
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Falah Mobile — Browser-Based Visual QA (Playwright Layer 9)
|
||||
===========================================================
|
||||
Catches client-side rendering errors that curl-based tests miss.
|
||||
Requires: playwright install chromium
|
||||
|
||||
Usage:
|
||||
python3 tests/qa-visual.py # Full visual suite
|
||||
python3 tests/qa-visual.py --url https://staging...
|
||||
python3 tests/qa-visual.py --viewport iphone14
|
||||
python3 tests/qa-visual.py --ci-exit
|
||||
|
||||
Scenarios:
|
||||
1. Page loads without console errors (JS exceptions, 404 fetches)
|
||||
2. Prayer page renders actual prayer times (not error state)
|
||||
3. Dhikr page shows counter (not login wall or error)
|
||||
4. Qibla page handles geolocation denied gracefully
|
||||
5. All async API calls complete within timeout
|
||||
6. No "Invalid response" in rendered DOM post-hydration
|
||||
"""
|
||||
|
||||
import argparse, json, os, sys, time, re
|
||||
|
||||
BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile")
|
||||
TIMEOUT_MS = 30000
|
||||
VIEWPORTS = {
|
||||
"iphone14": {"width": 390, "height": 844},
|
||||
"pixel7": {"width": 412, "height": 915},
|
||||
"ipad": {"width": 820, "height": 1180},
|
||||
"desktop": {"width": 1440, "height": 900},
|
||||
}
|
||||
PASS, FAIL, WARN = 0, 0, 0
|
||||
REPORT_LINES = []
|
||||
|
||||
def ok(msg): global PASS; PASS += 1; print(f" \033[0;32m✓\033[0m {msg}"); REPORT_LINES.append(("PASS", msg))
|
||||
def fail(msg): global FAIL; FAIL += 1; print(f" \033[0;31m✗\033[0m {msg}"); REPORT_LINES.append(("FAIL", msg))
|
||||
def warn(msg): global WARN; WARN += 1; print(f" \033[1;33m⚠\033[0m {msg}"); REPORT_LINES.append(("WARN", msg))
|
||||
def section(title):
|
||||
print(f"\n\033[0;36m════════════════════════════════════════════════════════════\033[0m")
|
||||
print(f"\033[0;36m {title}\033[0m")
|
||||
print(f"\033[0;36m════════════════════════════════════════════════════════════\033[0m")
|
||||
def sub(title):
|
||||
print(f"\n\033[0;36m── {title} ──\033[0m")
|
||||
|
||||
CHECK_SCRIPT = """
|
||||
// ── Wait for React hydration & API calls ──
|
||||
async function waitForPageStable() {
|
||||
// Wait for network to idle
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
// Check for common loading indicators
|
||||
const loadingEls = document.querySelectorAll('[class*="animate-spin"], [class*="loading"], [class*="Loader"]');
|
||||
return loadingEls.length;
|
||||
}
|
||||
// ── Collect all visible text ──
|
||||
function getVisibleText() {
|
||||
return document.body?.innerText || '';
|
||||
}
|
||||
// ── Collect all API fetch URLs ──
|
||||
function getFetchUrls() {
|
||||
return performance.getEntriesByType('resource')
|
||||
.filter(e => e.initiatorType === 'fetch' || e.initiatorType === 'xmlhttprequest')
|
||||
.map(e => ({ url: e.name, status: e.responseStatus, duration: e.duration }));
|
||||
}
|
||||
// ── Check if a text is in the visible DOM ──
|
||||
function hasText(t) {
|
||||
return document.body?.innerText?.includes(t) || false;
|
||||
}
|
||||
// ── Check for error patterns in DOM ──
|
||||
function findErrors() {
|
||||
const patterns = [%s];
|
||||
const body = document.body?.innerHTML || '';
|
||||
return patterns.filter(p => body.toLowerCase().includes(p.toLowerCase()));
|
||||
}
|
||||
"""
|
||||
|
||||
# The actual error patterns to search for — passed as JS array
|
||||
ERROR_PATTERNS_JS = json.dumps([
|
||||
"Invalid response", "Internal Server Error", "Application error",
|
||||
"Cannot read properties of", "undefined is not", "TypeError",
|
||||
"Failed to fetch", "not found", "500 Internal",
|
||||
])
|
||||
|
||||
def vischeck(page, name: str, check_after_load: bool = True, expected_texts: list = None):
|
||||
"""Run a visual check on a page using Playwright."""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
fail("Playwright not installed. Run: pip install playwright && playwright install chromium")
|
||||
return
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context(viewport=VIEWPORTS[viewport])
|
||||
page_obj = context.new_page()
|
||||
|
||||
# Collect console errors
|
||||
console_errors = []
|
||||
page_obj.on("console", lambda msg: console_errors.append(msg.text) if msg.type == "error" else None)
|
||||
page_obj.on("pageerror", lambda err: console_errors.append(f"PAGE ERROR: {err}"))
|
||||
|
||||
try:
|
||||
url = f"{BASE_URL.rstrip('/')}/{name.lstrip('/')}"
|
||||
t0 = time.time()
|
||||
page_obj.goto(url, timeout=TIMEOUT_MS, wait_until="networkidle")
|
||||
load_ms = (time.time() - t0) * 1000
|
||||
|
||||
# Wait for React to settle
|
||||
page_obj.wait_for_timeout(2000)
|
||||
|
||||
# More checks
|
||||
visible_text = page_obj.evaluate("document.body?.innerText || ''")
|
||||
has_errors = page_obj.evaluate(f"document.body?.innerHTML?.toLowerCase() || ''")
|
||||
|
||||
except Exception as e:
|
||||
fail(f"{name} — browser error: {e}")
|
||||
browser.close()
|
||||
return
|
||||
|
||||
# ── Result analysis ──
|
||||
ok(f"{name} — loaded in {load_ms:.0f}ms")
|
||||
|
||||
# 1. Console errors
|
||||
js_errors = [e for e in console_errors if "favicon" not in e.lower() and "404" not in e]
|
||||
if js_errors:
|
||||
for err in js_errors[:3]:
|
||||
fail(f"{name} — console error: {err[:120]}")
|
||||
else:
|
||||
ok(f"{name} — no JS console errors")
|
||||
|
||||
# 2. Expected content (post-hydration)
|
||||
if expected_texts:
|
||||
for et in expected_texts:
|
||||
if et.lower() in visible_text.lower():
|
||||
ok(f"{name} — visible text contains \"{et}\"")
|
||||
else:
|
||||
# Check if it's in the raw HTML (might be inside a shadow DOM or canvas)
|
||||
if et.lower() in (page_obj.content() or "").lower():
|
||||
warn(f"{name} — \"{et}\" in HTML but not visible text (maybe offscreen)")
|
||||
else:
|
||||
fail(f"{name} — \"{et}\" not found in page at all")
|
||||
|
||||
# 3. Error patterns in rendered DOM
|
||||
for pat in json.loads(ERROR_PATTERNS_JS):
|
||||
if pat.lower() in has_errors:
|
||||
idx = has_errors.find(pat.lower())
|
||||
ctx = has_errors[max(0,idx-30):idx+len(pat)+30]
|
||||
fail(f"{name} — ERROR \"{pat}\" in rendered DOM: ...{ctx}...")
|
||||
else:
|
||||
ok(f"{name} — clean of \"{pat}\"")
|
||||
|
||||
# 4. Network fetch analysis
|
||||
fetch_urls = page_obj.evaluate("""
|
||||
performance.getEntriesByType('resource')
|
||||
.filter(e => e.initiatorType === 'fetch')
|
||||
.map(e => ({ url: e.name, status: e.responseStatus, duration: e.duration }))
|
||||
""")
|
||||
failed_fetches = [f for f in fetch_urls if f["status"] and f["status"] >= 400]
|
||||
if failed_fetches:
|
||||
for ff in failed_fetches:
|
||||
short_url = ff["url"].split("/")[-2] + "/" + ff["url"].split("/")[-1] if "/" in ff["url"] else ff["url"]
|
||||
fail(f"{name} — API fetch failed: {short_url} → {ff['status']} ({ff['duration']:.0f}ms)")
|
||||
else:
|
||||
ok(f"{name} — all API fetches succeeded")
|
||||
|
||||
# 5. Loading spinners cleared
|
||||
spinners = page_obj.evaluate("""
|
||||
document.querySelectorAll('[class*="animate-spin"], [class*="loading"], [class*="Loader"]').length
|
||||
""")
|
||||
if spinners > 3:
|
||||
warn(f"{name} — {spinners} loading indicators still visible (may be stuck)")
|
||||
else:
|
||||
ok(f"{name} — loading states resolved ({spinners} remaining)")
|
||||
|
||||
browser.close()
|
||||
|
||||
|
||||
def main():
|
||||
global BASE_URL, viewport
|
||||
parser = argparse.ArgumentParser(description="Falah Mobile Visual QA")
|
||||
parser.add_argument("--url", default=BASE_URL)
|
||||
parser.add_argument("--viewport", choices=list(VIEWPORTS.keys()), default="iphone14")
|
||||
parser.add_argument("--ci-exit", action="store_true")
|
||||
parser.add_argument("--page", choices=["prayer", "dhikr", "qibla", "halal-monitor", "home", "all"], default="all")
|
||||
args = parser.parse_args()
|
||||
BASE_URL = args.url.rstrip("/")
|
||||
viewport = args.viewport
|
||||
CI_EXIT = args.ci_exit
|
||||
|
||||
print(f"{'='*60}")
|
||||
print(f" Falah Mobile — Browser-Based Visual QA")
|
||||
print(f" URL: {BASE_URL}")
|
||||
print(f" Viewport: {args.viewport} {VIEWPORTS[args.viewport]}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
pages = {
|
||||
"home": ("/", ["Assalamualaikum", "Falah", "Daily Verse"]),
|
||||
"prayer": ("/prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha", "Kuala Lumpur"]),
|
||||
"dhikr": ("/dhikr", ["Dhikr", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]),
|
||||
"qibla": ("/qibla", ["Qibla", "Kaaba"]),
|
||||
"halal-monitor": ("/halal-monitor", ["Halal", "Restaurant"]),
|
||||
}
|
||||
|
||||
if args.page == "all":
|
||||
for pname, (path, texts) in pages.items():
|
||||
sub(f"Visual check: {pname} ({path})")
|
||||
try:
|
||||
vischeck({}, pname, path, check_after_load=True, expected_texts=texts)
|
||||
except Exception as e:
|
||||
fail(f"{pname} — crashed: {e}")
|
||||
else:
|
||||
path, texts = pages[args.page]
|
||||
vischeck({}, args.page, path, check_after_load=True, expected_texts=texts)
|
||||
|
||||
total = PASS + FAIL + WARN
|
||||
grade = "A"
|
||||
if FAIL > 0: grade = "B"
|
||||
if FAIL > 3: grade = "C"
|
||||
if FAIL > 5: grade = "D"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f" VISUAL QA RESULTS — Grade: {grade}")
|
||||
print(f"{'='*60}")
|
||||
print(f" Pass: {PASS} Fail: {FAIL} Warn: {WARN} Total: {total}")
|
||||
|
||||
if CI_EXIT and FAIL > 0:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user