root
14d7127e41
- Error Feedback Service: POST /api/feedback saves to JSONL, GET with admin token - ErrorFeedback component: warm amber tones, friendly messages, 'Report Issue' button - Replaced ALL red-themed error displays across 15+ pages with ErrorFeedback - Kind classification: network, auth, location, upgrade, default messages - QA test suite v2: 8-layer testing (system, API, render, scenario, edge, mobile, perf, security) - Browser-based visual QA with Playwright (Layer 9)
231 lines
9.4 KiB
Python
231 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Falah Mobile — Browser-Based Visual QA (Playwright Layer 9)
|
|
===========================================================
|
|
Catches client-side rendering errors that curl-based tests miss.
|
|
Requires: playwright install chromium
|
|
|
|
Usage:
|
|
python3 tests/qa-visual.py # Full visual suite
|
|
python3 tests/qa-visual.py --url https://staging...
|
|
python3 tests/qa-visual.py --viewport iphone14
|
|
python3 tests/qa-visual.py --ci-exit
|
|
|
|
Scenarios:
|
|
1. Page loads without console errors (JS exceptions, 404 fetches)
|
|
2. Prayer page renders actual prayer times (not error state)
|
|
3. Dhikr page shows counter (not login wall or error)
|
|
4. Qibla page handles geolocation denied gracefully
|
|
5. All async API calls complete within timeout
|
|
6. No "Invalid response" in rendered DOM post-hydration
|
|
"""
|
|
|
|
import argparse, json, os, sys, time, re
|
|
|
|
BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile")
|
|
TIMEOUT_MS = 30000
|
|
VIEWPORTS = {
|
|
"iphone14": {"width": 390, "height": 844},
|
|
"pixel7": {"width": 412, "height": 915},
|
|
"ipad": {"width": 820, "height": 1180},
|
|
"desktop": {"width": 1440, "height": 900},
|
|
}
|
|
PASS, FAIL, WARN = 0, 0, 0
|
|
REPORT_LINES = []
|
|
|
|
def ok(msg): global PASS; PASS += 1; print(f" \033[0;32m✓\033[0m {msg}"); REPORT_LINES.append(("PASS", msg))
|
|
def fail(msg): global FAIL; FAIL += 1; print(f" \033[0;31m✗\033[0m {msg}"); REPORT_LINES.append(("FAIL", msg))
|
|
def warn(msg): global WARN; WARN += 1; print(f" \033[1;33m⚠\033[0m {msg}"); REPORT_LINES.append(("WARN", msg))
|
|
def section(title):
|
|
print(f"\n\033[0;36m════════════════════════════════════════════════════════════\033[0m")
|
|
print(f"\033[0;36m {title}\033[0m")
|
|
print(f"\033[0;36m════════════════════════════════════════════════════════════\033[0m")
|
|
def sub(title):
|
|
print(f"\n\033[0;36m── {title} ──\033[0m")
|
|
|
|
CHECK_SCRIPT = """
|
|
// ── Wait for React hydration & API calls ──
|
|
async function waitForPageStable() {
|
|
// Wait for network to idle
|
|
await new Promise(r => setTimeout(r, 2000));
|
|
// Check for common loading indicators
|
|
const loadingEls = document.querySelectorAll('[class*="animate-spin"], [class*="loading"], [class*="Loader"]');
|
|
return loadingEls.length;
|
|
}
|
|
// ── Collect all visible text ──
|
|
function getVisibleText() {
|
|
return document.body?.innerText || '';
|
|
}
|
|
// ── Collect all API fetch URLs ──
|
|
function getFetchUrls() {
|
|
return performance.getEntriesByType('resource')
|
|
.filter(e => e.initiatorType === 'fetch' || e.initiatorType === 'xmlhttprequest')
|
|
.map(e => ({ url: e.name, status: e.responseStatus, duration: e.duration }));
|
|
}
|
|
// ── Check if a text is in the visible DOM ──
|
|
function hasText(t) {
|
|
return document.body?.innerText?.includes(t) || false;
|
|
}
|
|
// ── Check for error patterns in DOM ──
|
|
function findErrors() {
|
|
const patterns = [%s];
|
|
const body = document.body?.innerHTML || '';
|
|
return patterns.filter(p => body.toLowerCase().includes(p.toLowerCase()));
|
|
}
|
|
"""
|
|
|
|
# The actual error patterns to search for — passed as JS array
|
|
ERROR_PATTERNS_JS = json.dumps([
|
|
"Invalid response", "Internal Server Error", "Application error",
|
|
"Cannot read properties of", "undefined is not", "TypeError",
|
|
"Failed to fetch", "not found", "500 Internal",
|
|
])
|
|
|
|
def vischeck(page, name: str, check_after_load: bool = True, expected_texts: list = None):
|
|
"""Run a visual check on a page using Playwright."""
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
except ImportError:
|
|
fail("Playwright not installed. Run: pip install playwright && playwright install chromium")
|
|
return
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
context = browser.new_context(viewport=VIEWPORTS[viewport])
|
|
page_obj = context.new_page()
|
|
|
|
# Collect console errors
|
|
console_errors = []
|
|
page_obj.on("console", lambda msg: console_errors.append(msg.text) if msg.type == "error" else None)
|
|
page_obj.on("pageerror", lambda err: console_errors.append(f"PAGE ERROR: {err}"))
|
|
|
|
try:
|
|
url = f"{BASE_URL.rstrip('/')}/{name.lstrip('/')}"
|
|
t0 = time.time()
|
|
page_obj.goto(url, timeout=TIMEOUT_MS, wait_until="networkidle")
|
|
load_ms = (time.time() - t0) * 1000
|
|
|
|
# Wait for React to settle
|
|
page_obj.wait_for_timeout(2000)
|
|
|
|
# More checks
|
|
visible_text = page_obj.evaluate("document.body?.innerText || ''")
|
|
has_errors = page_obj.evaluate(f"document.body?.innerHTML?.toLowerCase() || ''")
|
|
|
|
except Exception as e:
|
|
fail(f"{name} — browser error: {e}")
|
|
browser.close()
|
|
return
|
|
|
|
# ── Result analysis ──
|
|
ok(f"{name} — loaded in {load_ms:.0f}ms")
|
|
|
|
# 1. Console errors
|
|
js_errors = [e for e in console_errors if "favicon" not in e.lower() and "404" not in e]
|
|
if js_errors:
|
|
for err in js_errors[:3]:
|
|
fail(f"{name} — console error: {err[:120]}")
|
|
else:
|
|
ok(f"{name} — no JS console errors")
|
|
|
|
# 2. Expected content (post-hydration)
|
|
if expected_texts:
|
|
for et in expected_texts:
|
|
if et.lower() in visible_text.lower():
|
|
ok(f"{name} — visible text contains \"{et}\"")
|
|
else:
|
|
# Check if it's in the raw HTML (might be inside a shadow DOM or canvas)
|
|
if et.lower() in (page_obj.content() or "").lower():
|
|
warn(f"{name} — \"{et}\" in HTML but not visible text (maybe offscreen)")
|
|
else:
|
|
fail(f"{name} — \"{et}\" not found in page at all")
|
|
|
|
# 3. Error patterns in rendered DOM
|
|
for pat in json.loads(ERROR_PATTERNS_JS):
|
|
if pat.lower() in has_errors:
|
|
idx = has_errors.find(pat.lower())
|
|
ctx = has_errors[max(0,idx-30):idx+len(pat)+30]
|
|
fail(f"{name} — ERROR \"{pat}\" in rendered DOM: ...{ctx}...")
|
|
else:
|
|
ok(f"{name} — clean of \"{pat}\"")
|
|
|
|
# 4. Network fetch analysis
|
|
fetch_urls = page_obj.evaluate("""
|
|
performance.getEntriesByType('resource')
|
|
.filter(e => e.initiatorType === 'fetch')
|
|
.map(e => ({ url: e.name, status: e.responseStatus, duration: e.duration }))
|
|
""")
|
|
failed_fetches = [f for f in fetch_urls if f["status"] and f["status"] >= 400]
|
|
if failed_fetches:
|
|
for ff in failed_fetches:
|
|
short_url = ff["url"].split("/")[-2] + "/" + ff["url"].split("/")[-1] if "/" in ff["url"] else ff["url"]
|
|
fail(f"{name} — API fetch failed: {short_url} → {ff['status']} ({ff['duration']:.0f}ms)")
|
|
else:
|
|
ok(f"{name} — all API fetches succeeded")
|
|
|
|
# 5. Loading spinners cleared
|
|
spinners = page_obj.evaluate("""
|
|
document.querySelectorAll('[class*="animate-spin"], [class*="loading"], [class*="Loader"]').length
|
|
""")
|
|
if spinners > 3:
|
|
warn(f"{name} — {spinners} loading indicators still visible (may be stuck)")
|
|
else:
|
|
ok(f"{name} — loading states resolved ({spinners} remaining)")
|
|
|
|
browser.close()
|
|
|
|
|
|
def main():
|
|
global BASE_URL, viewport
|
|
parser = argparse.ArgumentParser(description="Falah Mobile Visual QA")
|
|
parser.add_argument("--url", default=BASE_URL)
|
|
parser.add_argument("--viewport", choices=list(VIEWPORTS.keys()), default="iphone14")
|
|
parser.add_argument("--ci-exit", action="store_true")
|
|
parser.add_argument("--page", choices=["prayer", "dhikr", "qibla", "halal-monitor", "home", "all"], default="all")
|
|
args = parser.parse_args()
|
|
BASE_URL = args.url.rstrip("/")
|
|
viewport = args.viewport
|
|
CI_EXIT = args.ci_exit
|
|
|
|
print(f"{'='*60}")
|
|
print(f" Falah Mobile — Browser-Based Visual QA")
|
|
print(f" URL: {BASE_URL}")
|
|
print(f" Viewport: {args.viewport} {VIEWPORTS[args.viewport]}")
|
|
print(f"{'='*60}")
|
|
|
|
pages = {
|
|
"home": ("/", ["Assalamualaikum", "Falah", "Daily Verse"]),
|
|
"prayer": ("/prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha", "Kuala Lumpur"]),
|
|
"dhikr": ("/dhikr", ["Dhikr", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]),
|
|
"qibla": ("/qibla", ["Qibla", "Kaaba"]),
|
|
"halal-monitor": ("/halal-monitor", ["Halal", "Restaurant"]),
|
|
}
|
|
|
|
if args.page == "all":
|
|
for pname, (path, texts) in pages.items():
|
|
sub(f"Visual check: {pname} ({path})")
|
|
try:
|
|
vischeck({}, pname, path, check_after_load=True, expected_texts=texts)
|
|
except Exception as e:
|
|
fail(f"{pname} — crashed: {e}")
|
|
else:
|
|
path, texts = pages[args.page]
|
|
vischeck({}, args.page, path, check_after_load=True, expected_texts=texts)
|
|
|
|
total = PASS + FAIL + WARN
|
|
grade = "A"
|
|
if FAIL > 0: grade = "B"
|
|
if FAIL > 3: grade = "C"
|
|
if FAIL > 5: grade = "D"
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f" VISUAL QA RESULTS — Grade: {grade}")
|
|
print(f"{'='*60}")
|
|
print(f" Pass: {PASS} Fail: {FAIL} Warn: {WARN} Total: {total}")
|
|
|
|
if CI_EXIT and FAIL > 0:
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|