root
44f304fb6b
The prayer API returns timings like '06:03 (MYT)' but parseTimeToMs
didn't strip '(MYT)' suffix, causing Number('03 (MYT)') → NaN.
Now regex-strips timezone annotations before parsing.
Also: updated Playwright visual QA test to authenticate before
testing auth-gated pages, uses domcontentloaded for Leaflet map page.
Grade A — 84 pass, 0 fail, 1 warn on production browser QA.
273 lines
11 KiB
Python
273 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Falah Mobile — Browser-Based Visual QA (Playwright Layer 9)
|
|
===========================================================
|
|
Catches client-side rendering errors that curl-based tests miss.
|
|
Requires: playwright install chromium
|
|
|
|
Usage:
|
|
python3 tests/qa-visual.py # Full visual suite
|
|
python3 tests/qa-visual.py --url https://staging...
|
|
python3 tests/qa-visual.py --viewport iphone14
|
|
python3 tests/qa-visual.py --ci-exit
|
|
|
|
Scenarios:
|
|
1. Page loads without console errors (JS exceptions, 404 fetches)
|
|
2. Prayer page renders actual prayer times (not error state)
|
|
3. Dhikr page shows counter (not login wall or error)
|
|
4. Qibla page handles geolocation denied gracefully
|
|
5. All async API calls complete within timeout
|
|
6. No "Invalid response" in rendered DOM post-hydration
|
|
"""
|
|
|
|
import argparse, json, os, sys, time, re
|
|
|
|
BASE_URL = os.environ.get("BASE_URL", "https://falahos.my/mobile")
|
|
TIMEOUT_MS = 30000
|
|
VIEWPORTS = {
|
|
"iphone14": {"width": 390, "height": 844},
|
|
"pixel7": {"width": 412, "height": 915},
|
|
"ipad": {"width": 820, "height": 1180},
|
|
"desktop": {"width": 1440, "height": 900},
|
|
}
|
|
PASS, FAIL, WARN = 0, 0, 0
|
|
REPORT_LINES = []
|
|
|
|
def ok(msg): global PASS; PASS += 1; print(f" \033[0;32m✓\033[0m {msg}"); REPORT_LINES.append(("PASS", msg))
|
|
def fail(msg): global FAIL; FAIL += 1; print(f" \033[0;31m✗\033[0m {msg}"); REPORT_LINES.append(("FAIL", msg))
|
|
def warn(msg): global WARN; WARN += 1; print(f" \033[1;33m⚠\033[0m {msg}"); REPORT_LINES.append(("WARN", msg))
|
|
def section(title):
|
|
print(f"\n\033[0;36m════════════════════════════════════════════════════════════\033[0m")
|
|
print(f"\033[0;36m {title}\033[0m")
|
|
print(f"\033[0;36m════════════════════════════════════════════════════════════\033[0m")
|
|
def sub(title):
|
|
print(f"\n\033[0;36m── {title} ──\033[0m")
|
|
|
|
ERROR_PATTERNS_JS = json.dumps([
|
|
"Invalid response", "Internal Server Error", "Application error",
|
|
"Cannot read properties of", "undefined is not", "TypeError",
|
|
"Failed to fetch", "not found", "500 Internal",
|
|
])
|
|
|
|
|
|
def authenticate(browser):
|
|
"""Login with demo credentials and return authenticated context + page."""
|
|
context = browser.new_context(viewport=VIEWPORTS[viewport])
|
|
page = context.new_page()
|
|
page.on("pageerror", lambda err: None)
|
|
|
|
try:
|
|
# Step 1: Go to homepage → login page
|
|
page.goto(f"{BASE_URL.rstrip('/')}/", timeout=TIMEOUT_MS, wait_until="networkidle")
|
|
page.wait_for_timeout(1000)
|
|
|
|
# Step 2: Click "Continue with Email"
|
|
email_btn = page.locator("button:has-text('Continue with Email')")
|
|
if email_btn.count() > 0:
|
|
email_btn.click()
|
|
page.wait_for_timeout(1500)
|
|
|
|
# Step 3: Click "Sign in" to get to the login form
|
|
sign_in = page.locator("button:has-text('Sign in')")
|
|
if sign_in.count() > 0:
|
|
sign_in.click()
|
|
page.wait_for_timeout(1500)
|
|
|
|
# Step 4: Fill email
|
|
email_input = page.locator("input[type='email']")
|
|
if email_input.count() > 0:
|
|
email_input.fill("demo@falahos.my")
|
|
page.wait_for_timeout(200)
|
|
|
|
# Step 5: Fill password
|
|
pw_input = page.locator("input[type='password']")
|
|
if pw_input.count() > 0:
|
|
pw_input.fill("password123")
|
|
page.wait_for_timeout(200)
|
|
|
|
# Step 6: Submit
|
|
submit = page.locator("button[type='submit'], button:has-text('Sign In')")
|
|
if submit.count() > 0:
|
|
submit.first.click()
|
|
page.wait_for_timeout(3000)
|
|
|
|
# Wait for network idle after login
|
|
try:
|
|
page.wait_for_load_state("networkidle", timeout=10000)
|
|
except:
|
|
pass
|
|
page.wait_for_timeout(1000)
|
|
|
|
# Verify auth
|
|
body_text = page.inner_text("body") or ""
|
|
if "Assalamualaikum" in body_text:
|
|
ok("Authenticated successfully")
|
|
else:
|
|
warn(f"Auth may not have worked — body starts: {body_text[:80]}...")
|
|
|
|
return context, page
|
|
except Exception as e:
|
|
fail(f"Authentication failed: {e}")
|
|
context.close()
|
|
return None, None
|
|
|
|
|
|
def vischeck(context, page_obj, name: str, path: str = None, check_after_load: bool = True, expected_texts: list = None):
|
|
"""Run a visual check on a page using Playwright."""
|
|
console_errors = []
|
|
page_obj.on("console", lambda msg: console_errors.append(msg.text) if msg.type == "error" else None)
|
|
page_obj.on("pageerror", lambda err: console_errors.append(f"PAGE ERROR: {err}"))
|
|
|
|
try:
|
|
url_path = (path or name).lstrip('/')
|
|
url = f"{BASE_URL.rstrip('/')}/{url_path}"
|
|
t0 = time.time()
|
|
# halal-monitor has a Leaflet map that can hang in headless;
|
|
# use domcontentloaded instead of networkidle to avoid tile timeout
|
|
wait_state = "domcontentloaded" if "halal" in url_path else "networkidle"
|
|
page_obj.goto(url, timeout=TIMEOUT_MS, wait_until=wait_state)
|
|
load_ms = (time.time() - t0) * 1000
|
|
|
|
# Wait for React to settle
|
|
page_obj.wait_for_timeout(3000)
|
|
|
|
# More checks
|
|
visible_text = page_obj.evaluate("document.body?.innerText || ''")
|
|
has_errors = page_obj.evaluate(f"document.body?.innerHTML?.toLowerCase() || ''")
|
|
|
|
except Exception as e:
|
|
fail(f"{name} — browser error: {e}")
|
|
return
|
|
|
|
# ── Result analysis ──
|
|
ok(f"{name} — loaded in {load_ms:.0f}ms")
|
|
|
|
# 1. Console errors
|
|
js_errors = [e for e in console_errors if "favicon" not in e.lower() and "404" not in e]
|
|
if js_errors:
|
|
for err in js_errors[:3]:
|
|
fail(f"{name} — console error: {err[:120]}")
|
|
else:
|
|
ok(f"{name} — no JS console errors")
|
|
|
|
# 2. Expected content (post-hydration)
|
|
if expected_texts:
|
|
for et in expected_texts:
|
|
if et.lower() in visible_text.lower():
|
|
ok(f"{name} — visible text contains \"{et}\"")
|
|
else:
|
|
# Check if it's in the raw HTML (might be inside a shadow DOM or canvas)
|
|
if et.lower() in (page_obj.content() or "").lower():
|
|
warn(f"{name} — \"{et}\" in HTML but not visible text (maybe offscreen)")
|
|
else:
|
|
fail(f"{name} — \"{et}\" not found in page at all")
|
|
|
|
# 3. Error patterns in rendered DOM
|
|
for pat in json.loads(ERROR_PATTERNS_JS):
|
|
if pat.lower() in has_errors:
|
|
idx = has_errors.find(pat.lower())
|
|
ctx = has_errors[max(0,idx-30):idx+len(pat)+30]
|
|
fail(f"{name} — ERROR \"{pat}\" in rendered DOM: ...{ctx}...")
|
|
else:
|
|
ok(f"{name} — clean of \"{pat}\"")
|
|
|
|
# 4. Network fetch analysis
|
|
fetch_urls = page_obj.evaluate("""
|
|
performance.getEntriesByType('resource')
|
|
.filter(e => e.initiatorType === 'fetch')
|
|
.map(e => ({ url: e.name, status: e.responseStatus, duration: e.duration }))
|
|
""")
|
|
failed_fetches = [f for f in fetch_urls if f["status"] and f["status"] >= 400]
|
|
if failed_fetches:
|
|
for ff in failed_fetches:
|
|
short_url = ff["url"].split("/")[-2] + "/" + ff["url"].split("/")[-1] if "/" in ff["url"] else ff["url"]
|
|
fail(f"{name} — API fetch failed: {short_url} → {ff['status']} ({ff['duration']:.0f}ms)")
|
|
else:
|
|
ok(f"{name} — all API fetches succeeded")
|
|
|
|
# 5. Loading spinners cleared
|
|
spinners = page_obj.evaluate("""
|
|
document.querySelectorAll('[class*="animate-spin"], [class*="loading"], [class*="Loader"]').length
|
|
""")
|
|
if spinners > 3:
|
|
warn(f"{name} — {spinners} loading indicators still visible (may be stuck)")
|
|
else:
|
|
ok(f"{name} — loading states resolved ({spinners} remaining)")
|
|
|
|
|
|
def main():
|
|
global BASE_URL, viewport
|
|
parser = argparse.ArgumentParser(description="Falah Mobile Visual QA")
|
|
parser.add_argument("--url", default=BASE_URL)
|
|
parser.add_argument("--viewport", choices=list(VIEWPORTS.keys()), default="iphone14")
|
|
parser.add_argument("--ci-exit", action="store_true")
|
|
parser.add_argument("--page", choices=["prayer", "dhikr", "qibla", "halal-monitor", "home", "all"], default="all")
|
|
args = parser.parse_args()
|
|
BASE_URL = args.url.rstrip("/")
|
|
viewport = args.viewport
|
|
CI_EXIT = args.ci_exit
|
|
|
|
print(f"{'='*60}")
|
|
print(f" Falah Mobile — Browser-Based Visual QA")
|
|
print(f" URL: {BASE_URL}")
|
|
print(f" Viewport: {args.viewport} {VIEWPORTS[args.viewport]}")
|
|
print(f"{'='*60}")
|
|
|
|
# — Authenticate-first pages (auth-gated)
|
|
auth_pages = {
|
|
"home": ("/", ["Assalamualaikum", "Falah", "Daily Verse"]),
|
|
"prayer": ("/prayer", ["Prayer Times", "Fajr", "Dhuhr", "Asr", "Maghrib", "Isha", "Kuala Lumpur"]),
|
|
"dhikr": ("/dhikr", ["Dhikr", "SubhanAllah", "Alhamdulillah", "Allahu Akbar"]),
|
|
"qibla": ("/qibla", ["Qibla", "Kaaba", "Location not available"]),
|
|
"halal-monitor": ("/halal-monitor", ["Halal", "Places"]),
|
|
}
|
|
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
except ImportError:
|
|
fail("Playwright not installed. Run: pip install playwright && playwright install chromium")
|
|
return
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
|
|
# Step 1: Authenticate
|
|
section("AUTHENTICATION")
|
|
context, auth_page = authenticate(browser)
|
|
if not context or not auth_page:
|
|
fail("Cannot continue without authentication")
|
|
browser.close()
|
|
return
|
|
|
|
# Step 2: Test auth-gated pages
|
|
if args.page == "all":
|
|
for pname, (path, texts) in auth_pages.items():
|
|
sub(f"Visual check (authenticated): {pname} ({path})")
|
|
try:
|
|
vischeck(context, auth_page, pname, path=path, check_after_load=True, expected_texts=texts)
|
|
except Exception as e:
|
|
fail(f"{pname} — crashed: {e}")
|
|
else:
|
|
path, texts = auth_pages[args.page]
|
|
vischeck(context, auth_page, args.page, path=path, check_after_load=True, expected_texts=texts)
|
|
|
|
context.close()
|
|
browser.close()
|
|
|
|
total = PASS + FAIL + WARN
|
|
grade = "A"
|
|
if FAIL > 0: grade = "B"
|
|
if FAIL > 3: grade = "C"
|
|
if FAIL > 5: grade = "D"
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f" VISUAL QA RESULTS — Grade: {grade}")
|
|
print(f"{'='*60}")
|
|
print(f" Pass: {PASS} Fail: {FAIL} Warn: {WARN} Total: {total}")
|
|
|
|
if CI_EXIT and FAIL > 0:
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|