nur-muslim-companion/scripts/scout_scrape.py

#!/usr/bin/env python3
"""
Weekly Scout MCP Lead Scrape — Run every Sunday.
Scrapes GitHub for Islamic tech contributors, enriches them, saves to quality_leads.json.
"""

import os, sys, json, csv, time
from datetime import datetime

sys.path.insert(0, "/Users/wmj24/.hermes/mcp-servers/scout-server/src")
from app.scrapers.github import scrape_profile
from app.scrapers.enrichment import enrich_lead

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LEADS_FILE = os.path.join(BASE_DIR, "quality_leads.json")

def load_leads():
    if os.path.exists(LEADS_FILE):
        with open(LEADS_FILE) as f:
            return json.load(f)
    return []

def save_leads(leads):
    with open(LEADS_FILE, "w") as f:
        json.dump(leads, f, indent=2)

def export_csv(leads):
    csv_path = os.path.join(BASE_DIR, "quality_leads.csv")
    with open(csv_path, "w", newline="") as f:
        fieldnames = ["email", "firstname", "lastname", "company", "position",
                      "tags", "source", "profile_url", "website", "lead_score", "notes"]
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for lead in leads:
            row = lead.copy()
            row["tags"] = "; ".join(row.get("tags", []))
            writer.writerow({k: row.get(k, "") for k in fieldnames})

def is_relevant(profile):
    """Check if a GitHub profile is relevant to Islamic tech."""
    bio = (profile.get("bio") or "").lower()
    name = (profile.get("full_name") or "").lower()
    username = (profile.get("username") or "").lower()
    company = (profile.get("company") or "").lower()
    website = (profile.get("website") or "").lower()

    keywords = [
        "muslim", "islam", "quran", "prayer", "halal", "arabic",
        "islamic", "deen", "faith", "ummah", "allah", "mosque",
        "saudi", "indonesia", "malaysia", "egypt", "pakistan",
        "turkey", "iran", "morocco", "tunisia", "algeria",
        "islamic", "muslim", "falah", "nur", "deen",
    ]

    text = " ".join([bio, name, username, company, website])
    return any(kw in text for kw in keywords)

def main():
    print(f"[{datetime.now()}] Scout weekly scrape starting...")

    existing = load_leads()
    known_usernames = {l.get("username") for l in existing}

    # Search queries for Islamic tech contributors
    search_queries = [
        ("quran api", "javascript", 50),
        ("prayer times", "javascript", 30),
        ("islamic", "typescript", 30),
        ("halal", "javascript", 30),
        ("muslim app", "javascript", 30),
        ("adhan", "javascript", 30),
        ("qibla", "javascript", 30),
        ("islamic finance", "javascript", 20),
        ("muslim developer", "javascript", 20),
    ]

    new_leads = []
    for query, lang, min_stars in search_queries:
        print(f"  Searching: '{query}' ({lang},>{min_stars} stars)...")
        try:
            r = requests.get(
                "https://api.github.com/search/repositories",
                params={
                    "q": f"{query} language:{lang} stars:>{min_stars}",
                    "per_page": "20",
                    "sort": "stars",
                },
                headers={"Accept": "application/vnd.github.v3+json"},
                timeout=15,
            )
            if r.status_code != 200:
                print(f"    Search failed: {r.status_code}")
                continue

            repos = r.json().get("items", [])
            print(f"    Found {len(repos)} repos")

            for repo in repos:
                owner = repo["owner"]["login"]
                if owner in known_usernames:
                    continue

                try:
                    profile = scrape_profile(owner)
                    if is_relevant(profile):
                        print(f"    Enriching: {owner}...")
                        time.sleep(2)  # Rate limit
                        enriched = enrich_lead(profile)
                        new_leads.append(enriched)
                        known_usernames.add(owner)
                        print(f"      ✓ Score: {enriched.get('lead_score', 0)}")
                    else:
                        known_usernames.add(owner)
                except Exception as e:
                    print(f"      ✗ Error: {e}")

        except Exception as e:
            print(f"    Query error: {e}")

    # Merge and save
    all_leads = existing + new_leads
    save_leads(all_leads)
    export_csv(all_leads)

    print(f"\n[{datetime.now()}] Scout scrape complete.")
    print(f"  New leads: {len(new_leads)}")
    print(f"  Total leads: {len(all_leads)}")
    print(f"  Saved to: {LEADS_FILE}")
    print(f"  CSV exported to: {os.path.join(BASE_DIR, 'quality_leads.csv')}")

if __name__ == "__main__":
    # Need requests
    import requests
    main()