football-api/scraper/scraper.py

#!/usr/bin/env python3
"""
FC Porto fixture scraper — ESPN
Server-side rendered, no JS needed, no API key required.
Writes /data/porto.json
"""

import json
import logging
import os
import re
from datetime import datetime
from pathlib import Path
from zoneinfo import ZoneInfo

import httpx
from bs4 import BeautifulSoup

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger(__name__)

DATA_FILE = Path("/data/porto.json")
DATA_FILE.parent.mkdir(parents=True, exist_ok=True)

TEAM_ID   = os.environ.get("ESPN_TEAM_ID", "437")
TEAM_SLUG = os.environ.get("ESPN_TEAM_SLUG", "por.porto")
TEAM_NAME = os.environ.get("TEAM_NAME", "FC Porto")
PT_TZ     = ZoneInfo("Europe/Lisbon")

COMP_MAP = {
    "Portuguese Primeira Liga": "LP",
    "UEFA Europa League":       "UE",
    "UEFA Champions League":    "CL",
    "Taca de Portugal":         "TP",
    "Taça de Portugal":         "TP",
    "FIFA Club World Cup":      "CW",
    "UEFA Super Cup":           "SC",
}

HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
                  "Chrome/124.0 Safari/537.36",
    "Accept-Language": "en-GB,en;q=0.9",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}


def scrape() -> list[dict]:
    log.info("Scraping ESPN for %s (id=%s)...", TEAM_NAME, TEAM_ID)

    base      = "https://www.espn.com/soccer/team"
    urls      = {
        "future": f"{base}/fixtures/_/id/{TEAM_ID}/{TEAM_SLUG}",
        "past":   f"{base}/results/_/id/{TEAM_ID}/{TEAM_SLUG}",
    }

    porto_names = {"fc porto", "porto"}
    matches     = []

    for mode, url in urls.items():
        is_past = mode == "past"

        try:
            r = httpx.get(url, headers=HEADERS, timeout=30, follow_redirects=True)
            r.raise_for_status()
            soup = BeautifulSoup(r.text, "lxml")
        except Exception as e:
            log.error("Failed to fetch %s: %s", url, e)
            continue

        for table in soup.find_all("table"):
            for row in table.find_all("tr"):
                cells = row.find_all("td")
                if len(cells) < 4:
                    continue
                try:
                    # ── Date ──────────────────────────────────────────
                    date_text = cells[0].get_text(strip=True)

                    # ── Teams + logos ──────────────────────────────────
                    match_cell = cells[1]
                    team_links = match_cell.find_all(
                        "a", href=re.compile(r"/soccer/team/_/id/\d+")
                    )
                    if len(team_links) < 2:
                        continue

                    home      = team_links[0].get_text(strip=True)
                    away      = team_links[-1].get_text(strip=True)
                    home_id   = re.search(r"/id/(\d+)/", team_links[0]["href"])
                    away_id   = re.search(r"/id/(\d+)/", team_links[-1]["href"])
                    home_logo = (
                        f"https://a.espncdn.com/i/teamlogos/soccer/500/{home_id.group(1)}.png"
                        if home_id else None
                    )
                    away_logo = (
                        f"https://a.espncdn.com/i/teamlogos/soccer/500/{away_id.group(1)}.png"
                        if away_id else None
                    )

                    is_home      = home.lower() in porto_names
                    opponent     = away if is_home else home
                    opponent_logo = away_logo if is_home else home_logo

                    # ── Score / time ───────────────────────────────────
                    score_cell = cells[3] if len(cells) > 3 else cells[2]
                    score_text = score_cell.get_text(strip=True)
                    score_str  = None
                    time_str   = "TBD"

                    if is_past and re.match(r"\d+\s*-\s*\d+", score_text):
                        score_str = score_text.replace(" ", "")
                    elif not is_past:
                        time_str = score_text if score_text not in ("", "TBD") else "TBD"

                    # ── Competition ────────────────────────────────────
                    comp_raw = cells[5].get_text(strip=True) if len(cells) > 5 else "—"
                    abbr     = COMP_MAP.get(comp_raw, comp_raw[:2].upper() if comp_raw != "—" else "—")

                    # ── Date parse ─────────────────────────────────────
                    dt   = None
                    year = datetime.now(PT_TZ).year
                    for fmt in ("%a, %b %d %Y", "%A, %B %d %Y"):
                        try:
                            dt = datetime.strptime(f"{date_text} {year}", fmt)
                            dt = dt.replace(tzinfo=PT_TZ)
                            # Roll over year if needed
                            if dt < datetime.now(PT_TZ).replace(month=1, day=1):
                                dt = dt.replace(year=year + 1)
                            break
                        except ValueError:
                            continue

                    matches.append({
                        "home":          home,
                        "away":          away,
                        "home_logo":     home_logo,
                        "away_logo":     away_logo,
                        "opponent":      opponent,
                        "opponent_logo": opponent_logo,
                        "is_home":       is_home,
                        "competition":   comp_raw,
                        "abbr":          abbr,
                        "date":          dt.strftime("%d/%m/%y") if dt else date_text,
                        "time":          time_str,
                        "timestamp":     dt.timestamp() if dt else 0,
                        "score":         score_str,
                        "is_past":       is_past,
                    })

                except Exception as e:
                    log.debug("Skipping row: %s", e)
                    continue

        log.info("  %s: parsed %d matches", mode, sum(1 for m in matches if m["is_past"] == is_past))

    matches.sort(key=lambda x: x["timestamp"])
    log.info("Total: %d past + %d future", 
             sum(1 for m in matches if m["is_past"]),
             sum(1 for m in matches if not m["is_past"]))
    return matches


def build_output(matches: list[dict]) -> dict:
    past   = [m for m in matches if m["is_past"] and m.get("score")]
    future = [m for m in matches if not m["is_past"]]

    last2 = past[-2:]  if len(past)   >= 2 else past
    next2 = future[:2] if len(future) >= 2 else future

    empty = {
        "home": "—", "away": "—", "home_logo": None, "away_logo": None,
        "opponent": "—", "opponent_logo": None, "is_home": True,
        "competition": "—", "abbr": "—",
        "date": "—", "time": "—", "timestamp": 0,
        "score": None, "is_past": False,
    }
    while len(last2) < 2:
        last2.insert(0, dict(empty))
    while len(next2) < 2:
        next2.append(dict(empty))

    return {
        "updated_at": datetime.now(tz=PT_TZ).isoformat(),
        "team":       TEAM_NAME,
        "display":    last2 + next2,
        "all_past":   past,
        "all_future": future,
    }


def main():
    matches = scrape()

    if not matches:
        log.warning("No matches found — keeping existing data if present")
        if DATA_FILE.exists():
            return
        matches = []

    output = build_output(matches)
    DATA_FILE.write_text(json.dumps(output, indent=2, ensure_ascii=False))
    log.info("Wrote %d total matches to %s", len(matches), DATA_FILE)


if __name__ == "__main__":
    main()