#!/usr/bin/env python3 """ FC Porto fixture scraper — ESPN Server-side rendered, no JS needed, no API key required. Writes /data/porto.json """ import json import logging import os import re from datetime import datetime from pathlib import Path from zoneinfo import ZoneInfo import httpx from bs4 import BeautifulSoup logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") log = logging.getLogger(__name__) DATA_FILE = Path("/data/porto.json") DATA_FILE.parent.mkdir(parents=True, exist_ok=True) TEAM_ID = os.environ.get("ESPN_TEAM_ID", "437") TEAM_SLUG = os.environ.get("ESPN_TEAM_SLUG", "por.porto") TEAM_NAME = os.environ.get("TEAM_NAME", "FC Porto") PT_TZ = ZoneInfo("Europe/Lisbon") COMP_MAP = { "Portuguese Primeira Liga": "LP", "UEFA Europa League": "UE", "UEFA Champions League": "CL", "Taca de Portugal": "TP", "Taça de Portugal": "TP", "FIFA Club World Cup": "CW", "UEFA Super Cup": "SC", } HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "Chrome/124.0 Safari/537.36", "Accept-Language": "en-GB,en;q=0.9", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", } def scrape() -> list[dict]: log.info("Scraping ESPN for %s (id=%s)...", TEAM_NAME, TEAM_ID) base = "https://www.espn.com/soccer/team" urls = { "future": f"{base}/fixtures/_/id/{TEAM_ID}/{TEAM_SLUG}", "past": f"{base}/results/_/id/{TEAM_ID}/{TEAM_SLUG}", } porto_names = {"fc porto", "porto"} matches = [] for mode, url in urls.items(): is_past = mode == "past" try: r = httpx.get(url, headers=HEADERS, timeout=30, follow_redirects=True) r.raise_for_status() soup = BeautifulSoup(r.text, "lxml") except Exception as e: log.error("Failed to fetch %s: %s", url, e) continue for table in soup.find_all("table"): for row in table.find_all("tr"): cells = row.find_all("td") if len(cells) < 4: continue try: # ── Date ────────────────────────────────────────── date_text = cells[0].get_text(strip=True) # ── Teams + logos ────────────────────────────────── match_cell = cells[1] team_links = match_cell.find_all( "a", href=re.compile(r"/soccer/team/_/id/\d+") ) if len(team_links) < 2: continue home = team_links[0].get_text(strip=True) away = team_links[-1].get_text(strip=True) home_id = re.search(r"/id/(\d+)/", team_links[0]["href"]) away_id = re.search(r"/id/(\d+)/", team_links[-1]["href"]) home_logo = ( f"https://a.espncdn.com/i/teamlogos/soccer/500/{home_id.group(1)}.png" if home_id else None ) away_logo = ( f"https://a.espncdn.com/i/teamlogos/soccer/500/{away_id.group(1)}.png" if away_id else None ) is_home = home.lower() in porto_names opponent = away if is_home else home opponent_logo = away_logo if is_home else home_logo # ── Score / time ─────────────────────────────────── score_cell = cells[3] if len(cells) > 3 else cells[2] score_text = score_cell.get_text(strip=True) score_str = None time_str = "TBD" if is_past and re.match(r"\d+\s*-\s*\d+", score_text): score_str = score_text.replace(" ", "") elif not is_past: time_str = score_text if score_text not in ("", "TBD") else "TBD" # ── Competition ──────────────────────────────────── comp_raw = cells[5].get_text(strip=True) if len(cells) > 5 else "—" abbr = COMP_MAP.get(comp_raw, comp_raw[:2].upper() if comp_raw != "—" else "—") # ── Date parse ───────────────────────────────────── dt = None year = datetime.now(PT_TZ).year for fmt in ("%a, %b %d %Y", "%A, %B %d %Y"): try: dt = datetime.strptime(f"{date_text} {year}", fmt) dt = dt.replace(tzinfo=PT_TZ) # Roll over year if needed if dt < datetime.now(PT_TZ).replace(month=1, day=1): dt = dt.replace(year=year + 1) break except ValueError: continue matches.append({ "home": home, "away": away, "home_logo": home_logo, "away_logo": away_logo, "opponent": opponent, "opponent_logo": opponent_logo, "is_home": is_home, "competition": comp_raw, "abbr": abbr, "date": dt.strftime("%d/%m/%y") if dt else date_text, "time": time_str, "timestamp": dt.timestamp() if dt else 0, "score": score_str, "is_past": is_past, }) except Exception as e: log.debug("Skipping row: %s", e) continue log.info(" %s: parsed %d matches", mode, sum(1 for m in matches if m["is_past"] == is_past)) matches.sort(key=lambda x: x["timestamp"]) log.info("Total: %d past + %d future", sum(1 for m in matches if m["is_past"]), sum(1 for m in matches if not m["is_past"])) return matches def build_output(matches: list[dict]) -> dict: past = [m for m in matches if m["is_past"] and m.get("score")] future = [m for m in matches if not m["is_past"]] last2 = past[-2:] if len(past) >= 2 else past next2 = future[:2] if len(future) >= 2 else future empty = { "home": "—", "away": "—", "home_logo": None, "away_logo": None, "opponent": "—", "opponent_logo": None, "is_home": True, "competition": "—", "abbr": "—", "date": "—", "time": "—", "timestamp": 0, "score": None, "is_past": False, } while len(last2) < 2: last2.insert(0, dict(empty)) while len(next2) < 2: next2.append(dict(empty)) return { "updated_at": datetime.now(tz=PT_TZ).isoformat(), "team": TEAM_NAME, "display": last2 + next2, "all_past": past, "all_future": future, } def main(): matches = scrape() if not matches: log.warning("No matches found — keeping existing data if present") if DATA_FILE.exists(): return matches = [] output = build_output(matches) DATA_FILE.write_text(json.dumps(output, indent=2, ensure_ascii=False)) log.info("Wrote %d total matches to %s", len(matches), DATA_FILE) if __name__ == "__main__": main()