"""Island-level source data for the revision: coordinates and land areas parsed from English Wikipedia infoboxes.

The hand-entered island table (curated_tables.PACIFIC_ISLANDS) names one Wikipedia article per island. This module
downloads the raw wikitext of each article (action=raw, one polite request every 1.5 s), parses the infobox coordinate
template and the area fields, and writes data/island_infobox_cache.json. run_all.py uses the cached JSON unless
--refresh-islands is given, so the analysis does not depend on the network. The land areas used in the analysis are the
hand-entered ones; the parsed values serve as a machine check (see check_areas)."""
import json
import os
import re
import time
import urllib.parse
import urllib.request

from common import DATA, RAW

CACHE = os.path.join(DATA, "island_infobox_cache.json")
RAWDIR = os.path.join(RAW, "wiki_raw")
SQMI = 2.589988


def _path(title):
    return os.path.join(RAWDIR, re.sub(r"[^A-Za-z0-9]+", "_", title) + ".wiki")


def fetch_page(title, force=False):
    p = _path(title)
    if force or not os.path.exists(p) or os.path.getsize(p) < 200:
        os.makedirs(RAWDIR, exist_ok=True)
        url = "https://en.wikipedia.org/w/index.php?title=" + urllib.parse.quote(title.replace(" ", "_")) + "&action=raw"
        req = urllib.request.Request(url, headers={"User-Agent": "expansion_analogs research script (python-urllib)"})
        with urllib.request.urlopen(req, timeout=60) as r:
            blob = r.read()
        with open(p, "wb") as f:
            f.write(blob)
        time.sleep(1.5)
    return open(p, encoding="utf-8").read()


def parse_coord(text):
    """First {{coord}} template of the article: degrees-minutes-seconds or decimal degrees."""
    for m in re.finditer(r"\{\{[Cc]oord\s*\|([^}]*)\}\}", text):
        parts = [x.strip() for x in m.group(1).split("|") if x.strip()]
        nums = []
        vals = []
        for x in parts:
            if re.fullmatch(r"-?\d+(\.\d+)?", x):
                nums.append(float(x))
            elif x in ("N", "S", "E", "W"):
                d = sum(v / 60 ** k for k, v in enumerate(nums))
                vals.append(-d if x in ("S", "W") else d)
                nums = []
            else:
                break
        if len(vals) == 2:
            return vals[0], vals[1]
        if len(vals) == 0 and len(nums) >= 2:
            return nums[0], nums[1]
    return None


def _num(x):
    m = re.search(r"-?\d[\d,]*\.?\d*", x)
    return float(m.group(0).replace(",", "")) if m else None


def parse_area(text):
    """Infobox area fields. Returns the raw fields and, where it can be read unambiguously, the land area in km2:
    an 'above water' figure in the footnote if present (atolls: the main figure is then the lagoon), else area_km2,
    else square miles converted."""
    head = text[:8000]
    f = {a.strip().replace(" ", "_"): b.strip() for a, b in
         re.findall(r"\|\s*(area[_a-z0-9 ]*)\s*=\s*([^\n]*)", head) if b.strip() and not b.strip().startswith("|")}
    land = None
    foot = f.get("area_footnotes", "")
    m = re.search(r"\{\{convert\|([\d.,]+)\|km2[^}]*\}\}\s*\(\[\[above sea level\|above water\]\]\)", foot)
    if m:
        land = float(m.group(1).replace(",", ""))
    elif "area_km2" in f and _num(f["area_km2"]) is not None and "lagoon" not in foot:
        land = _num(f["area_km2"])
    elif "area_total_km2" in f and _num(f["area_total_km2"]) is not None:
        land = _num(f["area_total_km2"])
    elif "area_sqmi" in f and _num(f["area_sqmi"]) is not None:
        land = round(_num(f["area_sqmi"]) * SQMI, 1)
    return {k: v[:160] for k, v in f.items()}, land


def build_cache(force=False, verbose=True):
    import curated_tables as ct
    out = {}
    for row in ct.PACIFIC_ISLANDS:
        title = row[2]
        text = fetch_page(title, force=force)
        c = parse_coord(text)
        fields, land = parse_area(text)
        out[title] = dict(island=row[0], lat=None if c is None else round(c[0], 5), lon=None if c is None else round(c[1], 5),
                          infobox_area_fields=fields, land_km2_parsed=land, retrieved="2026-10-12",
                          url="https://en.wikipedia.org/wiki/" + urllib.parse.quote(title.replace(" ", "_")))
        if verbose and c is None:
            print("      no coordinate parsed for", title)
    with open(CACHE, "w", encoding="utf-8") as f:
        json.dump(out, f, indent=1, ensure_ascii=False)
    return out


def load_cache():
    return json.load(open(CACHE, encoding="utf-8"))


def check_areas(tol=0.15):
    """Compare the hand-entered land areas with the parsed infobox values; return the rows that differ by more than tol."""
    import curated_tables as ct
    c = load_cache()
    bad = []
    for row in ct.PACIFIC_ISLANDS:
        p = c[row[2]]["land_km2_parsed"]
        if p is not None and abs(p - row[3]) > tol * max(p, row[3]):
            bad.append(dict(island=row[0], entered_km2=row[3], infobox_km2=p, area_source=row[4]))
    return bad


if __name__ == "__main__":
    build_cache()
    print(json.dumps(check_areas(), indent=1))