"""Download every raw input of the gate_lag study into RAW (default fermi_raw/gate_lag).

Nothing here is hand-edited: each file is fetched from its public source and cached. Re-running skips
files that already exist (delete RAW to force a fresh download).

Sources
  * GEOCARBSULFOR Monte Carlo outputs (Mills, Krause, Jarvis & Cramer 2023, Annu. Rev. Earth Planet. Sci.),
    file GCBS4_outputs_14_7_2022.zip in github.com/Alexjkrause/GEOCARBSULFOR (1000 realisations, 550-0 Ma).
  * COPSE outputs (Tostevin & Mills 2020 baseline and the feedback variants of Vitali et al.), Zenodo 17013752.
  * pySCION archive (Merdith et al. 2025), Zenodo 14695250: used only for the data files it bundles, namely
    the charcoal-based O2 envelope, the PhanDA temperature percentiles (Judd et al. 2024) and the
    Scotese et al. 2021 global average temperature curve.
  * DM-SED 0.0.1 (Deep-Time Marine Sedimentary Element Database, built on the SGP phase-1 product),
    Zenodo 14771859.  Inspected during the study (its Precambrian coverage is too thin to date a gate
    independently) and NOT used in any reported number; fetch_dmsed() is kept for completeness only.
  * Paleobiology Database API (data1.2): occurrences of the exemplar taxa of each innovation, genus diversity
    per stage for Animalia and Plantae, and first appearances of all orders and classes.
"""
import io
import os
import sys
import time
import urllib.parse
import urllib.request
import zipfile

from config import RAW

UA = {"User-Agent": "gate_lag-study/1.0 (research; python-urllib)"}


def _get(url, dest, timeout=600, tries=4):
    if os.path.exists(dest) and os.path.getsize(dest) > 0:
        return dest
    os.makedirs(os.path.dirname(dest), exist_ok=True)
    last = None
    for k in range(tries):
        try:
            req = urllib.request.Request(url, headers=UA)
            with urllib.request.urlopen(req, timeout=timeout) as r:
                blob = r.read()
            with open(dest + ".part", "wb") as f:
                f.write(blob)
            os.replace(dest + ".part", dest)
            print("  fetched", os.path.relpath(dest, RAW), len(blob), "bytes")
            return dest
        except Exception as e:  # noqa: BLE001
            last = e
            time.sleep(3 + 4 * k)
    raise RuntimeError("download failed: %s (%s)" % (url, last))


def fetch_curves():
    g = _get("https://raw.githubusercontent.com/Alexjkrause/GEOCARBSULFOR/master/GCBS4_outputs_14_7_2022.zip",
             os.path.join(RAW, "geocarbsulfor", "GCBS4_outputs_14_7_2022.zip"))
    out = os.path.join(RAW, "geocarbsulfor", "x")
    if not os.path.exists(os.path.join(out, "outputs_14_7_2022.mat")):
        zipfile.ZipFile(g).extractall(out)

    c = _get("https://zenodo.org/api/records/17013752/files/Rayannevitali/"
             "COPSE-oxygen-photorespiration-fire-v1.0-paper.zip/content",
             os.path.join(RAW, "copse", "copse.zip"))
    out = os.path.join(RAW, "copse", "x")
    if not os.path.isdir(out):
        z = zipfile.ZipFile(c)
        for n in z.namelist():
            if "/output/" in n and n.endswith(".txt"):
                z.extract(n, out)

    p = _get("https://zenodo.org/api/records/14695250/files/pySCION.zip/content",
             os.path.join(RAW, "pyscion", "pySCION.zip"))
    out = os.path.join(RAW, "pyscion", "x")
    need = ["pySCION/data/geochem_data_2024.mat", "pySCION/data/Scotese_GAT_2021.mat",
            "pySCION/data/PhanDA_GMSTandCO2_percentiles.csv"]
    if not all(os.path.exists(os.path.join(out, n)) for n in need):
        z = zipfile.ZipFile(p)
        for n in need:
            z.extract(n, out)


def fetch_dmsed():
    _get("https://zenodo.org/api/records/14771859/files/DM-SED%200.0.1.csv/content",
         os.path.join(RAW, "dmsed", "DM-SED_0.0.1.csv"))


PBDB = "https://paleobiodb.org/data1.2/"


def pbdb_occs(base_name, max_ma, min_ma, dest):
    q = dict(base_name=base_name, max_ma=max_ma, min_ma=min_ma, show="strat,class,ident", limit="all",
             timerule="overlap")
    return _get(PBDB + "occs/list.csv?" + urllib.parse.urlencode(q), dest, timeout=300)


def fetch_pbdb():
    from innovations import INNOVATIONS
    d = os.path.join(RAW, "pbdb")
    _get(PBDB + "occs/diversity.csv?base_name=Animalia&count=genera&time_reso=stage",
         os.path.join(d, "div_animalia_stage.csv"))
    _get(PBDB + "occs/diversity.csv?base_name=Plantae&count=genera&time_reso=stage",
         os.path.join(d, "div_plantae_stage.csv"))
    _get(PBDB + "taxa/list.csv?base_name=Life&rank=order,class&show=app,class&status=accepted",
         os.path.join(d, "orders_app.csv"))
    for inn in INNOVATIONS:
        if not inn.get("pbdb"):
            continue
        hi, lo = inn["pbdb_window"]
        pbdb_occs(",".join(inn["pbdb"]), hi, lo, os.path.join(d, "occs_%s.csv" % inn["id"]))
        time.sleep(0.5)


def main():
    print("raw directory:", RAW)
    fetch_curves()
    fetch_pbdb()        # fetch_dmsed() is available but DM-SED is not used in any reported result
    print("all raw inputs present")


if __name__ == "__main__":
    sys.exit(main())