"""Download every raw public dataset used by the study into RAW (outside the shared folder). All sources are open (Zenodo mirrors of Dryad deposits, Europe PMC open-access full text and supplementary files, DataverseNL, GitHub raw). Files that already exist are not re-downloaded. Run: python3 fetch_data.py (downloads what is missing) python3 fetch_data.py --force (re-downloads everything) NOT fetched by this script (provenance note): Vermeij (2006, PNAS 103:1804) Tables 1 and 2. The article is not in the Europe PMC open-access subset (the fullTextXML endpoint returns an error page) and the PMC HTML page (PMC1413639) refuses script user-agents, so the two tables were transcribed once from that page and are committed as data/vermeij2006_tables_as_published.csv (23 + 54 rows; the referee checked the transcription against the page row for row). That CSV is the only input that cannot be re-downloaded. """ import os import sys import time import zipfile import urllib.request RAW = os.environ.get("FERMI_RAW", "fermi_raw/convergence") UA = "fermi-convergence-study/1.0 (academic research script; python-urllib)" # (relative path under RAW, url, description) SOURCES = [ ("zenodo/burger2019_BrainAllometry_Supplement_Data.csv", "https://zenodo.org/api/records/4989800/files/BrainAllometry_Supplement_Data.csv/content", "Burger, George, Leadbetter & Shaikh 2019 J Mammal 100:276 (Dryad 10.5061/dryad.2r62k7s): brain and body mass, 1,552 extant mammal species"), ("zenodo/du2018_ECV_dataset.xlsx", "https://zenodo.org/api/records/4977822/files/ProcB%20SI%20ECV%20dataset%20FINAL.xlsx/content", "Du et al. 2018 Proc R Soc B 285:20172738 (Dryad 10.5061/dryad.c30g9): hominin endocranial volumes with dates"), ("zenodo/muller2019_CCExCet.xlsx", "https://zenodo.org/api/records/4068964/files/CCExCet_DRYAD.xlsx/content", "Muller & Montgomery 2019 J Evol Biol (Dryad 10.5061/dryad.rm4368f): cetacean brain and body masses"), ("zenodo/bertrand2022_Brain_Evolution_Placental.zip", "https://zenodo.org/api/records/5918811/files/Bertrand-Ornella/Brain_Evolution_Placental-v1.0.0.zip/content", "Bertrand et al. 2022 Science 376:80 (Zenodo 10.5281/zenodo.5918811): Paleogene and extant placental endocasts"), ("zenodo/sol2022_neurons.csv", "https://zenodo.org/api/records/6460346/files/Neurons%20data%20Feb%202022.csv/content", "Sol et al. 2022 Nat Ecol Evol 6:1381 (Zenodo 10.5281/zenodo.6460346): bird neuron numbers (111 spp.)"), ("trees/mamphy_mcc.tre", "https://raw.githubusercontent.com/n8upham/MamPhy_v1/master/_DATA/MamPhy_fullPosterior_BDvr_Completed_5911sp_topoCons_NDexp_MCC_v2_target.tre", "Upham, Esselstyn & Jetz 2019 PLoS Biol 17:e3000494: mammal MCC time tree (5,911 species); robustness only"), ("trees/mamphy_mcc_dnaonly.tre", "https://raw.githubusercontent.com/n8upham/MamPhy_v1/master/_DATA/MamPhy_fullPosterior_BDvr_DNAonly_4098sp_topoFree_NDexp_MCC_v2_target.tre", "Upham et al. 2019: DNA-only MCC tree (4,098 species); used only to check the cetacean anomaly of the completed tree"), ("trees/alvarez2022_4705sp_mean.nwk", "https://raw.githubusercontent.com/sabifo4/mammals_dating/main/02_SeqBayes_S2/03_Generate_final_mammal_tree/4705sp_mean.nwk", "Alvarez-Carretero et al. 2022 Nature 602:263: 4,705-taxon mammal timetree (mean ages; branch unit 100 Myr); primary tree"), ("kverkova/supp.zip", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC8931369/supplementaryFiles", "Kverkova et al. 2022 PNAS 119:e2121624119 supplementary files (Dataset S1: neuron numbers, birds and reptiles)"), ("epmc/PMC8478358.supp.zip", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC8478358/supplementaryFiles", "Waugh & Thewissen 2021 PLoS ONE 16:e0257803 supplementary files (S3 Table: fossil cetacean brain/body mass)"), ("epmc/PMC8478358.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC8478358/fullTextXML", "Waugh & Thewissen 2021 full text"), ("epmc/PMC12445405.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC12445405/fullTextXML", "Avelino-de-Souza et al. 2025 J Comp Neurol: cellular composition of the brain of a northern minke whale"), ("epmc/PMC5733047.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC5733047/fullTextXML", "Jardim-Messeder et al. 2017 Front Neuroanat 11:118: carnivoran neuron numbers"), ("epmc/PMC4228855.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC4228855/fullTextXML", "Kazu et al. 2014 Front Neuroanat 8:128: artiodactyl neuron numbers"), ("epmc/PMC4053853.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC4053853/fullTextXML", "Herculano-Houzel et al. 2014 Front Neuroanat 8:46: the elephant brain in numbers"), ("epmc/PMC4244864.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC4244864/fullTextXML", "Mortensen et al. 2014 Front Neuroanat 8:132: pilot whale neocortical neuron number (stereology)"), ("epmc/PMC6597534.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC6597534/fullTextXML", "Benoit et al. 2019 Sci Rep 9:9323: brain evolution in Proboscidea across the Cenozoic"), ("epmc/PMC11827626.xml", "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC11827626/fullTextXML", "Mills, Macalady, Frank & Wright 2025 Sci Adv 11:eads5698 (context only)"), ("massen/massen_brain_measures.xlsx", "https://dataverse.nl/api/access/datafile/26144", "Massen et al. 2021 Commun Biol 4:503 source data (DataverseNL 10.34894/ROFNL1): brain measures incl. cortical neuron numbers"), ] def fetch(url, dest, tries=3, timeout=900): os.makedirs(os.path.dirname(dest), exist_ok=True) last = None for k in range(tries): try: req = urllib.request.Request(url, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=timeout) as r, open(dest + ".part", "wb") as f: while True: chunk = r.read(1 << 16) if not chunk: break f.write(chunk) os.replace(dest + ".part", dest) return True except Exception as e: # noqa last = e time.sleep(3 * (k + 1)) print(f" !! failed: {url}: {last}") return False def unpack(): """Extract the few files needed from the downloaded archives.""" z = os.path.join(RAW, "kverkova/supp.zip") if os.path.exists(z): with zipfile.ZipFile(z) as zf: for n in zf.namelist(): if n.endswith("sd01.xlsx") or n.endswith("sd02.xlsx"): zf.extract(n, os.path.join(RAW, "kverkova")) z = os.path.join(RAW, "epmc/PMC8478358.supp.zip") if os.path.exists(z): os.makedirs(os.path.join(RAW, "epmc/waugh2021"), exist_ok=True) with zipfile.ZipFile(z) as zf: for n in zf.namelist(): if n.endswith(".docx"): zf.extract(n, os.path.join(RAW, "epmc/waugh2021")) z = os.path.join(RAW, "zenodo/bertrand2022_Brain_Evolution_Placental.zip") if os.path.exists(z): with zipfile.ZipFile(z) as zf: for n in zf.namelist(): if n.endswith("Placental_data_final.csv"): data = zf.read(n) with open(os.path.join(RAW, "zenodo/bertrand2022_Placental_data_final.csv"), "wb") as f: f.write(data) def main(force=False): ok = True for rel, url, desc in SOURCES: dest = os.path.join(RAW, rel) if os.path.exists(dest) and os.path.getsize(dest) > 1000 and not force: print(f"[have] {rel}") continue print(f"[get ] {rel} <- {desc}") ok = fetch(url, dest) and ok unpack() return ok if __name__ == "__main__": main(force="--force" in sys.argv)