"""Write the two hand-curated innovation tables used in Part 1. (1) data/vermeij2006_recoded_2026.csv : Vermeij's (2006) 23 "unique" + 54 "repeated" innovations exactly as printed (data/vermeij2006_tables_as_published.csv, transcribed from the PMC full text), plus our 2026 re-coding (status and age) with the reason for every change. (2) data/innovations_2026.csv : an extended list of general (not clade-defined) innovations, with number of independent origins (min / best / max), age of first origin (best, min, max), sources and a verification flag. Verification flag: V = the key number was checked against a source by web search / primary full text in this session (October 2026); L = standard literature value quoted from memory with its citation, NOT re-verified in this session. Coding rules strict : 'singular' if n_best == 1 (independent origins of the innovation as narrowly, structurally defined), else 'repeated'. functional : 'singular' only if n_max == 1, i.e. no independent functional analogue and no contested second origin is known; else 'repeated'. class : C = repeated (n_best >= 2); B = singular under the strict definition but with a contested second origin or an independent functional analogue (n_best == 1, n_max >= 2); A = singular, no known second origin of any kind (n_max == 1). """ import csv import os import re HERE = os.path.dirname(os.path.abspath(__file__)) DATA = os.path.join(HERE, "..", "data") def parse_age(s): """Parse Vermeij's printed time into Ma. 'Before 3.5 Ga' -> 3500. Two entries in his Table 2 are printed as '3.5 Ma' (nitrogen fixation) and '2.7 Ma' (aerobic respiration); these are obvious unit typos for Ga (his text counts four repeated innovations older than 1 Ga, which requires both) and are read as Ga. '550' without a unit is Ma.""" t = s.replace("\n", " ") m = re.search(r"([\d.]+)\s*(Ga|Ma)?", t) v = float(m.group(1)) unit = m.group(2) if unit == "Ga": return v * 1000.0, "" if v < 10 and unit == "Ma" and v in (3.5, 2.7): return v * 1000.0, "unit typo in original (Ma printed, Ga meant)" return v, "" # ---- 2026 re-coding of Vermeij's rows: innovation -> (strict, functional, age_2026, reason) # rows not listed keep the published status and age. RECODE = { "Origin of Earth’s life": ("singular", "singular", 4000, "LUCA dated 4.2 Ga (4.09-4.33) by Moody et al. 2024 Nat Ecol Evol; oldest broadly accepted fossils 3.48 Ga [V, evidence brief]"), "Compartments (protocells)": ("singular", "singular", 4000, "age tied to LUCA [L]"), "Universal genetic code": ("singular", "singular", 4000, "age tied to LUCA [L]"), "Coordinated chromosomal replication": ("singular", "repeated", 4000, "core DNA-replication machinery of Bacteria is not homologous to that of Archaea/Eukarya (Leipe, Aravind & Koonin 1999 NAR 27:3389), i.e. two origins of the function [L]"), "Oxygenic photosynthesis": ("singular", "singular", 2700, "3.5 Ga stromatolite inference no longer accepted as evidence of oxygenic photosynthesis; GOE 2.4-2.3 Ga, geochemical/molecular evidence up to ~3.0-3.2 Ga [V: search; range 2400-3200]"), "Formation of eukaryote": ("singular", "singular", 1650, "2.7 Ga steranes were contamination (Rasmussen 2008; French 2015); total-group fossils 1.63-1.67 Ga, crown group <1.84 Ga (Betts 2018; Mills 2025) [V]"), "Primary symbiosis between cyanobacterium and eukaryote": ("repeated", "repeated", 1600, "second primary plastid in Paulinella, 90-140 Ma (Marin 2005; Nowack 2008; Delaye 2016); Archaeplastida ~1.6 Ga (Gabr 2020) [V]"), "Sexual populations": ("singular", "singular", 1650, "LECA was meiotic, so age >= LECA; Bangiomorpha re-dated to 1.047 Ga (Gibson 2018) [V]"), "Eumetazoan nervous system": ("singular", "repeated", 600, "ctenophores sister to other animals (Schultz 2023 Nature) implies neuron loss in sponges or independent origin in ctenophores (Moroz 2014; Burkhardt 2023 Science); contested [V]"), "Cnidarian nematocysts": ("singular", "repeated", 550, "ballistic nematocysts evolved independently in dinoflagellates (Gavelis et al. 2017 Sci Adv 3:e1602552) [L]"), "Land-plant seed": ("singular", "repeated", 370, "seed-like megasporangia (e.g. Lepidocarpon) in Carboniferous lycopsids; heterospory itself >= 11 times (Bateman & DiMichele 1994) [L]"), "Chelonian turtle construction": ("singular", "repeated", 225, "functionally equivalent closed bony carapaces in placodonts (Henodus, Cyamodus), glyptodonts [L]"), "Theropod feathers": ("singular", "repeated", 160, "filamentous insulation also in mammals (hair) and pterosaurs (pycnofibres; homology with feathers debated, Yang 2019; Cincotta 2022) [L]"), "Human language": ("singular", "singular", 1, "unchanged; age very uncertain (0.1-2 Ma)"), # Table 2 changes "Nitrogen fixation": ("singular", "singular", 3200, "all nitrogenases (Nif/Vnf/Anf) are homologous: a single origin spread by HGT (Boyd & Peters 2013); N-isotope evidence 3.2 Ga (Stueken 2015) [L]"), "Aerobic respiration": ("repeated", "repeated", 2700, "unit typo corrected; status unchanged"), "C4 photosynthesis by land plants": ("repeated", "repeated", 30, "62 lineages (Sage, Christin & Edwards 2011 JXB 62:3155); oldest ~30-35 Ma, not 300 Ma [V]"), "Animal eusociality": ("repeated", "repeated", 150, "termite/ant crown ages ~130-170 Ma [L]"), } # ---- co-inherited (stem-lineage) clusters among Vermeij's Precambrian "unique" rows (added in revision, # referee point M2/M3). Traits fixed in the stem of one clade are not independent trials: 4 rows are # properties of LUCA, 2 of LECA and 7 are animal body-plan characters dated 550-600 Ma. # cluster_major : LUCA / LECA / animal body plan (15 Precambrian unique rows -> 5 events) # cluster_stem : finer, by the stem lineage that carries the trait (15 rows -> 9 events) CLUSTERS = { "Origin of Earth’s life": ("LUCA", "LUCA"), "Compartments (protocells)": ("LUCA", "LUCA"), "Universal genetic code": ("LUCA", "LUCA"), "Coordinated chromosomal replication": ("LUCA", "LUCA"), "Formation of eukaryote": ("LECA", "LECA"), "Sexual populations": ("LECA", "LECA"), "Eumetazoan nervous system": ("animal body plan", "Eumetazoa stem"), "Eumetazoan extracellular digestion": ("animal body plan", "Eumetazoa stem"), "Bilaterian pattern formation": ("animal body plan", "Bilateria stem"), "Triplobastian three-layered construction": ("animal body plan", "Bilateria stem"), "Cnidarian nematocysts": ("animal body plan", ""), "Ecdysozoan molting": ("animal body plan", ""), "Echinoderm water-vascular system": ("animal body plan", ""), } def write_vermeij(): src = os.path.join(DATA, "vermeij2006_tables_as_published.csv") rows = list(csv.DictReader(open(src, encoding="utf8"))) out = [] for r in rows: age, note = parse_age(r["time_as_printed"]) name = r["innovation"] if name in RECODE: st, fn, a26, why = RECODE[name] else: st, fn, a26, why = r["status_published"], r["status_published"], age, "" out.append(dict(table=r["table"], innovation=name, status_published=r["status_published"], time_as_printed=r["time_as_printed"].replace("\n", " "), age_published_Ma=age, parse_note=note, status_2026_strict=st, status_2026_functional=fn, age_2026_Ma=a26, cluster_major=CLUSTERS.get(name, ("", ""))[0], cluster_stem=CLUSTERS.get(name, ("", ""))[1], recode_reason=why)) with open(os.path.join(DATA, "vermeij2006_recoded_2026.csv"), "w", newline="", encoding="utf8") as f: w = csv.DictWriter(f, fieldnames=list(out[0].keys())) w.writeheader() w.writerows(out) return out # ---- extended list ------------------------------------------------------------------------ # id, innovation, set, n_min, n_best, n_max, age_best, age_min, age_max, verified, sources, notes # set: 'hardstep' = commonly proposed hard step / major transition; 'comparison' = other innovation E = [ ("abiogenesis", "Origin of life (abiogenesis -> LUCA)", "hardstep", 1, 1, 1, 4000, 3480, 4400, "V", "Moody et al. 2024 Nat Ecol Evol (LUCA 4.2 Ga, 4.09-4.33); Djokic et al. 2017 (3.48 Ga); project evidence brief abiogenesis.md", "Single surviving origin; no second genesis known on Earth or in the laboratory. Competitive exclusion / 'pulling up the ladder' cannot be excluded."), ("oxyphot", "Oxygenic photosynthesis (water-splitting, two coupled photosystems)", "hardstep", 1, 1, 1, 2700, 2400, 3200, "V", "Great Oxidation Event ~2.4-2.3 Ga; geochemical proxies and PSI/PSII gene-family ages suggest >=3.0 Ga (Planavsky et al. 2014 Nat Geosci; Cardona et al. 2019); single origin in the cyanobacterial stem lineage", "All oxygenic phototrophs (cyanobacteria, plastids) share one origin."), ("eukaryogenesis", "Eukaryogenesis (FECA->LECA: nucleus, endomembranes, cytoskeleton, mitochondrion)", "hardstep", 1, 1, 1, 1650, 1050, 2400, "V", "Betts et al. 2018 Nat Ecol Evol (crown eukaryotes <1.84 Ga); Mills et al. 2025 Sci Adv (total-group fossils 1.63-1.67 Ga, crown-group fossils 1.03-1.06 Ga)", "Components recur in prokaryotes (Asgard actin cytoskeleton; endosymbionts inside bacteria; compartmentalised DNA in Thiomargarita magnifica, Volland 2022 Science) but the grade is known once."), ("mitochondrion", "Respiratory (energy-supplying) endosymbiotic organelle", "hardstep", 1, 1, 2, 1650, 1210, 2053, "V", "Betts et al. 2018 (mitochondrial symbiosis 2.053-1.21 Ga); Graf et al. 2021 Nature 591:445 ('Candidatus Azoamicus ciliaticola', an obligate ATP-supplying denitrifying endosymbiont of a ciliate)", "Mitochondria arose once; a second, independent ATP-supplying endosymbiont (inside an already-eukaryotic host) is known."), ("meiosis", "Meiotic sex", "hardstep", 1, 1, 1, 1650, 1047, 2400, "V", "LECA was sexual (Speijer et al. 2015 PNAS); Bangiomorpha 1.047 Ga (Gibson et al. 2018 Geology) is the oldest fossil evidence", "Bundled with eukaryogenesis (not an independent event). Parasexual analogues in Bacteria/Archaea (conjugation, haloarchaeal cell fusion) are not meiosis."), ("plastid", "Primary plastid endosymbiosis (cyanobacterium -> organelle)", "hardstep", 2, 2, 2, 1600, 1050, 2100, "V", "Archaeplastida ~1.6 Ga; Paulinella chromatophore 90-140 Ma (Marin et al. 2005; Nowack et al. 2008; Delaye et al. 2016; Gabr, Grossman & Bhattacharya 2020 J Phycol)", "Classic 'singular' event until the Paulinella chromatophore was recognised as an independent, much younger primary plastid."), ("complex_multicell", "Complex multicellularity (3-D tissues, internal transport)", "hardstep", 5, 6, 8, 635, 575, 1050, "V", "Knoll 2011 Annu Rev Earth Planet Sci 39:217 (animals, embryophytes, florideophyte red algae, brown algae, and 2-3 fungal clades); Nagy et al. 2018", "Animal-grade (phagotrophic, motile, with neurons and muscle) multicellularity is known once; see separate row."), ("animal_multicell", "Animal-grade multicellularity (Metazoa)", "hardstep", 1, 1, 1, 635, 575, 800, "L", "dos Reis et al. 2015 Curr Biol (crown Metazoa 650-833 Ma); Ediacaran macrofossils 575 Ma", "Clade-defined; treated as singular by e.g. Lingam & Loeb, but the functional class (complex multicellularity) is repeated."), ("neurons", "Neurons / nervous systems", "hardstep", 1, 1, 2, 600, 560, 700, "V", "Moroz et al. 2014 Nature 510:109; Schultz et al. 2023 Nature 618:110 (ctenophores sister to all other animals); Burkhardt et al. 2023 Science 380:293", "Ctenophore-sister topology implies either neuron loss in sponges+placozoans or two origins of neurons; unresolved."), ("land", "Colonisation of land by macroscopic organisms", "hardstep", 5, 12, 25, 470, 425, 515, "L", "Embryophytes (cryptospores ~470 Ma; molecular 515 Ma, Morris et al. 2018 PNAS); arthropods (myriapods, arachnids, hexapods, isopods, amphipods, crabs), tetrapods, gastropods (many times), annelids, onychophorans (Little 1990; Vermeij & Dudley 2000)", "Many independent invasions by animals; embryophyte-grade land plants once."), ("hominin_brain", "Human-level brain 'hardware' (>=10^10 cortical neurons, EQ>~6)", "hardstep", 1, 1, 1, 0.5, 0.2, 2.0, "V", "Azevedo et al. 2009 J Comp Neurol (16.3e9 cortical neurons); Du et al. 2018 Proc R Soc B (hominin ECV series); Part 2 of this study", "Too young for its singularity to be informative (see recurrence-rate analysis)."), ("language", "Open-ended syntactic language", "hardstep", 1, 1, 1, 0.3, 0.1, 2.0, "L", "Vermeij 2006 lists human language at 1 Ma; date of origin unconstrained by fossils", "Vocal learning (a component) evolved >=8 times."), ("cum_culture", "Cumulative technological culture", "hardstep", 1, 1, 1, 2.6, 1.76, 3.3, "L", "Oldowan 2.6 Ma, Lomekwian 3.3 Ma (Harmand et al. 2015 Nature); experimental 'beyond-individual' social learning in bumblebees (Bridges et al. 2024 Nature) and chimpanzees (van Leeuwen et al. 2024 Nat Hum Behav) [V for the 2024 studies]", "Tool cultures exist in chimpanzees, capuchins, macaques, New Caledonian crows; open-ended ratcheting accumulation only in Homo."), # -------- comparison innovations ("simple_multicell", "Simple multicellularity", "comparison", 25, 25, 50, 2500, 1900, 3500, "V", "Grosberg & Strathmann 2007 Annu Rev Ecol Evol Syst 38:621 ('at least 25' origins); filamentous cyanobacteria by >=2.0-2.5 Ga (Schirrmeister et al. 2013 PNAS)", ""), ("eyes", "Image-forming eyes", "comparison", 4, 7, 65, 520, 515, 540, "L", "Salvini-Plawen & Mayr 1977 (photoreceptor organs 40-65 times); Land & Nilsson 2012 Animal Eyes; Vermeij 2006 (520 Ma)", ""), ("skeleton", "Biomineralised skeletons", "comparison", 8, 20, 30, 550, 548, 810, "L", "Knoll 2003 Rev Mineral Geochem 54:329; Cloudina ~550 Ma; apatitic scale microfossils ~810 Ma (Cohen et al. 2017 Sci Adv)", ""), ("c4", "C4 photosynthesis", "comparison", 61, 62, 70, 30, 25, 35, "V", "Sage, Christin & Edwards 2011 J Exp Bot 62:3155 (62 lineages); oldest origins ~30-35 Ma (Sage 2016)", ""), ("flight", "Powered flight", "comparison", 4, 4, 4, 325, 320, 400, "L", "insects (>=325 Ma fossils), pterosaurs (~228 Ma), birds (~150 Ma), bats (~52 Ma)", ""), ("endothermy", "Whole-body endothermy", "comparison", 2, 2, 4, 250, 200, 270, "L", "synapsid and archosaur lineages (Benton 2021 Gondwana Res; Grigg et al. 2022 Biol Rev); regional endothermy in tunas, lamnid sharks, billfishes, opah raises the functional count to ~8", "n_max counts whole-body origins only if pterosaurs and some marine reptiles were independent."), ("eusociality", "Eusociality", "comparison", 11, 17, 24, 150, 125, 170, "V", "Woodard et al. 2011 PNAS (>=11 origins in insects); termites, ants, bees and wasps (several), aphids, thrips, ambrosia beetle, Synalpheus shrimps, two mole-rat genera", ""), ("agriculture", "Agriculture (cultivation of a food organism)", "comparison", 9, 20, 25, 66, 50, 120, "V", "Mueller et al. 2005 Annu Rev Ecol Evol Syst 36:563 (ants 1, termites 1, ambrosia beetles >=7); Schultz et al. 2024 Science (ant agriculture ~66 Ma); humans >=7 independent centres (Holocene)", "Age of the oldest origin (ambrosia beetles vs ants) uncertain, hence the wide range. n_best counts human centres of domestication as separate origins (one species); n_min = 9 does not."), ("tool_use", "Tool use", "comparison", 10, 20, 40, None, None, None, "L", "Shumaker, Walkup & Beck 2011 Animal Tool Behavior; Bentley-Condit & Smith 2010 Behaviour 147:185", "Behavioural trait with no datable first origin; excluded from age tests."), ("echolocation", "Echolocation", "comparison", 5, 7, 9, 52, 48, 65, "V", "bats, odontocetes, shrews, tenrecs, Typhlomys (He et al. 2021 Science 372:eaay1513: five mammal lineages); oilbird and swiftlets", ""), ("electric_organ", "Electric organs", "comparison", 6, 6, 8, 100, 50, 150, "V", "Gallant et al. 2014 Science 344:1522 (six origins in fishes); age of the oldest origin poorly constrained [age L]", ""), ("vascular", "Vascular (long-distance conducting) tissue", "comparison", 1, 1, 3, 432, 425, 450, "L", "tracheophyte xylem/phloem once (Cooksonia ~432 Ma); analogous conducting cells in mosses (hydroids, leptoids) and kelps (trumpet hyphae); Vermeij 2006 lists it as repeated", ""), ("viviparity", "Viviparity (vertebrates)", "comparison", 115, 150, 160, 380, 375, 385, "V", "Blackburn 2015 J Morphol 276:961 (>150 origins, >=115 in squamates); oldest: placoderm Materpiscis ~380 Ma (Long et al. 2008 Nature) [age L]", ""), ("nitroplast", "Nitrogen-fixing endosymbiotic organelle", "comparison", 1, 2, 3, 100, 90, 140, "V", "Coale et al. 2024 Science 384:217 (UCYN-A 'nitroplast' in Braarudosphaera bigelowii, ~100 Ma); diazoplast (spheroid body) of rhopalodiacean diatoms (Nakayama et al. 2014 PNAS) [L]; 'Ca. Tectiglobus diatomicola' (Tschitschko et al. 2024 Nature) is an early-stage analogue [V]", "Organelle status (host-protein import) demonstrated only for UCYN-A; n_best=2 counts the vertically inherited diatom diazoplast."), ("secondary_plastid", "Secondary / tertiary plastid endosymbiosis", "comparison", 5, 7, 10, 700, 550, 1300, "L", "Keeling 2013 Annu Rev Plant Biol 64:583; Vermeij 2006 (550 Ma)", ""), ("venom", "Venom delivery", "comparison", 30, 100, 110, 540, 520, 600, "L", "Schendel et al. 2019 Toxins 11:666 (>100 origins); Vermeij 2006 (540 Ma)", ""), ("bioluminescence", "Bioluminescence", "comparison", 94, 94, 100, 540, 267, 540, "V", "Lau & Oakley 2021 Biol Rev 96:673 (>=94 origins); DeLeo et al. 2024 Proc R Soc B (octocorals ~540 Ma)", ""), ("vocal_learning", "Vocal production learning", "comparison", 7, 8, 9, 55, 30, 65, "V", "songbirds, parrots, hummingbirds, bats, cetaceans, pinnipeds, elephants, humans (Vernes et al. 2021 Phil Trans B 376:20200236) [count V; age L]", ""), ("big_brain", "Large neuron-rich forebrain (>=1.6e9 pallial neurons, i.e. within 10x of human)", "comparison", 4, 6, 13, 35, 15, 45, "V", "This study, Part 2 (catarrhine primates, proboscideans, cetaceans, giraffids, macaws, owls; isotropic-fractionator counts and scaling rules)", "Self-derived row (from this study's own Part 2, not an independent source). Six deep lineages is the count used; species-level parsimony on rule-predicted counts gives a larger, noise-inflated range. Age: first lineage to cross is uncertain (cetaceans or anthropoids, late Eocene-Miocene)."), ("complex_brain", "Centralised complex brains", "comparison", 3, 4, 5, 520, 518, 540, "L", "vertebrates, cephalopods, arthropods (Northcutt 2012 PNAS 109:10626); Cambrian arthropod brains ~520 Ma (Ma et al. 2012 Nature)", ""), ("c_fixation", "Autotrophic CO2 fixation pathways", "comparison", 6, 7, 7, 3800, 3500, 4200, "L", "Fuchs 2011 Annu Rev Microbiol 65:631 (six pathways) plus the reductive glycine pathway (Sanchez-Andrea et al. 2020)", ""), ("phototrophy", "Phototrophy (light-driven energy conservation)", "comparison", 2, 2, 2, 3400, 3000, 3800, "V", "chlorophyll- and retinal-based phototrophy are independent (Burnetti & Ratcliff 2020; Mills et al. 2025)", ""), ("n_fixation", "Biological nitrogen fixation (nitrogenase)", "comparison", 1, 1, 1, 3200, 1500, 3500, "L", "Boyd & Peters 2013 Front Microbiol 4:201 (single origin, HGT); Stueken et al. 2015 Nature 520:666 (3.2 Ga)", "Not a classic hard step, but a biochemical singularity; Vermeij 2006 listed it as repeated."), ("aerobic_resp", "Aerobic respiration (O2-reducing terminal oxidases)", "comparison", 2, 3, 3, 2700, 2400, 3100, "L", "heme-copper oxidases, cytochrome bd oxidases and alternative oxidase are non-homologous (Borisov et al. 2011 BBA 1807:1398)", ""), ("dna_replication", "Chromosomal DNA replication machinery", "comparison", 2, 2, 2, 4000, 3500, 4300, "L", "bacterial and archaeal/eukaryotic replicative polymerases, primases and helicases are non-homologous (Leipe, Aravind & Koonin 1999 NAR 27:3389)", ""), ("fire", "Controlled use of fire", "comparison", 1, 1, 1, 1.0, 0.4, 1.6, "L", "Wonderwerk Cave ~1.0 Ma (Berna et al. 2012 PNAS); Gesher Benot Ya'aqov 0.79 Ma", "Requires a terrestrial, manipulative animal; young."), ] def write_extended(): cols = ["id", "innovation", "set", "n_min", "n_best", "n_max", "age_best_Ma", "age_min_Ma", "age_max_Ma", "verified", "sources", "notes"] out = [] for e in E: d = dict(zip(cols, e)) d["coding_strict"] = "singular" if d["n_best"] == 1 else "repeated" d["coding_functional"] = "singular" if d["n_max"] == 1 else "repeated" d["class"] = "C" if d["n_best"] >= 2 else ("A" if d["n_max"] == 1 else "B") # the four singular rows younger than 3 Ma are properties of one lineage (Homo), i.e. one event d["lineage_cluster"] = "Homo" if d["id"] in ("hominin_brain", "language", "cum_culture", "fire") else "" out.append(d) with open(os.path.join(DATA, "innovations_2026.csv"), "w", newline="", encoding="utf8") as f: w = csv.DictWriter(f, fieldnames=cols[:9] + ["coding_strict", "coding_functional", "class", "lineage_cluster"] + cols[9:]) w.writeheader() w.writerows(out) return out def main(): v = write_vermeij() e = write_extended() print(f" wrote {len(v)} Vermeij rows and {len(e)} extended rows") if __name__ == "__main__": main()