"""
Step 1b. A crude, reproducible proxy for how much work each class of method has received: the number of PubMed records
whose title or abstract matches a query. A record count is not a sample count; it is used only to compare orders of
magnitude between sequence-based methods and methods that do not presuppose nucleic acids.

Output: ../data/pubmed_method_counts.csv   (cached under RAW/pubmed/; --refresh to re-query)
The last five rows (method_class "union") are OR-queries over a whole class, so that overlapping records are counted once.
"""
import csv
import datetime
import json
import os
import sys
import time
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(HERE, "..", "data")
RAW = os.environ.get("SOS_RAW", "fermi_raw/second_origin_search")
REFRESH = "--refresh" in sys.argv
EU = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmax=0&retmode=json&term="

# (class, label, sees life without standard nucleic acids?, query)
QUERIES = [
    ("sequence-based", "16S rRNA gene surveys", "no", '"16S rRNA"[tiab] AND (microbiome[tiab] OR microbiota[tiab] OR "microbial community"[tiab] OR "bacterial community"[tiab])'),
    ("sequence-based", "metagenomics", "no", "metagenom*[tiab]"),
    ("sequence-based", "metagenome-assembled genomes", "no", '"metagenome-assembled genome"[tiab] OR "metagenome-assembled genomes"[tiab]'),
    ("sequence-based", "metatranscriptomics", "no", "metatranscriptom*[tiab]"),
    ("database-dependent", "metaproteomics (peptides matched to predicted proteins)", "only as unmatched spectra", "metaproteom*[tiab]"),
    ("database-dependent", "untargeted metabolomics of microbial or environmental samples", "only as unannotated features",
     '"untargeted metabolomics"[tiab] AND (microbi*[tiab] OR soil[tiab] OR seawater[tiab] OR sediment[tiab] OR environmental[tiab])'),
    ("activity, label follows standard biochemistry", "BONCAT (amino-acid analogue incorporation)", "no", 'BONCAT[tiab] OR "bioorthogonal non-canonical amino acid tagging"[tiab] OR "bioorthogonal noncanonical amino acid tagging"[tiab]'),
    ("activity, agnostic to the genetic polymer", "stable isotope probing", "partly (nucleic-acid SIP: no; lipid, protein, NanoSIMS and Raman SIP: yes)", '"stable isotope probing"[tiab]'),
    ("activity, agnostic to the genetic polymer", "NanoSIMS on microbial samples", "yes", "nanosims[tiab] AND (microb*[tiab] OR bacteri*[tiab] OR archaea*[tiab] OR cell[tiab] OR cells[tiab])"),
    ("activity, agnostic to the genetic polymer", "Raman single-cell with heavy water", "yes", 'raman[tiab] AND ("single cell"[tiab] OR "single-cell"[tiab]) AND (deuterium[tiab] OR "heavy water"[tiab] OR D2O[tiab])'),
    ("activity, agnostic to the genetic polymer", "calorimetry of soils and sediments", "yes", "(microcalorimetry[tiab] OR calorimetry[tiab] OR calorespirometry[tiab]) AND (soil[tiab] OR sediment[tiab]) AND microb*[tiab]"),
    ("chirality", "D-amino acids in soils, sediments, sea water", "yes (mirror proteins)", '("D-amino acids"[tiab] OR "D-amino acid"[tiab]) AND (soil[tiab] OR sediment[tiab] OR seawater[tiab] OR "dissolved organic matter"[tiab])'),
    ("chirality", "L-glucose or L-sugar use by microbes", "yes (mirror metabolism)", '("L-glucose"[tiab] OR "L-sugars"[tiab]) AND (bacteri*[tiab] OR microb*[tiab] OR soil[tiab])'),
    ("imaging", "cryo-electron tomography of environmental or uncultivated cells", "yes", '("cryo-electron tomography"[tiab] OR "electron cryotomography"[tiab]) AND (environmental[tiab] OR uncultivated[tiab] OR uncultured[tiab])'),
    ("imaging", "fluorescence in situ hybridisation of environmental microbes", "only as unlabelled cells", '("fluorescence in situ hybridization"[tiab] OR "CARD-FISH"[tiab]) AND (bacterioplankton[tiab] OR sediment[tiab] OR soil[tiab] OR seawater[tiab] OR "microbial community"[tiab])'),
    ("explicit search", "shadow biosphere", "designed to", '"shadow biosphere"[tiab]'),
    ("explicit search", "mirror life / mirror bacteria", "designed to", '"mirror life"[tiab] OR "mirror bacteria"[tiab] OR "mirror-image life"[tiab]'),
    ("explicit search", "agnostic biosignatures", "designed to", '"agnostic biosignature"[tiab] OR "agnostic biosignatures"[tiab]'),
]


def _or(idx):
    return " OR ".join("(" + QUERIES[i][3] + ")" for i in idx)


# Union queries (added in revision): the per-method counts overlap (a paper can match "16S rRNA" and "metagenom*"), so class totals
# must come from one OR query, not from a sum. The stable-isotope-probing class is mostly DNA- or RNA-based, which is not agnostic to
# the genetic polymer; SIP_NA counts those records and UNION_AGNOSTIC_STRICT removes them. These rows have method_class "union".
N_METHODS = len(QUERIES)
SEQ_IDX = [i for i, q in enumerate(QUERIES) if q[0] == "sequence-based"]
AGN_IDX = [i for i, q in enumerate(QUERIES) if q[0] in ("activity, agnostic to the genetic polymer", "chirality") or (q[0] == "imaging" and "hybridisation" not in q[1])]
EXPL_IDX = [i for i, q in enumerate(QUERIES) if q[0] == "explicit search"]
SIP_NA = '"stable isotope probing"[tiab] AND (DNA[tiab] OR RNA[tiab] OR rRNA[tiab] OR sequenc*[tiab] OR metagenom*[tiab] OR amplicon*[tiab])'
QUERIES += [
    ("union", "UNION: sequence-based (4 queries)", "no", _or(SEQ_IDX)),
    ("union", "UNION: polymer-agnostic (7 queries)", "yes or partly", _or(AGN_IDX)),
    ("union", "stable isotope probing that names DNA, RNA or sequencing", "no", SIP_NA),
    ("union", "UNION: polymer-agnostic, excluding nucleic-acid stable isotope probing", "yes", "(" + _or(AGN_IDX) + ") NOT (" + SIP_NA + ")"),
    ("union", "UNION: explicit searches (3 queries)", "designed to", _or(EXPL_IDX)),
]


def main():
    os.makedirs(os.path.join(RAW, "pubmed"), exist_ok=True)
    rows = []
    for i, (cls, label, sees, q) in enumerate(QUERIES):
        path = os.path.join(RAW, "pubmed", f"q{i:02d}.json")
        if os.path.exists(path) and not REFRESH:
            j = json.load(open(path))
        else:
            for k in range(4):
                try:
                    j = json.loads(urllib.request.urlopen(EU + urllib.parse.quote(q), timeout=60).read())
                    break
                except Exception as e:
                    err = e
                    time.sleep(2 + 3 * k)
            else:
                raise RuntimeError(f"PubMed query failed: {q}: {err}")
            j["_query"] = q
            json.dump(j, open(path, "w"))
            time.sleep(0.5)
        if j.get("_query", q) != q:
            raise RuntimeError("cached query differs from the script; rerun with --refresh")
        n = int(j["esearchresult"]["count"])
        rows.append(dict(method_class=cls, method=label, sees_non_nucleic_acid_life=sees, pubmed_records=n, query=q, accessed=datetime.date.today().isoformat()))
        print(f"{n:8d}  {label}")
    with open(os.path.join(DATA, "pubmed_method_counts.csv"), "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0])); w.writeheader(); w.writerows(rows)


if __name__ == "__main__":
    main()