#!/usr/bin/env python3
"""Single entry point: regenerates every number, table and figure of the dim_stars study.

    python3 run_all.py                      # download (if missing) + full analysis
    python3 run_all.py --no-download        # skip stage 1 (raw files already in $DIM_STARS_RAW)
    python3 run_all.py --from analysis      # restart at a given stage (name without .py)
    python3 run_all.py --from analysis:orig # restart at the second (original-cut) pass

Stages
   1 download_data.py          raw catalogues -> $DIM_STARS_RAW (default fermi_raw/dim_stars)
   2 build_samples.py          one row per star per survey (+ Gaia random parent sample)
  -- pass 1, final astrometric cuts (RUWE < 1.2, parallax/error >= 50) -> data/
   3 analysis.py               cuts, calibration of M(Teff, logg, [Fe/H]), deficits D, tails, recovery
   4 injection.py              end-to-end injection: correction factor C to the recovery
   5 selection.py              modelled probability that a dimmed star is in each clean sample (eps),
                               and the model-free "twin" cross-check of it
   6 vet_candidates.py         second-survey / repeat-spectrum / neighbour / astrometry / AllWISE vetting
   7 limits.py                 95% upper limits
  -- pass 2, original astrometric cuts (RUWE < 1.4, parallax/error >= 20) -> data/origcuts/
   8-11 analysis.py, selection.py, vet_candidates.py, limits.py   (DIM_STARS_CUTSET=orig)
  -- both
  12 robustness.py             alternative cuts and calibration choices; astrometric-quality diagnostics
  13 cmd_check.py              colour-magnitude test with and without spectroscopy
  14 make_figures.py           figures/*.png
  15 report_numbers.py         prints the numbers quoted in the report (also data/report_numbers.txt)

Stage 1 is network-bound (VizieR TAP, CDS XMatch, Gaia archive sync TAP): about 1.5-2.5 h.  Every
download is cached on disk and skipped when present.  Stages 6, 7, 10 and 11 make a few small network
queries (Gaia archive, CDS XMatch, IRSA, SIMBAD and VizieR TAP) for the candidate stars; cached as well.
Stages 2-15: about 1 h to 1.5 h single-threaded on a busy 4-core machine, peak memory about 5 GB.
"""
import os
import subprocess
import sys
import time

HERE = os.path.dirname(os.path.abspath(__file__))
STAGES = [("download_data.py", "final"), ("build_samples.py", "final"),
          ("analysis.py", "final"), ("injection.py", "final"), ("selection.py", "final"),
          ("vet_candidates.py", "final"), ("limits.py", "final"),
          ("analysis.py", "orig"), ("selection.py", "orig"), ("vet_candidates.py", "orig"), ("limits.py", "orig"),
          ("robustness.py", "final"), ("cmd_check.py", "final"), ("make_figures.py", "final"),
          ("report_numbers.py", "final")]

if __name__ == "__main__":
    args = sys.argv[1:]
    stages = list(STAGES)
    if "--no-download" in args:
        stages = [x for x in stages if x[0] != "download_data.py"]
    if "--from" in args:
        first = args[args.index("--from") + 1]
        name, _, cs = first.partition(":")
        key = (name.replace(".py", "") + ".py", cs or "final")
        stages = stages[stages.index(key):]
    for script, cutset in stages:
        env = dict(os.environ, PYTHONPATH=HERE + os.pathsep + os.environ.get("PYTHONPATH", ""),
                   DIM_STARS_CUTSET=cutset)
        t0 = time.time()
        print("\n######## %s  [cut set: %s]" % (script, cutset), flush=True)
        for attempt in (1, 2):                      # one retry: the shared file system occasionally hiccups on import
            r = subprocess.run([sys.executable, os.path.join(HERE, script)], cwd=HERE, env=env)
            if r.returncode == 0:
                break
            print("######## %s exited with code %d (attempt %d)" % (script, r.returncode, attempt), flush=True)
            time.sleep(5)
        if r.returncode != 0:
            raise SystemExit("stage %s failed" % script)
        print("######## %s done in %.0f s" % (script, time.time() - t0), flush=True)