"""Single entry point of the stellar_salting study: regenerates every table, number and figure.

    python3 run_all.py            # full pipeline (raw files are re-downloaded only if missing)
    python3 run_all.py --from 6   # restart at a given step number (uses the caches of earlier steps)

Raw data go to fermi_raw/stellar_salting/ (about 4 GB of catalogues plus ~3.5 GB of
per-star GALAH DR4 spectra for the vetting step; the revision steps 10-11 fetch about 1,900 more
spectra but keep only the line segments, ~0.1 GB); derived tables to ../data, figures to ../figures.
Run time from scratch is dominated by downloads (catalogues ~10 min, ~9,000 spectra ~1.3 h on a
slow link); from caches about 15 minutes, single-threaded.
"""
import importlib, os, sys, time

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)


def imp(name):
    # the shared mount occasionally returns a stale directory listing; retry the import
    for k in range(6):
        try:
            importlib.invalidate_caches()
            return importlib.import_module(name)
        except ModuleNotFoundError:
            time.sleep(1.0)
    return importlib.import_module(name)


STEPS = [
    (0, 's00_download', 'main', 'download public catalogues'),
    (1, 's01_prepare', 'main', 'extract columns to caches'),
    (2, 's02_sensitivity', 'main', 'per-star detection thresholds'),
    (3, 's03_candidates', 'main', 'natural-pattern model, effective samples, catalogue candidates'),
    (4, 's04_vetting', 'run', 'spectrum-level twin test (downloads spectra)'),
    (5, 's05_physics', 'main', 'fission pattern, convective-envelope masses'),
    (6, 's07_other_surveys', 'main', 'APOGEE, Gaia GSP-Spec, DR3 cross-check, CP stars, Przybylski'),
    (7, 's06_limits', 'main', 'vetted limits, robustness, survivors, Galaxy conversion'),
    (8, 's09_pr_search', 'main', 'direct Pr II search in the downloaded spectra (indicative only)'),
    (9, 's10_survivors', 'main', 'astrophysical vetting of the spectrum-supported candidates'),
    (10, 's11_flagged', 'main', 'revision: census and twin test of candidates among flag_sp != 0 stars (downloads spectra)'),
    (11, 's12_injection', 'main', 'revision: injection-recovery test of the spectrum-vetting stage (downloads spectra)'),
    (12, 's08_figures', 'main', 'figures'),
]

if __name__ == '__main__':
    start = 0
    if '--from' in sys.argv:
        start = int(sys.argv[sys.argv.index('--from') + 1])
    for n, mod, fn, what in STEPS:
        if n < start:
            continue
        t0 = time.time()
        print('=== step %d: %s (%s)' % (n, mod, what)); sys.stdout.flush()
        getattr(imp(mod), fn)()
        print('    done in %.0f s' % (time.time() - t0)); sys.stdout.flush()