#!/usr/bin/env python3 """Collect every number quoted in report.md from the tables in data/ into one text file (data/report_numbers.txt), so that the hand-written report can be checked against the pipeline output.""" import json import os import numpy as np import pandas as pd import fpk_common as F def pct(x, nd=2): if x is None or not np.isfinite(x): return "none" return ("%%.%dg%%%%" % nd) % (100 * x) def main(): out = [] P = out.append D = lambda fn: os.path.join(F.DATA, fn) fits = json.load(open(D("fp_fits.json"))) P("== samples and fits") lad = pd.read_csv(D("sdss_cut_ladder.csv")) P("SDSS cut ladder: " + "; ".join("%s %d" % (c[:14], n) for c, n in zip(lad.cut, lad.n_remaining))) for k in ("sdss", "6df", "howlett"): f = fits[k] P("%s: N=%d c_sigma=%.3f c_R=%.3f c_z=%.3f s_int=%.4f MAD=%.4f orth a=%.3f b=%.3f FJ scatter=%.3f" % (k, f["n"], f["fpI_coef"][1], f["fpI_coef"][2], f["fpI_coef"][3], f["fpI_s_int"], f["fpI_mad"], f["classic_orth"]["a"], f["classic_orth"]["b"], f["fj_s_int"])) P("\n== individual limits (95%), FP, best of four thresholds") fin = pd.read_csv(D("limits_final.csv")) best = pd.read_csv(D("limits_individual_best.csv")) red = pd.read_csv(D("limits_redext.csv")) red = red[red.relation == "FP"].set_index("f") for samp in fin["sample"].unique(): for kind in ("conservative", "vetted"): g = fin[(fin["sample"] == samp) & (fin.kind == kind)] P("%s, %s: " % (samp, kind) + "; ".join("f=%.2f N_eff=%d M=%.0f n_obs=%d q95=%s" % ( r.f, r.N_eff, r.M_eff, r.n_obs, pct(r.q95)) for _, r in g.iterrows())) P("no red colour edge (N=%d): " % red.N.iloc[0] + "; ".join( "f=%.2f N_eff=%d n_obs=%d q95=%s" % (f, r.N_eff, r.n_obs, pct(r.q95_best)) for f, r in red.iterrows())) P("variants and other samples (f = 0.3 / 0.5 / 0.75 / 0.9):") for (samp, var, rel), g in best.groupby(["sample", "variant", "relation"]): g = g.set_index("f") P(" %-40s %-11s %s: %s" % (samp, var, rel, " / ".join(pct(g.loc[f, "q95_best"]) for f in (0.3, 0.5, 0.75, 0.9)))) P("mixtures:") for _, r in pd.read_csv(D("limits_mixture.csv")).iterrows(): P(" %s, %s, f in [%.2f, %.2f]: total fraction < %s (%s)" % ( r["sample"], r.convention, r.f_lo, r.f_hi, pct(r.max_total_fraction), r.composition)) P("population behind N_eff:") for _, r in pd.read_csv(D("neff_population.csv")).iterrows(): P(" %s f=%.2f: N_eff=%d (%.1f%% of sample; %d if S/N ~ flux), median z %.3f, 95%% below z=%.3f, faintest mag " "%.2f, mean dI %+.3f" % (r["sample"], r.f, r.N_eff, 100 * r.share_of_sample, r.N_eff_snr_linear, r.z_median, r.z_p95, r.mag_faintest, r.mean_dI)) P("\n== tails, outliers, clipping") for _, r in pd.read_csv(D("tail_asymmetry.csv")).iterrows(): P(" %s beyond %.2f dex: %d dim / %d bright (rate dim %s; binomial p for dim excess %.3g)" % ( r["sample"], r.beyond_dex, r.n_dim, r.n_bright, pct(r.rate_dim, 3), r.p_binomial_dim_excess)) far = pd.read_csv(D("far_tail_objects.csv")) P(" dI <= -0.60: " + "; ".join("%s (%.3f, %+.3f) dI=%.2f %s" % ( r.label, r.ra, r["dec"], r.dI, "explained:" + r.vetting_class if r.explained else "UNEXPLAINED") for _, r in far.iterrows())) for name in ("sdss", "6df"): c = pd.read_csv(D("candidates_%s_vetted.csv" % name)) P(" %s candidates (dI <= -0.45): %d, explained %d, unexplained %d" % ( name, len(c), int(c.explained.sum()), int((c.explained == 0).sum()))) for _, r in pd.read_csv(D("howlett_eligibility.csv")).iterrows(): P(" Howlett+22, dI <= %.2f: %d in tail, %d eligible, rate %.3f, expected %.1f, observed %d, binomial p %.2g, " "Poisson p %.2g" % (r.threshold_dex, r.n_tail, r.n_tail_eligible, r.inclusion_rate_of_eligible, r.expected_in_howlett, r.observed_in_howlett, r.p_binomial_le_observed, float(np.exp(-r.expected_in_howlett)) if r.observed_in_howlett == 0 else np.nan)) for _, r in pd.read_csv(D("clipping_diagnostics.csv")).iterrows(): P(" dI <= %.2f: 6dFGSv all %d, in published sample %d" % (r.threshold, r.sixdf_all, r.sixdf_published)) rs = pd.read_csv(D("redside_check.csv")) for _, r in rs.iterrows(): P(" colour: %s: N=%d; dI<=-0.30/-0.45/-0.60: %d / %d / %d; of the first, >0.2 mag beyond red edge: %d" % ( r.subset, r.N, r["n_dI_le_-0.30"], r["n_dI_le_-0.45"], r["n_dI_le_-0.60"], r["n_dI_le_-0.30_and_excess_gt_0.2mag"])) summ = json.load(open(D("revision_summary.json"))) w = summ["wise_apertures"] P(" WISE gamma, %d unexplained outliers: 300K psf <= %.3f, ap8 <= %.3f, max <= %.3f; 150K psf <= %.3f, ap8 <= %.3f, " "max <= %.3f" % (w["n_unexplained"], w["gamma300K_psf_max_unexplained"], w["gamma300K_ap8_max_unexplained"], w["gamma300K_max_max_unexplained"], w["gamma150K_psf_max_unexplained"], w["gamma150K_ap8_max_unexplained"], w["gamma150K_max_max_unexplained"])) P("\n== regions") for _, r in pd.read_csv(D("survey_volume.csv")).iterrows(): P(" %s: z %.4f-%.4f, %.0f deg2, %.3g Mpc^3" % (r["sample"], r.zmin, r.zmax, r.footprint_deg2, r.volume_mpc3)) lr = pd.read_csv(D("limits_regions.csv")) for (samp, var, R), g in lr[lr["mode"] == "volume"].groupby(["sample", "variant", "R_mpc"]): P(" %s %s R=%g: S_min=%.2f (p=%.4f), S_max=%.2f (p=%.3f)" % ( samp, var, R, g.S_min_obs.iloc[0], g.p_shuffle_min.iloc[0], g.S_max_obs.iloc[0], g.p_shuffle_max.iloc[0])) for _, r in pd.read_csv(D("region_coherent_rms.csv")).iterrows(): P(" coherent rms %s %s R=%g: %.4f dex (95%% upper %.4f)" % (r["sample"], r.variant, r.R_mpc, r.rms_dex, r.rms_dex_hi95)) for _, r in pd.read_csv(D("pv_expectation.csv")).iterrows(): P(" linear-theory bulk flow R=%g z=%.2f: %.0f km/s -> %.4f dex" % (r.R_mpc, r.z, r.v_los_rms_kms, r.dI_rms_dex)) for _, r in pd.read_csv(D("region_scan_howlett_groupz.csv")).iterrows(): P(" Howlett+22 group-z scan R=%g: S_min=%.2f (p=%.3f)" % (r.R_mpc, r.S_min, r.p_shuffle)) P(" region limits (weaker of rules A and B):") for _, r in pd.read_csv(D("limits_regions_final.csv")).iterrows(): P(" %s R=%g f=%.2f: B %s (k_obs=%d, MC %.0f%%, share %s, %.0f spheres, z90 %.3f) | A %s (MC %.0f%%, share %s, " "%.0f spheres, z90 %.3f) | quoted %s (rule %s), n < %.3g /Gpc^3 | galaxy-weighted: eff A %.3f B %.3f, " "inside < %s (A) %s (B)" % ( r["sample"], r.R_mpc, r.f, pct(r.fill95_ruleB), r.k_obs_ruleB, 100 * r.mc_relerr_ruleB, pct(r.tested_share_ruleB), r.n_sphere_volumes_ruleB, r.z_det_p90_ruleB, pct(r.fill95_ruleA), 100 * r.mc_relerr_ruleA, pct(r.tested_share_ruleA), r.n_sphere_volumes_ruleA, r.z_det_p90_ruleA, pct(r.fill95_weaker), r.weaker_rule, r.n95_per_gpc3_weaker, r.gal_eff_ruleA, r.gal_eff_ruleB, pct(r.frac_gal_inside95_ruleA), pct(r.frac_gal_inside95_ruleB))) P("\n== clusters") c = pd.read_csv(D("coma_decomposition.csv")).iloc[0] P(" Coma scan-selected sphere R=10: N=%d mean %.3f +- %.3f; FoG %.3f (n=%d); photometry %.3f +- %.3f; identified " "%.0f%%; remainder %.3f +- %.3f; trend individual-z %.3f +- %.3f, group-z %.3f +- %.3f; remainder - group trend " "%.3f +- %.3f" % (c.N, c.mean_dI, c.sem_dI, c.fog_part, c.n_with_group_z, c.phot_part, c.phot_err, 100 * c.identified_fraction, c.remainder, c.remainder_err, c.env_trend_individual_z, c.env_trend_individual_z_err, c.env_trend_group_z, c.env_trend_group_z_err, c.remainder_minus_group_trend, c.remainder_minus_group_trend_err)) for _, r in pd.read_csv(D("cluster_spheres.csv")).iterrows(): P(" %s catalogue-centred R=%g: N=%d mean %.3f +- %.3f (apparent dimming %.1f%%); group-z subset n=%d: %.3f +- " "%.3f" % (r.cluster, r.R_mpc, r.N, r.mean_dI, r.sem_dI, 100 * (1 - 10 ** r.mean_dI), r.n_group_z, r.mean_dI_groupz_subset, r.sem_groupz_subset)) pt = pd.read_csv(D("region_extreme_patches.csv")) q = pt[(pt["sample"] == "sdss") & (pt.variant == "raw") & pt.kind.str.contains("negative")] for _, r in q.iterrows(): P(" SDSS dim sphere R=%g: S=%.2f N=%d mean dI %.3f (%.0f%%) at (%.2f, %+.2f, z=%.4f)" % ( r.R_mpc, r.S, r.N, r.mean_dI_dex, 100 * r.equiv_dimming_f, r.ra, r["dec"], r.z)) for _, r in pd.read_csv(D("region_patch_identifications.csv")).iterrows(): P(" identification R=%g S=%.2f: %d catalogued clusters in sphere; nearest %s at %.2f deg (z=%.4f); Abell: %s" % ( r.R_mpc, r.S, r.n_clusters_in_sphere, r.nearest_cluster, r.nearest_sep_deg, r.nearest_z, r.abell_clusters_in_sphere if isinstance(r.abell_clusters_in_sphere, str) else "-")) txt = "\n".join(out) + "\n" with open(D("report_numbers.txt"), "w") as fh: fh.write(txt) print(txt) return True if __name__ == "__main__": main()