diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 32e76ca44..2bb4238a7 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -115,6 +115,17 @@ jobs: IMAGE=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) docker run --rm -e HYPOTHESIS_PROFILE=ci -e SHAPEPIPE_ON_CANDIDE=0 "$IMAGE" pytest -rX + # tests/workflow drives the Snakefile through snakemake's API. Snakemake + # is a host tool and stays out of the image (it wraps each job in the + # container), so the suite above leaves that directory out + # (the root conftest.py); here it is installed into this throwaway container + # at the host pin range (workflow/README.md) and the directory runs alone. + - name: Test — workflow DAG (snakemake) + run: | + IMAGE=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) + docker run --rm -e SHAPEPIPE_ON_CANDIDE=0 "$IMAGE" bash -c \ + "uv pip install 'snakemake>=9,<10' && pytest -rX --no-cov tests/workflow" + # ---------------------------------------------------------------- # Publish (push events only — never on pull_request, incl. forks). # Fires on any branch; the image is tagged with the branch name. diff --git a/astra.yaml b/astra.yaml index 87c844909..4a83c5308 100644 --- a/astra.yaml +++ b/astra.yaml @@ -572,9 +572,8 @@ analyses: No detection image, no flag image, and the default_noimaflags.param column list: tiles have no instrument flag image and no detection coadd exists, so detection sees every tile pixel and the tile - catalogue carries no IMAFLAGS_ISO. [LINT] - final_cat.param, read by the post-processing merge, requests - IMAFLAGS_ISO, which the tile chain never produces (issue #912). + catalogue carries no IMAFLAGS_ISO; final_cat.param, the merge's + exact allow-list, does not request it. Values: SEXTRACTOR_RUNNER.DETECTION_IMAGE = False; SEXTRACTOR_RUNNER.FLAG_IMAGE = False. diff --git a/conftest.py b/conftest.py index 5889c46f9..f481d46d1 100644 --- a/conftest.py +++ b/conftest.py @@ -28,6 +28,16 @@ settings.load_profile(os.environ.get("HYPOTHESIS_PROFILE", "ci")) +# ``tests/workflow/`` drives the Snakefile through snakemake's API. Snakemake is +# a host tool, not part of the image (it wraps each job in the container), so +# where it is absent the directory is left out of collection; CI runs it in its +# own step after installing snakemake (deploy-image.yml). +try: + import snakemake # noqa: F401 +except ModuleNotFoundError: + collect_ignore = ["tests/workflow"] + + # --------------------------------------------------------------------------- # # Candide detection # --------------------------------------------------------------------------- # @@ -36,7 +46,7 @@ # host this suite is most often driven from is ``c03``. We match the candide # node-name families rather than a fixed list so new nodes are covered, and # allow an explicit override for CI or odd hostnames. -_CANDIDE_HOST_RE = re.compile(r"^(c\d|n\d{2})", re.IGNORECASE) +_CANDIDE_HOST_RE = re.compile(r"^(c\d{2}|n\d{2})$", re.IGNORECASE) def on_candide(): @@ -44,7 +54,8 @@ def on_candide(): The check is, in order: an explicit ``SHAPEPIPE_ON_CANDIDE`` override (``1``/``0``), then the hostname against the candide node-name families - (``c0x`` login, ``nXX`` compute). Cheap, import-safe, no cluster calls. + (``c0x`` login, ``nXX`` compute; whole bare hostname, so ``c6.nibi.sharcnet`` + does not match). Cheap, import-safe, no cluster calls. """ override = os.environ.get("SHAPEPIPE_ON_CANDIDE") if override is not None: diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 7571cb052..d68a20eda 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -58,22 +58,32 @@ def params_from_run_config(params, defaults): raise ValueError(f"run config {params['run_config']} sets neither " "outputs.products_dir nor outputs.run_dir") - # The same derivation as the workflow's merge_final_cats rule: the patch - # dir is the branch dir holding product/tiles, and -i is its parent. + # The group and file are named for `run:`, as the workflow's + # final_cat_merge names them. -P is also the directory the -I walk + # matches under -i, so this needs the run template's layout, + # products_dir = //product, and -i is . + patch = cfg["run"] patch_dir = os.path.dirname(os.path.normpath(products)) - patch = os.path.basename(patch_dir) + if os.path.basename(patch_dir) != patch: + raise ValueError( + f"run config {params['run_config']}: products_dir {products} is " + f"not /{patch}/product, so -c cannot locate the tiles; " + "pass -i and -P explicitly") derived = { "image_sims": True, "input_root_dir": os.path.dirname(patch_dir), "patch": patch, "merged_cat_path": os.path.join(products, f"final_cat_{patch}.hdf5"), - "output_summary": os.path.join(products, "n_tiles_final.txt"), "param_path": os.path.join(repo, "workflow", "config", "cfis_image_sims", "final_cat.param"), } for key, value in derived.items(): if params.get(key) == defaults.get(key): params[key] = value + # The tile count is the file's n_tiles attribute; the text summary is + # written only when -o asks for it. + if params.get("output_summary") == defaults.get("output_summary"): + params["output_summary"] = None return params @@ -152,7 +162,8 @@ def params_default(): "param_path": "parameter file path, if not given use all columns, default={}", "patch": "patch number (data) or grid subdir (image_sims), default={}", "list_only": "print list of patches and IDs only, default={}", - "output_summary": "output file for numbre of tiles, default={}", + "output_summary": "output file for number of tiles (with -c, written" + " only if given), default={}", "ID": "ID for single-ID operation, default={}", "single_op": "single ID operation, allowed are 'check', 'add', 'remove'; default={}", "image_sims": "image simulations mode (different dir layout and run prefix), default={}", @@ -205,13 +216,17 @@ def read_param_file(path, verbose=False): print("No parameters read", end="") print(" into merged catalogue") - param_list_unique = list(set(param_list)) - + # Ordered dedup. list(set(...)) reordered the columns by the process's + # string hash seed, so two runs of this tool over the same inputs produced + # files whose datasets differed in column ORDER — which is part of a + # structured dtype, and therefore part of the file. + param_list_unique = list(dict.fromkeys(param_list)) + if verbose: n = len(param_list) - len(param_list_unique) - if n > 1: - print("Removed {n} duplicate entries") - + if n > 0: + print(f"Removed {n} duplicate entries") + return param_list_unique @@ -317,8 +332,9 @@ def print_list(params): if verbose: print(f"Total: {n_tiles} tiles") - with open(params["output_summary"], "w") as f_out: - print(n_tiles, file=f_out) + if params["output_summary"]: + with open(params["output_summary"], "w") as f_out: + print(n_tiles, file=f_out) # Write n_tiles to HDF5 file header with h5py.File(params["merged_cat_path"], "a") as hdf5_file: @@ -359,8 +375,15 @@ def get_patch_group(hdf5_file, patch, verbose=False): def read_data(fits_file, params): - """Read Data. - + """Read the parameter list's columns out of one catalogue. + + @sc [label:schema] read-data-raises-on-missing-column + A requested column the catalogue lacks raises `KeyError` naming it; it is + never skipped or filled. `copy_data` keeps only columns present in the + source, so this raise is the one place a missing name stops a merge, and + without it a tile short a per-epoch slot would land in the merged file + silently narrower, with that slot's exposure identity gone. Enforced by + tests/unit/test_final_cat_merge_invariants.py. """ with fits.open(fits_file) as hdu_list: try: @@ -373,16 +396,20 @@ def read_data(fits_file, params): if params["param_list"] is None: params["param_list"] = [col for col in data.keys()] - try: - extracted_data = {col: data[col] for col in params["param_list"]} - dtype = data.dtype - except: - print(f"Error for ID {id}, path {fits_file}") - for col in params["param_list"]: - if col not in data: - print(col, end=" ") - print() - continue + # RAISE, do not print and fall through. The bare `except:` this replaces + # left extracted_data and dtype unbound, so the caller's own error was an + # UnboundLocalError from the return statement below, naming neither the + # file nor the column that was actually missing. + present = set(data.dtype.names or ()) + missing = [col for col in params["param_list"] if col not in present] + if missing: + raise KeyError( + f"{fits_file}: missing {len(missing)} of the " + f"{len(params['param_list'])} requested column(s): " + f"{' '.join(missing)}" + ) + extracted_data = {col: data[col] for col in params["param_list"]} + dtype = data.dtype return extracted_data, dtype @@ -391,16 +418,29 @@ def copy_data(param_list, extracted_data, dtype): """Copy Data. """ + # THE REQUESTED COLUMNS ONLY, IN THE PARAMETER FILE'S ORDER. Two things + # are being fixed here and they are easy to conflate. Allocating with the + # source's full dtype and filling only the requested columns left every + # other column as uninitialised memory — meaningless values, and different + # bytes on every run over the same inputs. And ordering the result by the + # SOURCE catalogue's columns made the output dtype a property of the + # catalogue rather than of the parameter file: two tiles written by + # different ShapePipe versions, whose catalogues order or extend their + # columns differently, then landed in one merged file with two different + # structured dtypes, which np.concatenate refuses. The parameter file is + # the schema; it says which columns AND in what order. + wanted = set(dtype.names or ()) + columns = [col for col in param_list if col in wanted] + subset = np.dtype([(col, dtype[col]) for col in columns]) + # Initialize new data structure structured_data = np.empty( len(extracted_data[param_list[0]]), - dtype=dtype, + dtype=subset, ) # Loop over parameters - for col in param_list: - if not col in extracted_data: - print(f"Column {col} not in file with ID {id}") + for col in columns: structured_data[col] = extracted_data[col] #if isinstance(extracted_data[col][0], (np.ndarray, tuple, list)): @@ -547,12 +587,14 @@ def process(params): structured_data = copy_data(params["param_list"], extracted_data, dtype) - # Create a new dataset + # Create a new dataset. dtype comes from the array copy_data + # built, not from the source catalogue: they differ now that + # copy_data allocates the requested columns alone. try: patch_group.create_dataset( str(id), data=structured_data, - dtype=dtype, + dtype=structured_data.dtype, ) except: print(f"Error for {id}: Could not create dataset in group {patch}") diff --git a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py index 0d724d210..5e579f6dd 100644 --- a/src/shapepipe/modules/merge_starcat_package/merge_starcat.py +++ b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py @@ -17,6 +17,36 @@ from shapepipe.pipeline import file_io +def _stack(chunks, dtype=None): + """Concatenate one column's per-catalogue arrays into a single array. + + THE COLUMN ACCUMULATORS ARE LISTS OF ARRAYS, ONE PER INPUT CATALOGUE, and + not lists of values, because these classes are the last step of a whole + campaign. ``x += list(data["X"])`` turns 4 bytes of float32 payload into a + 32-byte python object plus an 8-byte pointer in a list that overallocates — + measured at ~10x the input bytes end to end, which put a full-survey merge + (~20k exposures x 40 CCDs) at ~400 GB of RAM and made it unrunnable on any + node. One array per catalogue plus one concatenate at the end holds ~1x, and + produces the identical output: np.array() over a list of numpy scalars and + np.concatenate() over the arrays they came from agree on dtype and on order. + + IT EMPTIES THE LIST IT IS GIVEN, and that is not a side effect to tidy away + later — it is half the saving. np.concatenate holds the chunks and the + result at once, so a caller that stacks every column while all its + chunk lists are still alive peaks at twice the campaign. Released column by + column, the peak is one campaign plus one column. Callers stack once, at the + end, and do not touch the accumulators afterwards. + + An empty input list is a merge over no catalogues, which the callers guard + against; it returns an empty array so the output column still exists. + """ + if not chunks: + return np.array([], dtype=dtype or np.float64) + out = np.concatenate(chunks) + del chunks[:] + return out + + class MergeStarCatMCCD(object): """Merge Star Catalogue MCCD. @@ -315,66 +345,37 @@ def process(self): model_var.append(model_var_val) model_var_size.append(model_var_val.size) + # ONE ARRAY PER CATALOGUE PER COLUMN (see _stack): the per-value + # python lists this replaces cost ~10x the input bytes. # positions - x += list( - starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 0] - ) - y += list( - starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"][:, 1] - ) + pos = starcat_j[self._hdu_table].data["GLOB_POSITION_IMG_LIST"] + x.append(np.asarray(pos[:, 0])) + y.append(np.asarray(pos[:, 1])) # RA and DEC positions try: - ra += list(starcat_j[self._hdu_table].data["RA_LIST"][:]) - dec += list(starcat_j[self._hdu_table].data["DEC_LIST"][:]) + ra.append(np.asarray(starcat_j[self._hdu_table].data["RA_LIST"][:])) + dec.append(np.asarray(starcat_j[self._hdu_table].data["DEC_LIST"][:])) except Exception: - ra += list( - np.zeros( - starcat_j[self._hdu_table] - .data["GLOB_POSITION_IMG_LIST"][:, 0] - .shape, - dtype=int, - ) - ) - dec += list( - np.zeros( - starcat_j[self._hdu_table] - .data["GLOB_POSITION_IMG_LIST"][:, 0] - .shape, - dtype=int, - ) - ) + ra.append(np.zeros(pos[:, 0].shape, dtype=int)) + dec.append(np.zeros(pos[:, 0].shape, dtype=int)) # shapes (convert sigmas to T = 2 sigma^2) - g1_psf += list( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 0] - ) - g2_psf += list( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 1] - ) - size_psf += list( - cs_size.sigma_to_T( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 2] - ) - ) - g1 += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 0]) - g2 += list(starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 1]) - size += list( - cs_size.sigma_to_T( - starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 2] - ) - ) + psf_mom = starcat_j[self._hdu_table].data["PSF_MOM_LIST"] + star_mom = starcat_j[self._hdu_table].data["STAR_MOM_LIST"] + g1_psf.append(np.asarray(psf_mom[:, 0])) + g2_psf.append(np.asarray(psf_mom[:, 1])) + size_psf.append(np.asarray(cs_size.sigma_to_T(psf_mom[:, 2]))) + g1.append(np.asarray(star_mom[:, 0])) + g2.append(np.asarray(star_mom[:, 1])) + size.append(np.asarray(cs_size.sigma_to_T(star_mom[:, 2]))) # flags - flag_psf += list( - starcat_j[self._hdu_table].data["PSF_MOM_LIST"][:, 3] - ) - flag_star += list( - starcat_j[self._hdu_table].data["STAR_MOM_LIST"][:, 3] - ) + flag_psf.append(np.asarray(psf_mom[:, 3])) + flag_star.append(np.asarray(star_mom[:, 3])) # ccd id list - ccd_nb += list(starcat_j[self._hdu_table].data["CCD_ID_LIST"]) + ccd_nb.append(np.asarray(starcat_j[self._hdu_table].data["CCD_ID_LIST"])) starcat_j.close() @@ -447,15 +448,21 @@ def process(self): ) # Mask and transform to numpy arrays - flagmask = np.abs(np.array(flag_star) - 1) * np.abs( - np.array(flag_psf) - 1 - ) - psf_e1 = np.array(g1_psf)[flagmask.astype(bool)] - psf_e2 = np.array(g2_psf)[flagmask.astype(bool)] - psf_r2 = np.array(size_psf)[flagmask.astype(bool)] - star_e1 = np.array(g1)[flagmask.astype(bool)] - star_e2 = np.array(g2)[flagmask.astype(bool)] - star_r2 = np.array(size)[flagmask.astype(bool)] + # Concatenate once, here: everything below already wanted arrays and + # was calling np.array() on python lists to get them (see _stack). + x, y, ra, dec = _stack(x), _stack(y), _stack(ra), _stack(dec) + g1_psf, g2_psf, size_psf = _stack(g1_psf), _stack(g2_psf), _stack(size_psf) + g1, g2, size = _stack(g1), _stack(g2), _stack(size) + flag_psf, flag_star = _stack(flag_psf), _stack(flag_star) + ccd_nb = _stack(ccd_nb) + + flagmask = np.abs(flag_star - 1) * np.abs(flag_psf - 1) + psf_e1 = g1_psf[flagmask.astype(bool)] + psf_e2 = g2_psf[flagmask.astype(bool)] + psf_r2 = size_psf[flagmask.astype(bool)] + star_e1 = g1[flagmask.astype(bool)] + star_e2 = g2[flagmask.astype(bool)] + star_r2 = size[flagmask.astype(bool)] rmse, mean, std_dev = MSC.stats_calculator(star_e1, psf_e1) self._w_log.info( @@ -551,85 +558,145 @@ def __init__( self._hdu_table = hdu_table self._input_cat_type = input_cat_type + # The columns this class writes, and where each comes from. Kept as data + # rather than as repeated lines, because a two-pass merge would otherwise + # state every column three times: to size it, to allocate it and to fill + # it. Size columns already hold T = 2 sigma^2. + _COLUMNS = ( + ("X", "X"), ("Y", "Y"), ("RA", "RA"), ("DEC", "DEC"), + ("HSM_G1_PSF", "HSM_G1_PSF"), ("HSM_G2_PSF", "HSM_G2_PSF"), + ("HSM_T_PSF", "HSM_T_PSF"), + ("HSM_M4_1_PSF", "HSM_M4_1_PSF"), ("HSM_M4_2_PSF", "HSM_M4_2_PSF"), + ("HSM_RHO4_PSF", "HSM_RHO4_PSF"), + ("HSM_G1_STAR", "HSM_G1_STAR"), ("HSM_G2_STAR", "HSM_G2_STAR"), + ("HSM_T_STAR", "HSM_T_STAR"), + ("HSM_M4_1_STAR", "HSM_M4_1_STAR"), ("HSM_M4_2_STAR", "HSM_M4_2_STAR"), + ("HSM_RHO4_STAR", "HSM_RHO4_STAR"), + ("HSM_FLAG_PSF", "HSM_FLAG_PSF"), ("HSM_FLAG_STAR", "HSM_FLAG_STAR"), + ) + # Present in psfex_interp output, absent from pix2wcs-converted files + # (MKDEBUG); zero-filled when missing rather than failing the merge. + _OPTIONAL = (("MAG", "MAG"), ("SNR", "SNR"), ("ACCEPTED", "ACCEPTED")) + + def _ccd_nb(self, path): + """The CCD number this catalogue's rows carry, parsed from its name.""" + return re.split(r"\-([0-9]*)\-([0-9]+)\.", path)[-2] + def process(self): """Process. Process merging. + TWO PASSES, AND NEITHER HOLDS THE CAMPAIGN TWICE. The first reads only + the FITS HEADER of every input — NAXIS2, the row count — and never + touches a data block; the second allocates the output columns once, at + their exact final length, and fills them slice by slice. Peak memory is + therefore ONE output plus ONE input catalogue. + @sc [label:schema] psfex-starcat-columns-strict Every ``HSM_*`` column is read by name with no fallback, so the set - read here equals the set ``PSFExInterpolator._write_output_validation`` - writes (``test_hsm_column_seams``). - + read here (``_COLUMNS``) equals the set + ``PSFExInterpolator._write_output_validation`` writes + (``test_hsm_column_seams``). + + What this replaces, in two steps, is instructive about the cost of the + obvious code. Accumulating each column into a python LIST OF VALUES — + ``x += list(data["X"])`` — turned 4 bytes of float32 payload into a + 32-byte object plus an 8-byte pointer, measured at ~10x the input bytes + end to end and putting a full-survey merge (~20k exposures x 40 CCDs) at + ~400 GB. Accumulating one ARRAY PER CATALOGUE and concatenating once + brought that to ~5.5x. This pass structure removes what was left of the + accumulation: there are no chunks, and no concatenate that must hold its + inputs and its result at the same time. + + ``self._input_file_list`` MUST BE ITERABLE TWICE, which the module + runner's list is. A one-shot generator is not, and would silently merge + nothing on the second pass — hence the explicit row-count check below. """ - x, y, ra, dec = [], [], [], [] - g1_psf, g2_psf, size_psf = [], [], [] - m4_1_psf, m4_2_psf, rho4_psf = [], [], [] - g1, g2, size = [], [], [] - m4_1_star, m4_2_star, rho4_star = [], [], [] - flag_psf, flag_star = [], [] - mag, snr, psfex_acc = [], [], [] - ccd_nb = [] - self._w_log.info( f"Merging {len(self._input_file_list)} star catalogues" ) + # --- pass 1: row counts and dtypes, from headers alone -------------- + # THE OPTIONAL COLUMNS ARE A PER-FILE QUESTION, NOT A PER-MERGE ONE. + # A pix2wcs-converted catalogue has no MAG/SNR/ACCEPTED while an + # ordinary one does, and a merge can be handed both. Deciding from the + # first file alone got it wrong in both directions: converted-first + # zero-filled the real values of every ordinary file behind it, and + # ordinary-first raised KeyError on the first converted one. So the + # dtype comes from ANY file that carries the column, and pass 2 asks + # each file for itself. + names, dtypes, opt_dtypes, n_total = [], None, {}, 0 for name in self._input_file_list: try: - starcat_j = fits.open(name[0], memmap=False, ignore_missing_simple=True) - except OSError as e: + with fits.open(name[0], memmap=False, + ignore_missing_simple=True) as starcat_j: + hdu = starcat_j[self._hdu_table] + n_rows = hdu.header["NAXIS2"] + # ColDefs.dtype describes the table without reading it. + # NOTE: it is the RAW storage dtype and ignores TSCAL/TZERO, + # so a scaled column would be allocated narrower than the + # values .data returns. Latent, not live: no validation_psf + # column is scaled. Read the dtype off .data if one ever is. + cols = hdu.columns.dtype + if dtypes is None: + dtypes = cols + for _, col in self._OPTIONAL: + if col not in opt_dtypes and col in (cols.names or ()): + opt_dtypes[col] = cols[col] + except OSError: print(f"Error while opening file '{name[0]}'") #raise continue - - data_j = starcat_j[self._hdu_table].data - - # positions - x += list(data_j["X"]) - y += list(data_j["Y"]) - ra += list(data_j["RA"]) - dec += list(data_j["DEC"]) - - # shapes (size column already holds T = 2 sigma^2) - g1_psf += list(data_j["HSM_G1_PSF"]) - g2_psf += list(data_j["HSM_G2_PSF"]) - size_psf += list(data_j["HSM_T_PSF"]) - m4_1_psf += list(data_j["HSM_M4_1_PSF"]) - m4_2_psf += list(data_j["HSM_M4_2_PSF"]) - rho4_psf += list(data_j["HSM_RHO4_PSF"]) - g1 += list(data_j["HSM_G1_STAR"]) - g2 += list(data_j["HSM_G2_STAR"]) - size += list(data_j["HSM_T_STAR"]) - m4_1_star += list(data_j["HSM_M4_1_STAR"]) - m4_2_star += list(data_j["HSM_M4_2_STAR"]) - rho4_star += list(data_j["HSM_RHO4_STAR"]) - - # flags - flag_psf += list(data_j["HSM_FLAG_PSF"]) - flag_star += list(data_j["HSM_FLAG_STAR"]) - - # misc - - # MKDEBUG: The following columns do not exist (yet) - # for psf converted (pix2wcs) files. - try: - mag += list(data_j["MAG"]) - except: - mag += list(np.zeros_like(data_j["X"])) - try: - snr += list(data_j["SNR"]) - except: - snr += list(np.zeros_like(data_j["X"])) + names.append(name[0]) + n_total += n_rows + + if dtypes is None: + raise ValueError("merge_starcat: no readable input catalogue") + + # --- allocate once, at the exact final length ----------------------- + data = {out: np.empty(n_total, dtype=dtypes[col]) + for out, col in self._COLUMNS} + for out, col in self._OPTIONAL: + # A column no file carries still gets a column, zero-filled, in the + # positional dtype the old code used for it. + data[out] = np.empty(n_total, dtype=opt_dtypes.get(col, dtypes["X"])) + # CCD_NB is one string per catalogue, repeated over its rows; its width + # is the widest CCD number in the campaign, which pass 1 already knows. + width = max((len(self._ccd_nb(n)) for n in names), default=1) + data["CCD_NB"] = np.empty(n_total, dtype=f"U{width}") + + # --- pass 2: fill --------------------------------------------------- + at = 0 + for name in self._input_file_list: try: - psfex_acc += list(data_j["ACCEPTED"]) - except: - psfex_acc += list(np.zeros_like(data_j["X"])) + starcat_j = fits.open(name[0], memmap=False, + ignore_missing_simple=True) + except OSError: + continue + data_j = starcat_j[self._hdu_table].data + n_rows = len(data_j) + sl = slice(at, at + n_rows) + + have = set(data_j.dtype.names or ()) + for out, col in self._COLUMNS: + data[out][sl] = data_j[col] + for out, col in self._OPTIONAL: + # THIS file's schema, not the merge's: zero-fill only the files + # that actually lack the column. + data[out][sl] = data_j[col] if col in have else 0 + data["CCD_NB"][sl] = self._ccd_nb(name[0]) + + at += n_rows + starcat_j.close() - # CCD number - ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", name[0])[-2]] * len( - data_j["RA"] - ) + if at != n_total: + # The two passes disagreed: an input changed under us, or the list + # was a one-shot iterable. Either way the output would be padded + # with uninitialised memory, so say so rather than write it. + raise ValueError( + f"merge_starcat: pass 1 counted {n_total} rows, pass 2 filled " + f"{at} — is the input list iterable more than once?") # Prepare output FITS catalogue # MKDEBUG: SEx_cat=True -> False @@ -640,31 +707,8 @@ def process(self): SEx_catalogue=False, ) - # Collect columns (size stored as T = 2 sigma^2) - data = { - "X": x, - "Y": y, - "RA": ra, - "DEC": dec, - "HSM_G1_PSF": g1_psf, - "HSM_G2_PSF": g2_psf, - "HSM_T_PSF": size_psf, - "HSM_M4_1_PSF": m4_1_psf, - "HSM_M4_2_PSF": m4_2_psf, - "HSM_RHO4_PSF": rho4_psf, - "HSM_G1_STAR": g1, - "HSM_G2_STAR": g2, - "HSM_T_STAR": size, - "HSM_M4_1_STAR": m4_1_star, - "HSM_M4_2_STAR": m4_2_star, - "HSM_RHO4_STAR": rho4_star, - "HSM_FLAG_PSF": flag_psf, - "HSM_FLAG_STAR": flag_star, - "MAG": mag, - "SNR": snr, - "ACCEPTED": psfex_acc, - "CCD_NB": ccd_nb, - } + # `data` was built by the two passes above (size stored as T = 2 + # sigma^2); every column is already an array of its final length. # Write file # MKDEBUG for psf conv (pix2WCS) files do not write as SExtractorCat; @@ -810,29 +854,36 @@ def process(self): data_j = starcat_j[self._hdu_table].data # positions - x += list(data_j["XWIN_IMAGE"]) - y += list(data_j["YWIN_IMAGE"]) - ra += list(data_j["XWIN_WORLD"]) - dec += list(data_j["YWIN_WORLD"]) - + x.append(np.asarray(data_j["XWIN_IMAGE"])) + y.append(np.asarray(data_j["YWIN_IMAGE"])) + ra.append(np.asarray(data_j["XWIN_WORLD"])) + dec.append(np.asarray(data_j["YWIN_WORLD"])) + + # PRE-EXISTING BUG, LEFT ALONE DELIBERATELY: these four REBIND the + # accumulators initialised above rather than appending to them, so + # only the LAST input file's ellipticities reach the output while + # every other column carries the whole merge. Setools is not wired + # to any workflow path today; fixing it is its own change with its + # own verification, and doing it silently inside a memory rewrite + # would bury it. m11, m20, m02 = self.get_moments(data_j) eps1, eps2 = self.get_ellipticity(m11, m20, m02, "epsilon") chi1, chi2 = self.get_ellipticity(m11, m20, m02, "chi") - size += list(data_j["FLUX_RADIUS"]) + size.append(np.asarray(data_j["FLUX_RADIUS"])) # flags - flags += list(data_j["FLAGS_WIN"]) - flags_ext += list(data_j["IMAFLAGS_ISO"]) + flags.append(np.asarray(data_j["FLAGS_WIN"])) + flags_ext.append(np.asarray(data_j["IMAFLAGS_ISO"])) # misc - mag += list(data_j["MAG_WIN"]) - snr += list(data_j["SNR_WIN"]) + mag.append(np.asarray(data_j["MAG_WIN"])) + snr.append(np.asarray(data_j["SNR_WIN"])) # CCD number - ccd_nb += [re.split(r"\-([0-9]*)\-([0-9]+)\.", name[0])[-2]] * len( - data_j["XWIN_IMAGE"] - ) + ccd_nb.append(np.full( + len(data_j["XWIN_IMAGE"]), + re.split(r"\-([0-9]*)\-([0-9]+)\.", name[0])[-2])) # Prepare output FITS catalogue output = file_io.FITSCatalogue( @@ -844,20 +895,20 @@ def process(self): # Collect columns # convert back to sigma for consistency data = { - "X": x, - "Y": y, - "RA": ra, - "DEC": dec, + "X": _stack(x), + "Y": _stack(y), + "RA": _stack(ra), + "DEC": _stack(dec), "EPS1": eps1, "EPS2": eps2, "CHI1": chi1, "CHI2": chi2, - "SIZE": size, - "FLAGS": flags, - "FLAGS_EXT": flags_ext, - "MAG": mag, - "SNR": snr, - "CCD_NB": ccd_nb, + "SIZE": _stack(size), + "FLAGS": _stack(flags), + "FLAGS_EXT": _stack(flags_ext), + "MAG": _stack(mag), + "SNR": _stack(snr), + "CCD_NB": _stack(ccd_nb, dtype="U1"), } # Write file diff --git a/tests/README.md b/tests/README.md index ea3e460e7..0f379365b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,6 +11,7 @@ is driven by `pytest` from the repo root (in the dev container — see the proje |----------|-------|----------| | `tests/module/` | **module-unit tests** — the fitter, file handler, split-exp, vignetmaker, ngmix internals, the GalSim weight-validation suite | per-module unit/property/integration tests; import package internals directly. (Relocated from `src/shapepipe/tests/` so the suite has one home.) | | `tests/unit/` | **structural tests** — every submodule imports, configs parse, shell scripts lint, runner metadata is well-formed, console entry points respond to `-h` | suite-level checks on the *tree*, not any one module | +| `tests/workflow/` | **Snakemake DAG checks** — isolated campaigns resolved through the Python API, without executing jobs | checks per-job dependencies, input modes, and campaign product paths | | `tests/science/` | **fast scientific guardrails** — controlled simulations with a known answer, runnable in the inner loop with nothing from the cluster | scientific correctness that must stay green on every commit | | `tests/cluster/` | **candide guardrails** — read real on-disk catalogs / submit cluster jobs | need the cluster + real data; marked and auto-skipped off it | | `tests/helpers/` | shared, non-test library code (cluster submission, artifact emission, the star-response R-function) | imported by tests as `tests.helpers.*`; not collected as tests | diff --git a/tests/module/test_hsm_column_seams.py b/tests/module/test_hsm_column_seams.py index b05c52b69..8459003b3 100644 --- a/tests/module/test_hsm_column_seams.py +++ b/tests/module/test_hsm_column_seams.py @@ -4,8 +4,9 @@ (psfex_interp ↔ merge_starcat) and ``psfex-me-shapes-columns`` / ``psf-epoch-slot-columns`` (psfex_interp ↔ make_cat). The producer side is ``_hsm_columns`` (``hsm-column-grammar``), which every psfex_interp writer -goes through; the consumers read column names as string literals with no -fallback, collected from each function's AST. +goes through; make_cat reads column names as string literals with no fallback, +collected from the function's AST, and merge_starcat reads its ``_COLUMNS`` +table. """ import ast @@ -24,25 +25,10 @@ ) -def _hsm_literals(func, subscript_of=None): - """Set of ``HSM_*`` string literals in ``func``'s source. - - With ``subscript_of``, only literals used as ``["HSM_..."]`` reads - count, so a column that is still *written* under the same name cannot - mask a dropped read. - """ +def _hsm_literals(func): + """Set of ``HSM_*`` string literals in ``func``'s source.""" tree = ast.parse(textwrap.dedent(inspect.getsource(func))) - if subscript_of is None: - nodes = (n for n in ast.walk(tree) if isinstance(n, ast.Constant)) - else: - nodes = ( - n.slice - for n in ast.walk(tree) - if isinstance(n, ast.Subscript) - and isinstance(n.value, ast.Name) - and n.value.id == subscript_of - and isinstance(n.slice, ast.Constant) - ) + nodes = (n for n in ast.walk(tree) if isinstance(n, ast.Constant)) return { n.value for n in nodes @@ -55,9 +41,18 @@ def _written(obj): def test_merge_starcat_reads_exactly_what_psfex_validation_writes(): - """psfex-validation-hsm-columns == psfex-starcat-columns-strict.""" + """psfex-validation-hsm-columns == psfex-starcat-columns-strict. + + ``process`` reads every ``_COLUMNS`` source by name with no fallback (only + ``_OPTIONAL`` is zero-filled), so the table is the read set. + """ written = _written("PSF") | _written("STAR") - read = _hsm_literals(MergeStarCatPSFEX.process, subscript_of="data_j") + read = { + col for _, col in MergeStarCatPSFEX._COLUMNS if col.startswith("HSM_") + } + assert not any( + col.startswith("HSM_") for _, col in MergeStarCatPSFEX._OPTIONAL + ) assert written == read, { "written_not_read": sorted(written - read), "read_not_written": sorted(read - written), diff --git a/tests/module/test_psf_grammar_properties.py b/tests/module/test_psf_grammar_properties.py index 6c7553f86..461597557 100644 --- a/tests/module/test_psf_grammar_properties.py +++ b/tests/module/test_psf_grammar_properties.py @@ -182,10 +182,12 @@ def _run_save_ngmix(ngmix_path, obj_ids, cat_size_target=None): ) # The shipped final-catalogue param files, two levels up from tests/module/. -# Both are consumer contracts updated to the new grammar, so both are checked. +# All are consumer contracts updated to the new grammar, so all are checked; +# final_cat_merge reads the cfis_image_sims one for image-sims campaigns. _ROOT = Path(__file__).resolve().parents[2] PARAM_PATHS = [ _ROOT / "workflow" / "config" / "cfis" / "final_cat.param", + _ROOT / "workflow" / "config" / "cfis_image_sims" / "final_cat.param", _ROOT / "example" / "unions_800" / "cat_matched.param", ] @@ -357,11 +359,12 @@ def test_emitted_column_names_match_grammar(obj_ids, tmp_path_factory): def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): """Every NGMIX_* token the param file names is a column the writer produces. - Each shipped final-catalogue param file (``workflow/config/cfis/final_cat.param`` - and ``example/unions_800/cat_matched.param``) is a consumer contract for the + Each shipped final-catalogue param file (``final_cat.param`` under + ``workflow/config/cfis/`` and ``cfis_image_sims/``, and + ``example/unions_800/cat_matched.param``) is a consumer contract for the final catalogue; ``create_final_cat`` keeps only the listed columns, so a token it names that the writer cannot emit is a silent, empty column - downstream. Both files are checked so a future divergence in either (a + downstream. Every file is checked so a future divergence in any (a typo'd or stale NGMIX token) cannot escape the consistency check. The one known exception — ``NGMIX_MOM_FAIL``, the moments-failure flag set by a different path — is excluded BY NAME, and we assert it is genuinely outside diff --git a/tests/unit/test_build_index_readiness.py b/tests/unit/test_build_index_readiness.py new file mode 100644 index 000000000..9757f8872 --- /dev/null +++ b/tests/unit/test_build_index_readiness.py @@ -0,0 +1,71 @@ +"""``build_index``: a tile is ready only while its exposure list exists. + +The index keeps a tile's edges after its exposure list disappears, because +clean_exposure's consumer sets must still see the tile that read an exposure. +Readiness must not ride on those edges: a tile named in ``missing.json`` is not +ready, for the merges (``campaign_tiles``) or for the Snakefile's +``TILES_READY``, and comes back when its list does. +""" + +import importlib.util +import json +import re +import sqlite3 +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" +TILE, OTHER = "210.282", "211.282" + + +def _load(): + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location( + "_build_index", SCRIPTS / "build_index.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +bi = _load() + + +def _list(run, tile, names): + path = bi.exp_list_path(run, tile) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(f"{n}\n" for n in names)) + return path + + +def test_a_missing_tile_is_not_ready_but_keeps_its_edges(tmp_path): + run, db = tmp_path / "run", tmp_path / "index.sqlite" + listed = _list(run, TILE, ["2605805p"]) + _list(run, OTHER, ["2605806p"]) + tiles = tmp_path / "tiles.txt" + tiles.write_text(f"{TILE}\n{OTHER}\n") + bi.build([TILE, OTHER], run, db) + assert bi.campaign_tiles(tiles, db) == [TILE, OTHER] + + listed.unlink() + bi.build([TILE, OTHER], run, db, missing_threshold=1.0) + assert json.loads((db.parent / "missing.json").read_text()) == [TILE] + assert bi.campaign_tiles(tiles, db) == [OTHER] + assert bi.campaign_exposures(tiles, db) == ["2605806"] + with sqlite3.connect(db) as con: + edges = con.execute("SELECT exp_id FROM tile_exposures " + "WHERE tile_id = ?", (TILE,)).fetchall() + assert edges == [("2605805",)], "the cleanup consumer edge was dropped" + + _list(run, TILE, ["2605805p"]) + bi.build([TILE, OTHER], run, db) + assert bi.campaign_tiles(tiles, db) == [TILE, OTHER] + + +def test_the_snakefile_reads_readiness_from_build_index(): + snakefile = (REPO_ROOT / "workflow" / "Snakefile").read_text() + line = re.search(r"^TILES_READY = .*$", snakefile, re.M).group(0) + assert "build_index.ready_tiles(" in snakefile and "_READY_INDEXED" in line diff --git a/tests/unit/test_campaign_lineage.py b/tests/unit/test_campaign_lineage.py new file mode 100644 index 000000000..492ec0ec8 --- /dev/null +++ b/tests/unit/test_campaign_lineage.py @@ -0,0 +1,149 @@ +"""Config lineage of the campaign name and the campaign product paths. + +Enforces workflow/CONTRACTS ``campaign-name-is-run``: the campaign's name has +one source, the run config's ``run:``, and every campaign product path is +rooted in ``PRODUCTS_DIR``. + +The skill-level lineage checker scans ``.py`` only, and the name is bound in +the Snakefile, so this reads the workflow's sources directly. The path helpers +are lifted out of the Snakefile by name and evaluated against sentinel roots, +which tests what they RETURN rather than how they are spelled. +""" + +import re +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / "workflow" +SNAKEFILE = WORKFLOW / "Snakefile" +RULES = sorted((WORKFLOW / "rules").glob("*.smk")) +SCRIPTS = sorted((WORKFLOW / "scripts").glob("*.py")) +LAUNCHER = WORKFLOW / "bin" / "sp" + +SNAKEMAKE_SOURCES = [SNAKEFILE, *RULES] +ALL_SOURCES = [*SNAKEMAKE_SOURCES, *SCRIPTS, LAUNCHER] + +PRODUCTS = "/sentinel/products" +SCRATCH = "/sentinel/scratch" +CAMPAIGN = "campaign-sentinel" + +# Paths that hold the campaign's durable products, and whether each one +# carries the campaign name. Called with a tile/exposure ID where they take one. +PRODUCT_HELPERS = { + "final_cat": (("210.282",), False), + "final_cat_hdf5": ((), True), + "full_starcat": ((), True), + "prod_exp_dir": (("2605805",), False), + "prod_exp_manifest": (("2605805", "exp_persist"), False), + "prod_exp_tar": (("2605805",), False), +} +PRODUCT_TEMPLATES = ("PROD_TILE_DIR", "PROD_EXP_DIR") + + +def _code_lines(path): + """Source lines with comment-only lines dropped.""" + return [(n, line) for n, line in + enumerate(path.read_text().splitlines(), 1) + if not line.lstrip().startswith("#")] + + +def _snakefile_def(name): + """The text of one top-level ``def`` in the Snakefile.""" + text = SNAKEFILE.read_text() + m = re.search(rf"^def {name}\(.*?(?=^\S)", text, re.M | re.S) + assert m, f"Snakefile no longer defines {name}(); update PRODUCT_HELPERS" + return m.group(0) + + +def _snakefile_assignment(name): + m = re.search(rf"^{name}\s*=\s*(.+)$", SNAKEFILE.read_text(), re.M) + assert m, f"Snakefile no longer assigns {name}; update PRODUCT_TEMPLATES" + return m.group(1) + + +@pytest.fixture(scope="module") +def helpers(): + """The Snakefile's product-path helpers, bound to sentinel roots.""" + ns = {"Path": Path, "PRODUCTS_DIR": Path(PRODUCTS), + "RUN_DIR": Path(SCRATCH), "CAMPAIGN": CAMPAIGN} + for name in PRODUCT_HELPERS: + exec(_snakefile_def(name), ns) + for name in PRODUCT_TEMPLATES: + ns[name] = eval(_snakefile_assignment(name), ns) + return ns + + +def test_campaign_is_bound_once_from_run(): + """``CAMPAIGN`` has exactly one binding, and it reads ``config["run"]``.""" + bindings = [(p.name, n, line.strip()) for p in SNAKEMAKE_SOURCES + for n, line in _code_lines(p) + if re.match(r"\s*CAMPAIGN\s*=", line)] + assert [b[2] for b in bindings] == ['CAMPAIGN = config["run"]'], bindings + + +def test_no_second_campaign_source(): + """No source reads a ``campaign`` config key or names a campaign after a + root directory; either could disagree with ``run:``.""" + forbidden = re.compile( + r"""\[\s*['"]campaign['"]\s*\]""" + r"""|\.get\(\s*['"]campaign['"]""" + r"|\b(?:PRODUCTS_DIR|RUN_DIR|products_dir)\b[\w)\]]*" + r"(?:\.parent)*\.(?:name|stem)\b") + hits = [f"{p.relative_to(REPO_ROOT)}:{n}: {line.strip()}" + for p in ALL_SOURCES for n, line in _code_lines(p) + if forbidden.search(line)] + assert not hits, "second campaign-name source:\n" + "\n".join(hits) + + +def test_merge_rules_pass_campaign_from_campaign(): + """Every rule param named ``campaign`` is ``CAMPAIGN``, and every + ``--campaign`` on a shell line is that param.""" + params, flags = [], [] + for p in RULES: + for n, line in _code_lines(p): + m = re.match(r"\s*campaign\s*=\s*(.+?),?\s*$", line) + if m: + params.append((p.name, n, m.group(1))) + for arg in re.findall(r"--campaign\s+(\S+)", line): + flags.append((p.name, n, arg)) + assert params, "no rule passes a campaign param; update this test" + assert all(v == "CAMPAIGN" for *_, v in params), params + assert flags and all(a.strip("'\"") == "{params.campaign}" + for *_, a in flags), flags + + +@pytest.mark.parametrize("name", sorted(PRODUCT_HELPERS)) +def test_product_paths_are_rooted_in_products_dir(helpers, name): + args, named = PRODUCT_HELPERS[name] + path = str(helpers[name](*args)) + assert path.startswith(PRODUCTS + "/"), path + assert (CAMPAIGN in path) == named, path + + +@pytest.mark.parametrize("name", PRODUCT_TEMPLATES) +def test_product_templates_are_rooted_in_products_dir(helpers, name): + assert str(helpers[name]).startswith(PRODUCTS + "/"), helpers[name] + + +def _machine_outputs(): + config = yaml.safe_load((WORKFLOW / "config.yaml").read_text()) + for machine, entry in (config.get("machines") or {}).items(): + for input_type, defaults in entry.items(): + if isinstance(defaults, dict) and "outputs" in defaults: + yield f"{machine}.{input_type}", defaults["outputs"] + + +@pytest.mark.parametrize("where,outputs", list(_machine_outputs())) +def test_machine_defaults_name_the_campaign_by_run(where, outputs): + """Where a machine default sets the persistent root, the campaign in it + is ``$run``, and the index lives beneath it.""" + products = outputs.get("products_dir") + if products is None: + pytest.skip(f"{where} sets no products_dir") + assert products.rstrip("/").endswith("/$run"), products + index = outputs.get("index_db") + if index is not None: + assert index.startswith(products.rstrip("/") + "/"), (index, products) diff --git a/tests/unit/test_final_cat_merge_invariants.py b/tests/unit/test_final_cat_merge_invariants.py new file mode 100644 index 000000000..7dcfc6183 --- /dev/null +++ b/tests/unit/test_final_cat_merge_invariants.py @@ -0,0 +1,225 @@ +"""What ``final_cat_merge`` owes the catalogues it merges. + +Enforces three contracts on a small campaign of tile catalogues, run through +``workflow/scripts/merge_final_cat.py`` as the rule's shell runs it: + +* ``final-cat-param-is-exact-allow-list`` (workflow/CONTRACTS): the merged + datasets carry the param file's columns, in its order, and nothing else; +* ``read-data-raises-on-missing-column`` (create_final_cat.py): a tile short a + listed column stops the merge, naming the column; +* ``never-fit-rows-pass-through`` (merge_final_cat.py): objects ngmix never fit + reach the merged file unchanged. + +Plus the property the per-epoch families exist for: an object's slot-n tuple +(``EXP_ID_n``, ``CCD_n``, ``HSM_*_PSF_n``) is the same after the merge as in its +tile catalogue. Objects are matched on ``NUMBER``, so a merge may reorder rows +but not move one field of a row without the others. + +Failure modes each test was checked against, by editing the code under test: +a listed column dropped or an unlisted one kept (columns); one family's slots +swapped, or one family's rows reordered (per-epoch); never-fit rows dropped or +their sentinels filled (never-fit); the missing-column raise turned into a skip +(missing column). +""" + +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +from numpy.lib.recfunctions import repack_fields + +h5py = pytest.importorskip("h5py") +fits = pytest.importorskip("astropy.io.fits") + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "merge_final_cat.py" + +CAMPAIGN = "fixture-campaign" +TILES = ("210.282", "211.282", "212.283") +N_SLOTS = 3 +EPOCH_FAMILIES = ("EXP_ID", "CCD", "HSM_G1_PSF", "HSM_G2_PSF") +SCALARS = ( + ("NUMBER", "i4"), ("XWIN_WORLD", "f8"), ("NGMIX_N_EPOCH", "i4"), + ("NGMIX_MCAL_FLAGS", "i4"), ("NGMIX_G1_NOSHEAR", "f8"), + ("NGMIX_T_NOSHEAR", "f8"), +) +EPOCH_DTYPE = {"EXP_ID": "i4", "CCD": "i4", + "HSM_G1_PSF": "f8", "HSM_G2_PSF": "f8"} +# In each tile but not in the param file: the merge must leave these behind. +UNLISTED = (("MAG_UNLISTED", "f4"), ("EXP_ID_4", "i4"), ("CCD_4", "i4")) + +PARAM_LIST = [name for name, _ in SCALARS] + [ + f"{fam}_{n}" for n in range(1, N_SLOTS + 1) for fam in EPOCH_FAMILIES] + + +def _slot(fam, n): + return f"{fam}_{n}" + + +def _tile_catalogue(seed, rows=40, n_unfit=5): + """One tile's catalogue: listed columns, unlisted ones, never-fit rows.""" + rng = np.random.default_rng(seed) + columns = list(SCALARS) + [ + (_slot(fam, n), EPOCH_DTYPE[fam]) + for n in range(1, N_SLOTS + 1) for fam in EPOCH_FAMILIES + ] + list(UNLISTED) + # The catalogue's own column order is not the param file's. + columns = [columns[i] for i in rng.permutation(len(columns))] + cat = np.zeros(rows, dtype=columns) + cat["NUMBER"] = rng.permutation(rows) + 1 + cat["XWIN_WORLD"] = rng.uniform(0, 360, rows) + cat["MAG_UNLISTED"] = rng.uniform(18, 25, rows) + cat["EXP_ID_4"] = cat["CCD_4"] = -1 + n_epoch = rng.integers(1, N_SLOTS + 1, rows) + unfit = rng.choice(rows, n_unfit, replace=False) + n_epoch[unfit] = 0 + cat["NGMIX_N_EPOCH"] = n_epoch + cat["NGMIX_G1_NOSHEAR"] = np.where(n_epoch > 0, + rng.normal(0, 0.3, rows), -10.0) + cat["NGMIX_T_NOSHEAR"] = np.where(n_epoch > 0, + rng.uniform(0.1, 1, rows), 0.0) + for n in range(1, N_SLOTS + 1): + used = n_epoch >= n + # Distinct values across slots and rows, so any exchange shows. + cat[_slot("EXP_ID", n)] = np.where( + used, rng.choice(10**7, rows, replace=False), -1) + cat[_slot("CCD", n)] = np.where(used, rng.integers(0, 40, rows), -1) + cat[_slot("HSM_G1_PSF", n)] = np.where( + used, rng.normal(0, 0.05, rows), -10.0) + cat[_slot("HSM_G2_PSF", n)] = np.where( + used, rng.normal(0, 0.05, rows), -10.0) + return cat + + +def _campaign(root: Path, drop=None): + """Lay out a campaign the way the workflow does; return (argv, sources). + + ``drop`` removes one listed column from the last tile. + """ + products = root / "products" + sources = {} + for i, tile in enumerate(TILES): + cat = _tile_catalogue(seed=1000 + i) + if drop and tile == TILES[-1]: + keep = [c for c in cat.dtype.names if c != drop] + cat = repack_fields(cat[keep]) + path = products / "tiles" / tile[:2] / tile / f"final_cat-{tile}.fits" + path.parent.mkdir(parents=True) + fits.HDUList([fits.PrimaryHDU(), fits.BinTableHDU(cat)]).writeto(path) + sources[tile] = fits.getdata(path, 1) + tile_list = root / "tiles.txt" + tile_list.write_text("\n".join(TILES) + "\n") + index = root / "index.sqlite" + con = sqlite3.connect(index) + con.execute("CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, " + "n_exp INTEGER)") + con.execute("CREATE TABLE tile_exposures(tile_id TEXT, exp_id TEXT)") + con.executemany("INSERT INTO tiles VALUES (?, ?, 1)", + [(t, t.split(".")[0]) for t in TILES]) + con.executemany("INSERT INTO tile_exposures VALUES (?, ?)", + [(t, "2605805") for t in TILES]) + con.commit() + con.close() + param = root / "final_cat.param" + param.write_text("# fixture schema\nNUMBER\n\n# per-epoch\n" + + "\n".join(PARAM_LIST[1:]) + "\n") + output = products / f"final_cat_{CAMPAIGN}.hdf5" + argv = [sys.executable, str(SCRIPT), "--products-dir", str(products), + "--tile-list", str(tile_list), "--index-db", str(index), + "--output", str(output), "--campaign", CAMPAIGN, + "--param-file", str(param)] + return argv, output, sources + + +@pytest.fixture(scope="module") +def merged(tmp_path_factory): + argv, output, sources = _campaign(tmp_path_factory.mktemp("campaign")) + run = subprocess.run(argv, capture_output=True, text=True) + assert run.returncode == 0, run.stderr + with h5py.File(output, "r") as f: + group = f[f"patches/{CAMPAIGN}"] + out = {tile: group[tile][()] for tile in group} + return out, sources + + +def _by_number(arr): + return arr[np.argsort(arr["NUMBER"])] + + +def test_columns_are_exactly_the_param_list(merged): + out, _ = merged + assert sorted(out) == sorted(TILES) + for tile, data in out.items(): + assert list(data.dtype.names) == PARAM_LIST, tile + + +def test_per_epoch_tuples_survive_the_merge(merged): + out, sources = merged + for tile, data in out.items(): + got, want = _by_number(data), _by_number(sources[tile]) + assert np.array_equal(got["NUMBER"], want["NUMBER"]), tile + for n in range(1, N_SLOTS + 1): + for fam in EPOCH_FAMILIES: + col = _slot(fam, n) + assert np.array_equal(got[col], want[col]), (tile, col) + + +def test_never_fit_rows_pass_through(merged): + out, sources = merged + for tile, data in out.items(): + src = sources[tile] + assert len(data) == len(src), tile + got = _by_number(data[data["NGMIX_N_EPOCH"] == 0]) + want = _by_number(src[src["NGMIX_N_EPOCH"] == 0]) + assert len(want) > 0, "fixture lost its never-fit rows" + for col in ("NUMBER", "NGMIX_G1_NOSHEAR", "NGMIX_T_NOSHEAR", + "NGMIX_MCAL_FLAGS"): + assert np.array_equal(got[col], want[col]), (tile, col) + + +def test_a_tile_missing_a_listed_column_stops_the_merge(tmp_path): + missing = _slot("HSM_G1_PSF", N_SLOTS) + argv, output, _ = _campaign(tmp_path, drop=missing) + run = subprocess.run(argv, capture_output=True, text=True) + assert run.returncode != 0, "merge succeeded over a tile short a column" + assert missing in run.stderr + assert not output.exists() + + +def _rewrite_tile(tile_path, retype=None): + """Rewrite one tile's catalogue, optionally narrowing one column to f4.""" + cat = fits.getdata(tile_path, 1) + arr = np.array(cat) + if retype: + dtype = [(n, "f4" if n == retype else arr.dtype[n]) + for n in arr.dtype.names] + arr = arr.astype(dtype) + fits.HDUList([fits.PrimaryHDU(), fits.BinTableHDU(arr)]).writeto( + tile_path, overwrite=True) + + +def test_one_column_type_per_campaign(tmp_path): + """A tile rewritten with the same types refreshes alone (FITS byte order + is not a type change); a tile whose column changes type beside tiles that + kept the old one is refused, naming the column and both dtypes, and the + published catalogue is left as it was.""" + argv, output, _ = _campaign(tmp_path) + assert subprocess.run(argv, capture_output=True).returncode == 0 + tile = lambda t: (output.parent / "tiles" / t[:2] / t + / f"final_cat-{t}.fits") + + _rewrite_tile(tile(TILES[0])) + run = subprocess.run(argv, capture_output=True, text=True) + assert run.returncode == 0, run.stderr + assert "0 added, 1 refreshed" in run.stdout, run.stdout + + before = output.read_bytes() + _rewrite_tile(tile(TILES[-1]), retype="NGMIX_T_NOSHEAR") + run = subprocess.run(argv, capture_output=True, text=True) + assert run.returncode != 0, "a campaign with two dtypes for one column" + assert "NGMIX_T_NOSHEAR" in run.stderr + assert "float32" in run.stderr and "float64" in run.stderr, run.stderr + assert output.read_bytes() == before diff --git a/tests/unit/test_hdf5_reconcile_props.py b/tests/unit/test_hdf5_reconcile_props.py new file mode 100644 index 000000000..8ea6c9949 --- /dev/null +++ b/tests/unit/test_hdf5_reconcile_props.py @@ -0,0 +1,458 @@ +"""Property-based state machine over ``workflow/scripts/hdf5_reconcile.py``. + +The module's contract is that an hdf5 catalogue reconciled against a campaign +is a FUNCTION OF ITS INPUT SET — the same units with the same sources give the +same datasets, the same dtypes and the same count attribute, however they got +there. That is a claim about every reachable sequence of appends, refreshes and +removals, not about the three the unit tests happen to walk, so it is tested +here against a model: a random sequence of campaign edits, each followed by a +real plan/apply against a real file on disk, with the model asserted after +every step. + +The operations are the four things a campaign can do between invocations — +add a unit, change a unit's source, drop a unit, change the column set — plus +a no-op, which is the one that must leave the file's mtime alone. + +Source mtimes are set EXPLICITLY with ``os.utime`` rather than left to the +clock. ``stamp()`` is (size, mtime_ns), so a test that rewrote a file with the +same length inside one filesystem tick would silently exercise "nothing +changed" while believing it exercised a refresh. +""" + +import importlib.util +import os +import sys +from pathlib import Path + +import numpy as np +import pytest +from hypothesis import HealthCheck, settings +from hypothesis import strategies as st +from hypothesis.stateful import ( + RuleBasedStateMachine, + initialize, + invariant, + precondition, + rule, +) + +h5py = pytest.importorskip("h5py") + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" + + +def _load(name): + path = SCRIPTS / f"{name}.py" + assert path.exists(), f"{path} not found; the rules call it by path" + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(f"_{name}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +reconcile = _load("hdf5_reconcile") + +GROUP = "cat/campaign_a" +COUNT_ATTR = "n_units" +UNITS = ["u0", "u1", "u2", "u3"] +# Two column sets, so a schema change is a real change of dtype and width. +COLUMN_SETS = [("RA", "DEC", "E1"), ("RA", "DEC", "E1", "FWHM")] +# What a rebuild may cost over a from-scratch build of the same campaign: the +# metadata h5py's group copy writes for a moved dataset. Measured at ~1.4 kB +# and constant in the number of rebuilds; the allowance is generous because the +# property being defended is "does not grow with history", not an exact size. +COPY_SLACK = 8192 + + +def _array(columns, rows, seed): + rng = np.random.default_rng(seed) + dtype = [(c, " None: + np.save(path, array, allow_pickle=False) + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def _read(unit, source): + return np.load(source, allow_pickle=False) + + +def _build(output: Path, sources: dict, columns): + """Plan and apply once, exactly as the two merge rules do; return the plan.""" + units = sorted(sources.items()) + digest = reconcile.schema_digest(columns) + todo = reconcile.plan(output, GROUP, units, digest) + if todo.empty(): + return todo + reconcile.apply(output, GROUP, todo, units, _read, digest, COUNT_ATTR) + return todo + + +class ReconcileMachine(RuleBasedStateMachine): + """A campaign that changes under a catalogue that must keep up with it.""" + + @initialize() + def setup(self): + self.dir = Path( + __import__("tempfile").mkdtemp(prefix="reconcile-props-") + ) + self.output = self.dir / "cat.h5" + self.columns = COLUMN_SETS[0] + self.sources = {} # unit -> source path + self.expected = {} # unit -> array as last written + self.clock = 1_000_000_000_000_000_000 + # Compaction is only claimed of the rebuild path (a plan that removes + # or refreshes). An add-only plan copies the file and appends, so its + # layout carries whatever the previous writes left behind. + self.rebuilt = False + + def teardown(self): + __import__("shutil").rmtree(self.dir, ignore_errors=True) + + # --- the campaign's moves ------------------------------------------- + def _tick(self): + self.clock += 1_000_000_000 + return self.clock + + def _step(self, changed): + before = (self.output.stat().st_mtime_ns + if self.output.exists() else None) + todo = _build(self.output, self.sources, self.columns) + self.rebuilt = bool(todo.remove or todo.refresh) + if not changed and before is not None: + assert self.output.stat().st_mtime_ns == before, ( + "a no-op reconcile rewrote the file; mtime is a rerun trigger" + ) + + @rule(pick=st.integers(0, 2**16), rows=st.integers(1, 5), + seed=st.integers(0, 2**16)) + @precondition(lambda self: len(self.sources) < len(UNITS)) + def add_unit(self, pick, rows, seed): + free = sorted(set(UNITS) - set(self.sources)) + unit = free[pick % len(free)] + path = self.dir / f"{unit}.npy" + array = _array(self.columns, rows, seed) + _write_source(path, array, self._tick()) + self.sources[unit] = path + self.expected[unit] = array + self._step(changed=True) + + @rule(pick=st.integers(0, 2**16), rows=st.integers(1, 5), + seed=st.integers(0, 2**16), resize=st.booleans()) + @precondition(lambda self: bool(self.sources)) + def modify_source(self, pick, rows, seed, resize): + unit = sorted(self.sources)[pick % len(self.sources)] + old = self.expected[unit] + rows = rows if resize else len(old) + array = _array(self.columns, rows, seed) + _write_source(self.sources[unit], array, self._tick()) + self.expected[unit] = array + self._step(changed=True) + + @rule(pick=st.integers(0, 2**16)) + @precondition(lambda self: bool(self.sources)) + def remove_unit(self, pick): + unit = sorted(self.sources)[pick % len(self.sources)] + self.sources.pop(unit).unlink() + self.expected.pop(unit) + self._step(changed=True) + + @rule() + def change_columns(self): + """Flip to the other column set — a digest change, so every unit + refreshes.""" + columns = next(c for c in COLUMN_SETS if c != self.columns) + self.columns = columns + # A schema change is a change to how the SOURCES are read, so the + # sources are rewritten under the new column set as the campaign would. + for i, (unit, path) in enumerate(sorted(self.sources.items())): + array = _array(columns, len(self.expected[unit]), 4242 + i) + _write_source(path, array, self._tick()) + self.expected[unit] = array + self._step(changed=True) + + @rule() + def no_op(self): + self._step(changed=False) + + # --- what must be true after every step ------------------------------ + @invariant() + def file_matches_campaign(self): + if not self.expected: + return + assert self.output.exists() + with h5py.File(self.output, "r") as f: + assert set(f[GROUP]) == set(self.expected), ( + "datasets and campaign units disagree") + assert f.attrs[COUNT_ATTR] == len(self.expected) + assert (f.attrs["param_digest"] + == reconcile.schema_digest(self.columns)) + dtypes = set() + for unit, want in self.expected.items(): + got = f[GROUP][unit][...] + assert got.dtype.names == want.dtype.names + np.testing.assert_array_equal(got, want) + dtypes.add(got.dtype) + stamp = reconcile.stamp(self.sources[unit]) + assert (int(f[GROUP][unit].attrs["src_bytes"]), + int(f[GROUP][unit].attrs["src_mtime_ns"])) == stamp + assert len(dtypes) == 1, ( + "sources share a column list; datasets must share a dtype") + + @invariant() + def compact(self): + """A rebuild does not carry the old file's dead space forward. + + HDF5 never reclaims a deleted dataset's space, which is why ``apply`` + builds the tmp FRESH whenever a plan removes or refreshes anything + instead of copying and editing in place. If that path stopped firing, + a long-lived campaign would grow by one unit per refresh forever. + + The bound is a from-scratch build of the same campaign plus a fixed + allowance: moving a dataset across with h5py's group copy costs a + little more metadata than creating it from an array does, measured at + ~1.4 kB here and — see the cycle test below — independent of how many + times the file has been rebuilt. What must never hold is growth that + tracks the history. + """ + if not self.rebuilt or not self.output.exists(): + return + fresh = self.dir / "fresh.h5" + fresh.unlink(missing_ok=True) + try: + _build(fresh, self.sources, self.columns) + if not fresh.exists(): + return + assert (self.output.stat().st_size + <= fresh.stat().st_size + COPY_SLACK), ( + "a rebuilt file is carrying dead space: " + f"{self.output.stat().st_size} bytes against " + f"{fresh.stat().st_size} from scratch") + finally: + fresh.unlink(missing_ok=True) + + +ReconcileMachine.TestCase.settings = settings( + max_examples=150, + stateful_step_count=14, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) +TestReconcileMachine = ReconcileMachine.TestCase + + +def test_crash_between_tmp_and_replace_leaves_the_file_untouched(): + """A failed rename must leave the previous catalogue byte-identical. + + Not a hypothesis case: the interesting axis is the crash point, and there + is one. ``os.replace`` is made to raise where the tmp is moved into place. + """ + import shutil + import tempfile + + work = Path(tempfile.mkdtemp(prefix="reconcile-crash-")) + try: + output = work / "cat.h5" + columns = COLUMN_SETS[0] + sources = {} + for i, unit in enumerate(UNITS[:2]): + path = work / f"{unit}.npy" + _write_source(path, _array(columns, 3, i), + 1_000_000_000_000_000_000 + i) + sources[unit] = path + _build(output, sources, columns) + before = output.read_bytes() + before_mtime = output.stat().st_mtime_ns + + # A third unit arrives, and the rename fails. + path = work / "u2.npy" + _write_source(path, _array(columns, 3, 99), 1_000_000_000_000_000_099) + sources["u2"] = path + units = sorted(sources.items()) + digest = reconcile.schema_digest(columns) + todo = reconcile.plan(output, GROUP, units, digest) + assert todo.add == ["u2"] + + real_replace = Path.replace + + def boom(self, target): + raise OSError("simulated crash between write and rename") + + Path.replace = boom + try: + with pytest.raises(OSError): + reconcile.apply(output, GROUP, todo, units, _read, digest, + COUNT_ATTR) + finally: + Path.replace = real_replace + + assert output.read_bytes() == before, "the old catalogue was modified" + assert output.stat().st_mtime_ns == before_mtime + assert not (work / "cat.h5.tmp").exists(), "tmp outlived the failure" + finally: + shutil.rmtree(work, ignore_errors=True) + + +def test_repeated_refresh_does_not_grow_the_file(): + """The leak the rebuild path exists to prevent, asserted directly. + + Twelve refreshes of one unit in a two-unit campaign. If ``apply`` ever + copied the file and edited it in place, each would strand the previous + dataset's bytes and the size would climb monotonically. + """ + import shutil + import tempfile + + work = Path(tempfile.mkdtemp(prefix="reconcile-growth-")) + try: + output = work / "cat.h5" + columns = COLUMN_SETS[0] + sources = {} + for i, unit in enumerate(("u0", "u1")): + path = work / f"{unit}.npy" + _write_source(path, _array(columns, 4, i), 10**18 + i) + sources[unit] = path + _build(output, sources, columns) + + sizes = [] + for k in range(12): + _write_source(sources["u0"], _array(columns, 4, 100 + k), + 10**18 + 100 + k) + todo = _build(output, sources, columns) + assert todo.refresh == ["u0"], todo.describe() + sizes.append(output.stat().st_size) + assert len(set(sizes)) == 1, f"file size drifted across refreshes: {sizes}" + finally: + shutil.rmtree(work, ignore_errors=True) + + +def test_a_type_change_in_the_reader_refreshes_every_unit(tmp_path): + """The column list is unchanged, so the digest is too; a reader that now + yields another dtype must still leave one dtype in the file, by re-reading + the units it would otherwise keep.""" + columns = COLUMN_SETS[0] + sources = {} + for i, unit in enumerate(UNITS[:2]): + sources[unit] = tmp_path / f"{unit}.npy" + _write_source(sources[unit], _array(columns, 3, i), 10**18 + i) + _build(tmp_path / "cat.h5", sources, columns) + + sources["u2"] = tmp_path / "u2.npy" + _write_source(sources["u2"], _array(columns, 3, 2), 10**18 + 2) + units = sorted(sources.items()) + digest = reconcile.schema_digest(columns) + todo = reconcile.plan(tmp_path / "cat.h5", GROUP, units, digest) + assert (todo.add, todo.refresh) == (["u2"], []) + + narrow = lambda unit, source: _read(unit, source).astype( + [(c, " str: + return hashlib.md5(path.read_bytes()).hexdigest() + + +def _members(tar: Path) -> list: + with tarfile.open(tar) as tf: + return [ti.name for ti in tf.getmembers() if ti.isfile()] + + +class Store: + """One exposure's scratch store, its destination, and how to pack it.""" + + def __init__(self): + self.root = Path(tempfile.mkdtemp(prefix="persist-exp-props-")) + self.exp_dir = self.root / "exp" / EXP + self.dest = self.root / "products" / "psf" + self.manifest = self.root / "products" / "manifests" / f"{EXP}.json" + self.tar = self.dest / f"{EXP}.tar" + + def close(self): + shutil.rmtree(self.root, ignore_errors=True) + + def path_of(self, product: str) -> Path: + module, sub, name = LAYOUT[product] + base = (self.exp_dir / "output" / persist.RUN_NAME + / f"run_sp_{module}" / "output") + return (base / sub / name) if sub else (base / name) + + def write(self, product: str, payload: bytes) -> None: + path = self.path_of(product) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + + def drop(self, product: str) -> None: + self.path_of(product).unlink(missing_ok=True) + + def pack(self, keep: list) -> int: + """Run the script's ``main`` as the rule does. 0 on success.""" + argv = ["persist_exp.py", "--exp-dir", str(self.exp_dir), + "--exp", EXP, "--dest", str(self.dest), + "--manifest", str(self.manifest)] + for entry in keep: + argv += ["--pattern", entry] + old = sys.argv + sys.argv = argv + try: + persist.main() + return 0 + except SystemExit as exc: + return 1 if exc.code not in (0, None) else 0 + finally: + sys.argv = old + + +class PersistExpMachine(RuleBasedStateMachine): + """A store that gains and loses products under a keep list that changes.""" + + @initialize() + def setup(self): + self.store = Store() + self.present = set() + self.prior_members = [] # members of the last tar written + self.prior_labels = {} # member -> product recorded for it + + def teardown(self): + self.store.close() + + # --- the store and the config move ---------------------------------- + @rule(product=st.sampled_from(sorted(LAYOUT)), size=st.integers(1, 64)) + def add_product(self, product, size): + self.store.write(product, bytes([len(product) % 251]) * size) + self.present.add(product) + + @rule(product=st.sampled_from(sorted(LAYOUT))) + def drop_product(self, product): + self.store.drop(product) + self.present.discard(product) + + @rule(keep=st.lists(st.sampled_from(ENTRIES), max_size=4, unique=True)) + def pack(self, keep): + self._pack_and_check(keep) + + @rule(keep=st.lists(st.sampled_from(ENTRIES), max_size=3, unique=True)) + def pack_with_unknown_product(self, keep): + """An unknown product name is refused before anything is written.""" + before = (_md5(self.store.tar) if self.store.tar.exists() else None) + code = self.store.pack(keep + [UNKNOWN]) + assert code != 0, "an unknown product name was accepted" + after = (_md5(self.store.tar) if self.store.tar.exists() else None) + assert after == before, "a refused keep list still touched the tar" + + @rule() + @precondition(lambda self: ALWAYS in self.present) + def pack_twice_unchanged(self): + """A rerun over an unchanged store must not move a single byte.""" + keep = sorted(OPTIONAL)[:2] + self._pack_and_check(keep) + tar_md5, man_md5 = _md5(self.store.tar), _md5(self.store.manifest) + tar_mtime = self.store.tar.stat().st_mtime_ns + man_mtime = self.store.manifest.stat().st_mtime_ns + assert self.store.pack(keep) == 0 + assert _md5(self.store.tar) == tar_md5, "the tar is not byte-stable" + assert _md5(self.store.manifest) == man_md5, "the manifest is not byte-stable" + assert self.store.tar.stat().st_mtime_ns == tar_mtime, ( + "an unchanged rerun rewrote the tar; mtime is a rerun trigger") + assert self.store.manifest.stat().st_mtime_ns == man_mtime, ( + "an unchanged rerun rewrote the manifest") + + # --- what a pack must leave behind ----------------------------------- + def _pack_and_check(self, keep): + had_tar = self.store.tar.exists() + tar_before = _md5(self.store.tar) if had_tar else None + man_before = (_md5(self.store.manifest) + if self.store.manifest.exists() else None) + code = self.store.pack(keep) + + if ALWAYS not in self.present: + # The star catalogue's input is not optional: the job fails and + # nothing downstream may be told the store is safe to reclaim. + assert code != 0, ( + f"{ALWAYS} is missing and the pack still succeeded") + assert (_md5(self.store.tar) if self.store.tar.exists() + else None) == tar_before, "a failed pack touched the tar" + assert (_md5(self.store.manifest) + if self.store.manifest.exists() + else None) == man_before, ( + "a failed pack wrote a manifest; clean_exposure would take " + "that as permission to delete the store") + return + + assert code == 0, f"pack failed with {ALWAYS} present and keep={keep}" + assert self.store.tar.exists() and self.store.manifest.exists() + members = _members(self.store.tar) + assert len(members) == len(set(members)), ( + f"duplicate member names in the tar: {members}") + + # ADDITIVE: an existing tar is a floor. + assert set(members) >= set(self.prior_members), ( + "members vanished from the tar: " + f"{sorted(set(self.prior_members) - set(members))}") + + body = json.loads(self.store.manifest.read_text()) + listed = {f["name"] for f in body["files"]} + assert listed == set(members), ( + "manifest and tar disagree about what was packed: " + f"{sorted(listed ^ set(members))}") + assert body["n_files"] == len(members) + assert body["unit"] == EXP and body["status"] == "complete" + + entries = [ALWAYS] + [e for e in keep if e != ALWAYS] + assert body["products"] == entries + for f in body["files"]: + if f["src"] is None: # carried from the previous tar + assert f["name"] in self.prior_members + assert f["product"] == self.prior_labels.get(f["name"], "?") + continue + assert f["product"] in entries, ( + f"{f['name']} labelled {f['product']!r}, not in the keep list") + assert fnmatch.fnmatch(f["name"], persist.resolve(f["product"])), ( + f"{f['name']} does not match {f['product']!r}'s glob") + assert Path(f["src"]).exists() + assert f["bytes"] == Path(f["src"]).stat().st_size + + # Every present product the keep list asks for is in there. + for entry in entries: + glob = persist.resolve(entry) + for product in self.present: + if fnmatch.fnmatch(LAYOUT[product][2], glob): + assert LAYOUT[product][2] in listed, ( + f"{product} matched {entry!r} but was not packed") + + self.prior_members = members + self.prior_labels = {f["name"]: f["product"] for f in body["files"]} + + +PersistExpMachine.TestCase.settings = settings( + max_examples=120, + stateful_step_count=12, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) +TestPersistExpMachine = PersistExpMachine.TestCase + + +@pytest.fixture() +def store(): + s = Store() + yield s + s.close() + + +def _seed(store, products=(ALWAYS,)): + for i, product in enumerate(products): + store.write(product, bytes([i + 1]) * (16 + i)) + + +def test_corrupt_existing_tar_is_refused_and_left_alone(store): + """A tar that cannot be read may still hold the only copy of something.""" + _seed(store, (ALWAYS, "psf_model")) + assert store.pack(["psf_model"]) == 0 + store.tar.write_bytes(b"not a tar at all, not even close" * 8) + corrupt = store.tar.read_bytes() + man_before = _md5(store.manifest) + + assert store.pack(["psf_model"]) != 0, "a corrupt tar was overwritten" + assert store.tar.read_bytes() == corrupt, "the corrupt tar was modified" + assert _md5(store.manifest) == man_before, ( + "a manifest was written over a tar that could not be read") + assert not store.tar.with_name(store.tar.name + ".tmp").exists() + + +def test_two_sources_with_one_member_name_is_fatal(store): + """Members are flat, so a real name clash would silently overwrite.""" + _seed(store, (ALWAYS,)) + # The same file name under a second module output dir. + clash = (store.exp_dir / "output" / persist.RUN_NAME / "run_sp_setools" + / "output" / "new_cat" / LAYOUT[ALWAYS][2]) + clash.parent.mkdir(parents=True, exist_ok=True) + clash.write_bytes(b"a different file with the same name") + + assert store.pack([]) != 0, "two different sources shared a member name" + assert not store.manifest.exists() + assert not store.tar.exists() + + +def test_shrinking_the_keep_list_cannot_delete_a_product(store): + """The property the additive rule exists for, stated end to end.""" + _seed(store, (ALWAYS, "psf_model", "star_train")) + assert store.pack(["psf_model", "star_train"]) == 0 + wide = set(_members(store.tar)) + assert LAYOUT["psf_model"][2] in wide + + # The campaign changes its mind, and the scratch store is gone. + for product in ("psf_model", "star_train"): + store.drop(product) + assert store.pack([]) == 0 + assert set(_members(store.tar)) == wide, ( + "shrinking persist_exp: deleted products from the backed-up tar") + body = json.loads(store.manifest.read_text()) + carried = {f["name"] for f in body["files"] if f["src"] is None} + assert LAYOUT["psf_model"][2] in carried + assert {f["name"]: f["product"] for f in body["files"]}[ + LAYOUT["psf_model"][2]] == "psf_model", ( + "a carried member lost the product label the old manifest had") + + +@settings(max_examples=80, deadline=None, + suppress_health_check=[HealthCheck.too_slow]) +@given(st.lists(st.sampled_from( + [ALWAYS, "*.fits", "validation_psf-*.fits", "psf_validation", + "star_*", "*.psf", "psf_model", "star_train"]), + min_size=1, max_size=5)) +def test_overlapping_patterns_never_fail(keep): + """Two patterns matching one file is one file, not a name collision. + + Overlap is ordinary — ``validation_psf-*.fits`` beside ``*.fits`` is a + perfectly reasonable way to say "the validation catalogues, and everything + else FITS while we are here" — and treating the second match as a clash + once failed every exposure in a campaign. + + A fresh store per example, because the additive rule makes packing + stateful and this property is about ONE pack. + """ + s = Store() + try: + _seed(s, tuple(LAYOUT)) + assert s.pack(keep) == 0, f"overlapping keep list failed: {keep}" + members = _members(s.tar) + assert len(members) == len(set(members)), members + # Every product present matched something, so all seven are packed. + assert set(members) == {name for _, _, name in LAYOUT.values()} & set( + members) + finally: + s.close() + + +def test_same_size_new_bytes_changes_the_manifest(store): + """The manifest is the DAG edge star_cat_merge waits on, so a refit that + keeps every member's size must still change it; a rerun over the same + bytes must not.""" + store.write(ALWAYS, b"\x01" * 32) + assert store.pack([]) == 0 + before = store.manifest.read_bytes() + assert store.pack([]) == 0 + assert store.manifest.read_bytes() == before, "a no-op rerun moved it" + + store.write(ALWAYS, b"\x02" * 32) + assert store.pack([]) == 0 + assert store.manifest.read_bytes() != before + (entry,) = json.loads(store.manifest.read_text())["files"] + assert entry["sha256"] == hashlib.sha256(b"\x02" * 32).hexdigest() diff --git a/tests/unit/test_run_config.py b/tests/unit/test_run_config.py new file mode 100644 index 000000000..3290f9fc3 --- /dev/null +++ b/tests/unit/test_run_config.py @@ -0,0 +1,98 @@ +"""``workflow/scripts/run_config.py``: `run:` is a required key, and every +machine-key path expands fully or is reported. + +`run:` names the campaign's merged catalogues (``final_cat_.hdf5``, +``full_starcat_.hdf5``). ``unresolved()`` already reports a ``$run`` left +unexpanded in a path; these tests pin that a run config whose paths never +mention ``$run`` is refused too, and that the shipped ``config.yaml`` leaves the +name to the run config. A ``$base_dir`` whose value holds ``$run`` expands +through both, and an optional path left holding a ``$`` is reported. +""" + +import importlib.util +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "workflow" / "scripts" / "run_config.py" +CONFIG_YAML = REPO_ROOT / "workflow" / "config.yaml" + +_spec = importlib.util.spec_from_file_location("_run_config", SCRIPT) +run_config = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(run_config) + +# Every other REQUIRED key, as literal paths with no `$run` in them. +LITERAL = { + "tile_list": "/data/tiles.txt", + "inputs": {"tiles": "/data/tiles", "exposures": "/data/exp"}, + "outputs": {"run_dir": "/scratch/run", "index_db": "/data/index.sqlite"}, +} + + +def test_run_is_required(): + assert "run" in run_config.REQUIRED + + +def test_literal_paths_without_run_are_refused(): + assert run_config.unresolved(dict(LITERAL)) == ["run"] + + +def test_literal_paths_with_run_resolve(): + assert run_config.unresolved({**LITERAL, "run": "smk-g6"}) == [] + + +def test_shipped_config_leaves_run_to_the_run_config(tmp_path, monkeypatch): + monkeypatch.setenv("SP_PROFILE", "nibi") + assert "run" not in (yaml.safe_load(CONFIG_YAML.read_text()) or {}) + + no_run = tmp_path / "no_run.yaml" + no_run.write_text(yaml.safe_dump(LITERAL)) + assert "run" in run_config.unresolved( + run_config.load(CONFIG_YAML, no_run)) + + with_run = tmp_path / "with_run.yaml" + with_run.write_text(yaml.safe_dump({"run": "smk-test"})) + cfg = run_config.load(CONFIG_YAML, with_run) + assert run_config.unresolved(cfg) == [] + assert cfg["outputs"]["run_dir"].endswith("/smk-test") + + +def _machine_config(base_dir, **over): + return {"run": "smk-g6", "machine": "m", "input_type": "data", + "machines": {"m": {"base_dir": base_dir, "data": { + "tile_list": "$base_dir/tiles.txt", + "inputs": {"tiles": "$base_dir/tiles", + "exposures": "$base_dir/exp"}, + "outputs": {"run_dir": "$base_dir/run", + "index_db": "$base_dir/index.sqlite"}}}}, + **over} + + +def test_base_dir_holding_run_expands_fully(): + cfg = run_config.apply_machine_defaults(_machine_config("/b/$run")) + assert cfg["outputs"]["run_dir"] == "/b/smk-g6/run" + assert run_config.unresolved(cfg) == [] + + +def test_run_config_shorthands_nest_and_run_is_written_back(): + """Top-level scalars are variables, ${name} delimits, nesting resolves.""" + cfg = run_config.apply_machine_defaults(_machine_config( + "/b", shear="1p2z", grid="grid_2", run="${shear}_${grid}", + outputs={"run_dir": "/o/$run/scratch"})) + assert cfg["run"] == "1p2z_grid_2" + assert cfg["outputs"]["run_dir"] == "/o/1p2z_grid_2/scratch" + assert run_config.unresolved(cfg) == [] + + +def test_run_holding_an_unknown_variable_is_reported(): + cfg = run_config.apply_machine_defaults(_machine_config("/b", run="$nope")) + assert "run" in run_config.unresolved(cfg) + + +def test_optional_path_with_an_unknown_variable_is_reported(): + cfg = run_config.apply_machine_defaults(_machine_config( + "/b", outputs={"products_dir": "/p/$nope/products"}, + inputs={"masks": "/m/$nope"}, container="/c/$nope.sif")) + assert set(run_config.unresolved(cfg)) == { + "outputs.products_dir", "inputs.masks", "container"} diff --git a/tests/unit/test_star_cat_columns.py b/tests/unit/test_star_cat_columns.py new file mode 100644 index 000000000..f1de09a82 --- /dev/null +++ b/tests/unit/test_star_cat_columns.py @@ -0,0 +1,95 @@ +"""The star catalogue's 22 columns are defined twice, and must not drift. + +Two writers emit a full_starcat, for two consumers that have to agree about it: + + * ``MergeStarCatPSFEX`` (``src/shapepipe/modules/merge_starcat_package``), + which the ``merge_starcat`` MODULE RUNNER calls, writing the flat FITS table + sp_validation opens today; + * ``workflow/scripts/merge_star_cat.py``, the Snakemake workflow's + ``star_cat_merge`` rule, writing the per-exposure hdf5 that replaces it + (CosmoStat/sp_validation#340 moves the readers). + +They were one definition until the workflow stopped calling the module class: +the rule reads validation_psf members out of the per-exposure tars, keeps their +native dtypes and reconciles its output, none of which the class does or should +do. Two implementations is the right answer for the behaviour; two COLUMN LISTS +is not, and nothing else would notice them diverging — a column added to one +writer would simply be absent from the other's product, discovered by whoever +next tried to compute rho statistics from the wrong one. + +Hence this module, which asserts the one thing they must share. It does NOT +assert the dtypes: the whole point of the hdf5 writer is that they differ (the +FITS one widens every float to 1D). Only the names, and their order. + +Deliberately import-light on the workflow side: merge_star_cat.py pulls in h5py +and astropy, which the class does too, so a container-free run is not on offer +here and is not worth contorting for. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" +SCRIPT = SCRIPTS / "merge_star_cat.py" + + +def _load_workflow_merge(): + """Import the rule's script by path — ``workflow/scripts`` is not a package. + + Its own imports (build_index, hdf5_reconcile, persist_exp) are siblings it + reaches through ``sys.path[0]``, which is how the rule invokes it, so the + directory goes on the path here too. + """ + assert SCRIPT.exists(), f"{SCRIPT} not found; the rule calls it by path" + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location("_merge_star_cat", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +@pytest.fixture(scope="module") +def writers(): + """The two column lists, each in the order its writer emits them.""" + h5py = pytest.importorskip("h5py") # noqa: F841 - workflow dep + pytest.importorskip("astropy") + workflow = _load_workflow_merge() + from shapepipe.modules.merge_starcat_package.merge_starcat import ( + MergeStarCatPSFEX, + ) + # The class carries (output name, source column) pairs plus its optional + # set and appends CCD_NB last; the script carries output names throughout. + module_columns = ( + tuple(out for out, _ in MergeStarCatPSFEX._COLUMNS) + + tuple(out for out, _ in MergeStarCatPSFEX._OPTIONAL) + + ("CCD_NB",) + ) + return module_columns, tuple(workflow.ALL_COLUMNS) + + +def test_column_names_and_order_agree(writers): + """Same names, same order — the schema both products promise.""" + module_columns, workflow_columns = writers + assert workflow_columns == module_columns + + +def test_twenty_two_columns(writers): + """The count is itself the documented contract (README).""" + module_columns, workflow_columns = writers + assert len(module_columns) == 22 + assert len(workflow_columns) == 22 + + +def test_ccd_nb_is_last(writers): + """CCD_NB is appended per input file rather than read from one, in both.""" + module_columns, workflow_columns = writers + assert module_columns[-1] == "CCD_NB" + assert workflow_columns[-1] == "CCD_NB" diff --git a/tests/unit/test_star_cat_refresh.py b/tests/unit/test_star_cat_refresh.py new file mode 100644 index 000000000..2e5e02461 --- /dev/null +++ b/tests/unit/test_star_cat_refresh.py @@ -0,0 +1,94 @@ +"""A PSF refit reaches the campaign star catalogue. + +Real ``persist_exp.py`` and ``merge_star_cat.py``, run as the rules run them. +A refit that changes a value but no size rewrites the exposure's tar; its +manifest (the edge ``star_cat_merge`` waits on) must change with it, and the +next merge must refresh that exposure's dataset. +""" + +import importlib.util +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import h5py +import numpy as np +from astropy.io import fits + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "workflow" / "scripts" +EXP = "2605805" + + +def _load(name): + sys.path.insert(0, str(SCRIPTS)) + try: + spec = importlib.util.spec_from_file_location(f"_{name}", + SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + finally: + sys.path.remove(str(SCRIPTS)) + return module + + +merge = _load("merge_star_cat") +persist = _load("persist_exp") + + +def _run(name, *args): + run = subprocess.run([sys.executable, str(SCRIPTS / f"{name}.py"), + *map(str, args)], capture_output=True, text=True) + assert run.returncode == 0, run.stderr + return run.stdout + + +def _validation(path, value): + """A PSFEx-shaped validation table: the rows live in HDU 2.""" + cols = [fits.Column(name=c, format="I" if "FLAG" in c else "E", + array=[value, value]) for c in merge.COLUMNS] + path.parent.mkdir(parents=True, exist_ok=True) + fits.HDUList([fits.PrimaryHDU(), fits.ImageHDU(), + fits.BinTableHDU.from_columns(cols)]).writeto( + path, overwrite=True) + + +def test_refit_with_equal_sizes_refreshes_the_exposure(tmp_path): + store = tmp_path / "scratch" / EXP + src = (store / "output" / persist.RUN_NAME / "psfex_interp_runner" + / "output" / f"validation_psf-{EXP}-3.fits") + dest = tmp_path / "exp" / EXP[:2] / EXP / "psf" + manifest = dest.parent / "manifests" / "exp_persist.json" + persist_args = ("--exp-dir", store, "--exp", EXP, "--dest", dest, + "--manifest", manifest) + + tiles = tmp_path / "tiles.txt" + tiles.write_text("210.282\n") + db = tmp_path / "index.sqlite" + with sqlite3.connect(db) as con: + con.execute("CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, " + "ra_dir TEXT, n_exp INTEGER)") + con.execute("CREATE TABLE tile_exposures(tile_id TEXT, exp_id TEXT)") + con.execute("INSERT INTO tiles VALUES ('210.282', '210', 1)") + con.execute("INSERT INTO tile_exposures VALUES ('210.282', ?)", (EXP,)) + out = tmp_path / "stars.h5" + merge_args = ("--products-dir", tmp_path, "--tile-list", tiles, + "--index-db", db, "--output", out, "--campaign", "t") + + _validation(src, 0.25) + _run("persist_exp", *persist_args) + _run("merge_star_cat", *merge_args) + size, before = src.stat().st_size, manifest.read_bytes() + + _validation(src, 0.75) + assert src.stat().st_size == size, "fixture must keep the size" + _run("persist_exp", *persist_args) + assert manifest.read_bytes() != before, ( + "the tar changed but the manifest star_cat_merge waits on did not") + + assert "1 refreshed" in _run("merge_star_cat", *merge_args) + with h5py.File(out) as f: + rows = f[f"exposures/{EXP}"][:] + np.testing.assert_allclose(rows["X"], [0.75, 0.75]) diff --git a/tests/workflow/README.md b/tests/workflow/README.md new file mode 100644 index 000000000..062b4ce49 --- /dev/null +++ b/tests/workflow/README.md @@ -0,0 +1,81 @@ +# Workflow DAG checks + +These checks need snakemake, which is a host tool and not in the image. Run them +in the development container with snakemake installed on top (a writable sandbox, +or a throwaway container), at the host pin range from `workflow/README.md`: + +```bash +uv pip install 'snakemake>=9,<10' +python -m pytest tests/workflow -o addopts='' -q -p no:cacheprovider +``` + +Where snakemake is absent, the root `conftest.py` leaves this directory out of +collection, so the in-image suite stays green. The image-build CI runs it as its +own step, `Test — workflow DAG (snakemake)`, which installs snakemake first. +These tests check planning, not job execution, container validity, or SLURM group execution. +They need no survey files, cluster access, or nested Apptainer process. + +## Campaign and API + +`harness.Campaign` writes run YAML, a tile list, exposure lists, and empty prepared-stage manifests under `tmp_path`. +Two ready tiles use two exposures, one of them shared; the declared list also contains a duplicate and an unready tile. +An out-of-scope consumer has finished its vignets, and another consumer is explicitly ignored. +`build_index.build()` seeds those two consumers in the SQLite history; the Snakefile's compute parse builds the current batch's index from the exposure lists. +A prepare parse does not build the index, so no prepare invocation is necessary. + +The run configuration exercises `machine: candide` and `$base_dir`/`$run` expansion for both input types. +Scratch and persistent roots differ, and the products directory's basename deliberately differs from `run:`. +The three modes are `data+psfex`, `data+mccd`, and `image_sims+fake`. +`final_cat_merge` is present in all three: it merges galaxy catalogues regardless of the PSF source. +Only `exp_persist` and `star_cat_merge` disappear for fake PSFs. + +`resolve()` uses the same `SP_PHASE`, `SP_PROFILE`, `SP_RUN_CONFIG`, image selection, and state-directory conventions as `workflow/bin/sp`. +It isolates the environment, source cache, bare script imports, and Snakemake's shared global namespace. +An empty image sentinel satisfies parse-time image resolution without deploying software. +The candide profile supplies the resource defaults, resource overrides, Apptainer arguments, and rerun triggers; no executor starts. + +The API sequence is `SnakemakeApi()` → `api.workflow(...)` → `workflow_api.dag(DAGSettings(targets={"all"}, ...))` → `dag_api.printdag()`. +`print_dag_as="dot"` matches Snakemake 9's renderer, which compares the CLI string rather than its enum default. +The resolved jobs come from `workflow_api._workflow.dag.jobs`; this last access is private because the public API exposes graph printing but not Job objects. +`ResolvedDAG` exposes the active rule set, declared rule set, namespace, and `jobs_for(rule)` for input/output, wildcard, params, and rendered-shell inspection. +Jobs retain the rules' thread counts rather than a local `--cores` cap. +The API context remains open while tests inspect jobs and closes before the fixture restores process state. + +## Campaign-boundary pin + +`params_pin.json` pins SHA-256 digests of: + +- `unit_pre()` rendered for every stage; +- every rule's shell template; +- every resolved job's `params.pre` and formatted shell, including all ngmix chunks. + +Rules without `params.pre` use null; aggregation-only rules have no shell. +The pin includes per-stage and per-rule digests to identify which strings change. +Fixture and checkout roots become `` and `` before hashing; two independently located campaigns must give the same digest. +The normalized strings are also written to the fixture's `params_rendered.json` for inspection. +Script fingerprints and non-pre params that do not appear in a shell are outside this pin's scope. +Separate checks require `params` in both profiles' rerun triggers. + +A pin change requires a campaign boundary on a fresh root. +Review the rendered strings, then regenerate and commit the pin with the intentional workflow change: + +```bash +python -m pytest tests/workflow/test_params_pin.py --update-params-pin \ + -o addopts='' -q -p no:cacheprovider +``` + +## Failure modes and mutation probes + +Each check has a mutation that must make it fail. +Apply mutations only to a disposable checkout, run the named test without `--update-params-pin`, and require assertion failures rather than collection/setup errors. + +| Check | Mutation probes | +|---|---| +| `test_rule_set_matches_input_mode` | Invert `PERSISTS_PSF`; omit `star_cat_targets()`; rename `final_cat_merge` to `merge_final_cats`. | +| `test_clean_exposure_waits_on_persist_iff_psf` | Drop the persist edge; make it unconditional under fake PSFs; drop vignets consumers; remove the in-scope consumer filter. | +| `test_final_cat_merge_reads_every_ready_tile` | Drop one ready tile; append an out-of-scope tile. | +| `test_products_use_products_dir_and_run_name` | Rename either merged catalogue or the persist manifest; route products to scratch; derive `CAMPAIGN` from the products directory's basename. | +| `test_missing_run_fails_during_parse` | Remove `run` from `run_config.REQUIRED`; literal paths must still receive the required-key diagnostic, not a later `KeyError`. | +| `test_unit_pre_changes_at_campaign_boundary` | Append a line to `unit_pre`; change one rule's `params.pre`; change one shell; change a rendered thread count. | +| `test_params_pin_ignores_fixture_root` | Remove fixture-root normalization. | +| `test_params_is_a_rerun_trigger_under_both_profiles` | Remove `params` from candide or nibi's `rerun-triggers`. | diff --git a/tests/workflow/__init__.py b/tests/workflow/__init__.py new file mode 100644 index 000000000..5ebccbe7c --- /dev/null +++ b/tests/workflow/__init__.py @@ -0,0 +1 @@ +"""Planning-only checks of the ShapePipe Snakemake workflow.""" diff --git a/tests/workflow/conftest.py b/tests/workflow/conftest.py new file mode 100644 index 000000000..134a9c62a --- /dev/null +++ b/tests/workflow/conftest.py @@ -0,0 +1,45 @@ +"""Isolated campaign fixtures for planning-only Snakemake tests.""" + +import pytest + +from tests.workflow.harness import MODES, Campaign, load_profile, resolve + + +def pytest_addoption(parser): + """Expose the explicit campaign-boundary pin update switch.""" + parser.addoption( + "--update-params-pin", action="store_true", default=False, + help="Update the reviewed params.pre/shell pin at a campaign boundary", + ) + + +@pytest.fixture(params=MODES, ids=[f"{mode}+{psf}" for mode, psf in MODES]) +def campaign(request, tmp_path): + """Create a disposable campaign for each supported input/PSF pair.""" + return Campaign(tmp_path / "campaign", *request.param) + + +@pytest.fixture +def resolve_dag(monkeypatch): + """Expose the resolver so tests can also assert parse-time failures.""" + return lambda campaign: resolve(campaign, monkeypatch) + + +@pytest.fixture +def dag(campaign, resolve_dag): + """Keep the API and its jobs alive for the duration of one check.""" + with resolve_dag(campaign) as resolved: + yield resolved + + +@pytest.fixture +def psfex_dag(tmp_path, resolve_dag): + """Resolve the canonical data+psfex campaign for the prologue pin.""" + with resolve_dag(Campaign(tmp_path / "campaign", "data", "psfex")) as dag: + yield dag + + +@pytest.fixture(params=["candide", "nibi"]) +def profile(request): + """Read each profile's actual rerun-trigger policy.""" + return load_profile(request.param) diff --git a/tests/workflow/harness.py b/tests/workflow/harness.py new file mode 100644 index 000000000..e3e487b28 --- /dev/null +++ b/tests/workflow/harness.py @@ -0,0 +1,269 @@ +"""Build isolated campaigns and inspect Snakemake's resolved jobs. + +No executor runs: the API's ``printdag`` operation resolves input functions +and job wildcards, including the compute parse's real SQLite index build. +Only the final access to ``WorkflowApi._workflow`` is private; Snakemake's +public API exposes graph printing but not the resolved Job objects. +""" + +import io +import os +import sys +from contextlib import contextmanager, redirect_stdout +from dataclasses import dataclass +from pathlib import Path + +import yaml +from snakemake.api import SnakemakeApi +from snakemake.settings.enums import RerunTrigger +from snakemake.settings.types import ( + DAGSettings, + DeploymentSettings, + ResourceSettings, + WorkflowSettings, +) +from snakemake_interface_executor_plugins.settings import DeploymentMethod +from workflow.scripts import build_index + +REPO = Path(__file__).resolve().parents[2] +MODES = [("data", "psfex"), ("image_sims", "fake")] +# psf_model=mccd is refused at parse time (refuse_unpersistable_psf), so it +# has no DAG to resolve; test_mccd_is_refused_during_parse pins the refusal. + + +@dataclass +class Campaign: + """A campaign with two ready tiles and campaign-wide consumer edges.""" + + root: Path + input_type: str + psf_model: str + name: str = "dag-campaign" + + def __post_init__(self): + """Write exposure lists, prepared manifests, and campaign history.""" + self.root.mkdir(parents=True, exist_ok=True) + self.ready = { + "123.456": ("2243881", "2243882"), + "124.456": ("2243882",), + } + self.exposures = ("2243881", "2243882") + self.unready = "125.456" + self.outside = "126.456" + self.ignored = "127.456" + self.run_dir = self.root / "scratch" / self.name + # The basename deliberately differs from run: to catch name sniffing. + self.products_dir = self.root / "persistent" / self.name / "products" + self.index_db = self.products_dir / "index" / "run_index.sqlite" + self.tile_list = self.root / "tiles.txt" + self.config_path = self.root / "run.yaml" + self.state_dir = self.root / "state" + self.image = self.root / "planning-only.sif" + # Image resolution checks existence; DAG inspection never opens it. + self.image.touch() + self.state_dir.mkdir() + self.tile_list.write_text("\n".join([ + *self.ready, self.unready, next(iter(self.ready)), "", + ])) + for tile, exposures in { + **self.ready, self.outside: ("2243881",), + self.ignored: ("2243882",), + }.items(): + path = build_index.exp_list_path(self.run_dir, tile) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(f"{exp}p\n" for exp in exposures)) + for tile in self.ready: + for stage in ("tile_get_images", "tile_uncompress", + "tile_find_exposures"): + self._manifest(tile, stage) + self._manifest(self.outside, "tile_vignets") + # Seed out-of-scope history; the compute parse indexes ready tiles. + build_index.build( + [self.outside, self.ignored], self.run_dir, self.index_db, + ) + machine_defaults = { + "tile_list": "$base_dir/tiles.txt", + "retrieve": "symlink", + "container": str(self.image), + "inputs": { + "tiles": "$base_dir/inputs/$run/tiles", + "exposures": "$base_dir/inputs/$run/exposures", + }, + "outputs": { + "run_dir": "$base_dir/scratch/$run", + "products_dir": "$base_dir/persistent/$run/products", + "index_db": ( + "$base_dir/persistent/$run/products/index/run_index.sqlite" + ), + }, + } + self.config = { + "run": self.name, + "machine": "candide", + "input_type": self.input_type, + "psf_model": self.psf_model, + "psf_dict": str(self.root / "psf_dict.pickle") + if self.psf_model == "fake" else "", + "clean": True, + "clean_tiles": True, + "clean_ignore_tiles": [self.ignored], + "machines": {"candide": { + "base_dir": str(self.root), + "data": machine_defaults, + "image_sims": machine_defaults, + }}, + } + self.write_config() + + def _manifest(self, tile, stage): + path = self.tile_manifest(tile, stage) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n") + + def write_config(self): + """Write the fixture's run configuration.""" + self.config_path.write_text(yaml.safe_dump(self.config)) + + def omit_run(self): + """Omit run: while keeping every path explicit and resolvable.""" + self.config.pop("run") + for mode in ("data", "image_sims"): + defaults = self.config["machines"]["candide"][mode] + # Resolve only fixture templates, not the production config reader. + text = yaml.safe_dump(defaults) + text = text.replace("$base_dir", str(self.root)) + text = text.replace("$run", self.name) + self.config["machines"]["candide"][mode] = yaml.safe_load(text) + self.write_config() + + def tile_manifest(self, tile, stage): + """Return an independently specified scratch manifest path.""" + return (self.run_dir / "tiles" / tile[:2] / tile + / "manifests" / f"{stage}.json") + + def final_cat(self, tile): + """Return the expected persistent catalogue for one tile.""" + return (self.products_dir / "tiles" / tile[:2] / tile + / f"final_cat-{tile}.fits") + + def persist_manifest(self, exp): + """Return the expected persistent manifest for one exposure.""" + return (self.products_dir / "exp" / exp[:2] / exp + / "manifests" / "exp_persist.json") + + +@dataclass +class ResolvedDAG: + """A live API context's workflow, campaign, and graph rendering.""" + + workflow: object + campaign: Campaign + dot: str + + @property + def namespace(self): + """Return the parsed Snakefile's namespace.""" + return self.workflow.globals + + @property + def jobs(self): + """Return resolved jobs, including already-satisfied dependencies.""" + return tuple(self.workflow.dag.jobs) + + @property + def rule_names(self): + """Return only rules with jobs in this campaign's compute DAG.""" + return {job.rule.name for job in self.jobs} + + @property + def declared_rule_names(self): + """Return every parsed rule, including rules without requested jobs.""" + return {rule.name for rule in self.workflow.rules} + + def jobs_for(self, rule): + """Return jobs of a named rule in stable wildcard order.""" + return sorted( + (job for job in self.jobs if job.rule.name == rule), + key=lambda job: sorted(job.wildcards_dict.items()), + ) + + +def load_profile(name): + """Read the committed cluster profile without invoking its executor.""" + path = REPO / "profiles" / name / "config.yaml" + return yaml.safe_load(path.read_text()) + + +@contextmanager +def resolve(campaign, monkeypatch): + """Resolve ``all`` in an isolated state directory, without running jobs.""" + from snakemake import workflow as sm_workflow + + scripts = REPO / "workflow" / "scripts" + module_names = {path.stem for path in scripts.glob("*.py")} + # Snakefile declarations share Snakemake's module-global namespace. + namespace = sm_workflow.__dict__ + original_namespace = dict(namespace) + with monkeypatch.context() as patch: + patch.syspath_prepend(str(scripts)) + for name in module_names: + patch.delitem(sys.modules, name, raising=False) + for name in list(os.environ): + if name.startswith("SP_") or name == "SNAKEMAKE_PROFILE": + patch.delenv(name) + for name, value in { + "SP_PHASE": "compute", + "SP_PROFILE": "candide", + "SP_RUN_CONFIG": campaign.config_path, + "SP_STATE_DIR": campaign.state_dir, + "SP_CONTAINER": campaign.image, + "SP_CACHE_DIR": campaign.root / "cache", + "SP_SANDBOX": campaign.root / "no-sandbox", + "SP_MISSING_THRESHOLD": "0.34", + "XDG_CACHE_HOME": campaign.root / "cache", + }.items(): + patch.setenv(name, str(value)) + profile = load_profile("candide") + try: + with SnakemakeApi() as api: + workflow_api = api.workflow( + snakefile=REPO / "workflow" / "Snakefile", + workdir=campaign.state_dir, + resource_settings=ResourceSettings( + # Cluster jobs retain each rule's full thread count. + nodes=profile["jobs"], + default_resources=profile["default-resources"], + overwrite_resources=profile.get("set-resources", {}), + ), + deployment_settings=DeploymentSettings( + deployment_method={DeploymentMethod.APPTAINER}, + apptainer_args=profile["apptainer-args"], + ), + workflow_settings=WorkflowSettings( + runtime_source_cache_path=( + campaign.root / "source-cache" + ), + ), + ) + dag_api = workflow_api.dag(DAGSettings( + targets={"all"}, + # Snakemake 9's graph renderer compares the CLI string. + print_dag_as="dot", + rerun_triggers=RerunTrigger.parse_choices_set( + profile["rerun-triggers"], + ), + )) + stream = io.StringIO() + with redirect_stdout(stream): + dag_api.printdag() + yield ResolvedDAG( + workflow_api._workflow, campaign, stream.getvalue(), + ) + finally: + # Bare script imports carry module-level environment constants. + # Remove this parse's copies before restoring any caller's modules. + for name in module_names: + sys.modules.pop(name, None) + for name in namespace.keys() - original_namespace.keys(): + del namespace[name] + namespace.update(original_namespace) diff --git a/tests/workflow/params.py b/tests/workflow/params.py new file mode 100644 index 000000000..bb0baf86f --- /dev/null +++ b/tests/workflow/params.py @@ -0,0 +1,57 @@ +"""Fingerprint rendered prologues and shells without checkout or tmp paths.""" + +import hashlib +import json + +from tests.workflow.harness import REPO + + +def params_pin(dag): + """Return SHA-256 digests for unit_pre and every declared rule's shells. + + Each rule includes all resolved wildcard instances, its shell template, + and its rendered ``params.pre`` (null for rules without that parameter). + The normalized payload is saved beside the fixture for review on failure. + Script hashes and other non-pre params are outside this pin's scope. + """ + campaign = dag.campaign + + def normalize(value): + if value is None: + return None + return (value.replace(str(campaign.root), "") + .replace(str(REPO), "")) + + def digest(value): + text = json.dumps(value, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(text.encode()).hexdigest() + + namespace = dag.namespace + unit_pre = {} + for stage, (level, _) in sorted(namespace["STAGE_DIR"].items()): + unit = next(iter(campaign.ready)) if level == "tile" else "2243881" + unit_pre[stage] = normalize(namespace["unit_pre"](stage, unit)) + rules = {} + for rule in sorted(dag.workflow.rules, key=lambda rule: rule.name): + jobs = dag.jobs_for(rule.name) + # Aggregation-only targets have no shell or pre to expand. + assert jobs or (not rule.shellcmd and not rule.params), rule.name + rules[rule.name] = { + "shell_template": normalize(rule.shellcmd), + "jobs": [{ + "wildcards": dict(sorted(job.wildcards_dict.items())), + "pre": normalize(getattr(job.params, "pre", None)), + "shell": normalize(job.shellcmd), + } for job in jobs], + } + payload = {"unit_pre": unit_pre, "rules": rules} + (campaign.root / "params_rendered.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + ) + return { + "schema": 1, + "algorithm": "sha256", + "sha256": digest(payload), + "unit_pre": {stage: digest(pre) for stage, pre in unit_pre.items()}, + "rules": {name: digest(rule) for name, rule in rules.items()}, + } diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json new file mode 100644 index 000000000..1924265dc --- /dev/null +++ b/tests/workflow/params_pin.json @@ -0,0 +1,41 @@ +{ + "algorithm": "sha256", + "rules": { + "all": "572122d8d1901e12ff591b30adf405f8920be8183129befececbb819f3392ed8", + "clean_exposure": "22cb76b13a5205d20a02a9bd3b8c8bea5ea2801555e24dd2b84b11145f7e79d9", + "clean_tile": "a5c07b0461526ed407df36a291deb866d4181524b4c8e0fbc3dd047fd9d28479", + "exp_get_images": "71e76ff7f96af1c5d2b85697cc5819e2271911f177a2075253f3b8cfa268c1a9", + "exp_persist": "302e2837542bc1102430c27c81c600b7cda32e8bddcb5fd60d33950987609fff", + "exp_psf": "2c4f6d00f1939ccbf05b4982a0202a0ff92a4373a727f4e00f3aaab4eba03352", + "exp_split": "6e954f8f3d06f44d3f164675912ce27d6216648855d04968f9168bd7d0f2c4fa", + "final_cat_merge": "e7f46859c4503a2220713d7bb2507555515d0a9632d780b20f14c59e32210023", + "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844", + "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0", + "tile_detect": "457456f4f71a500e4166a63933b88d14eed34a019a9814f44546e137b0e71d7a", + "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28", + "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", + "tile_get_images": "331a67e747f211ebf4c14b946a7af7f9fc9f55243d69d2791d74aecc3ca228c3", + "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23", + "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb", + "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111", + "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415", + "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3", + "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" + }, + "schema": 1, + "sha256": "74adbcd14bc304f82c3f2d1a50374bc607e5313fa5f3f8b3db6302653bad67ea", + "unit_pre": { + "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", + "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", + "exp_split": "358fa8bbe59680d4f9839e007b343dd25fed733cf3156dccd30d305d33ac9480", + "tile_detect": "adad5d671fa65dd04433e1c82a725b845635e9d2e368a1e70833ed990df2d55a", + "tile_find_exposures": "c5922fb507f6fd040a179b53fba0818697661016c9688984b6ce249186dc6986", + "tile_get_images": "45b47c44bfeb34ac973b89d4e752c8c82e78028f9f0da379c44627005b45e279", + "tile_make_cat": "7579a52e32e76523c0b77d75c468a47d7f2e2ca0c55865f40e9628a7481cd79d", + "tile_merge_cats": "69cfe94ba941d2ba7b1ce24883961b47b191b80aff381da1865c6bc38de0c354", + "tile_merge_headers": "8590cfa3281c88d43eb8c4fe5760176a13e928a419cbea9bea3a8f88b0634b42", + "tile_ngmix": "b6b75b553a62df54aea6c337dcde1e0cdcd0fe30000e86880c1a4689e42c4a72", + "tile_uncompress": "91a15538491e53ee2b0d52b0472909374270918e4b4c80880c4f5a8329e40161", + "tile_vignets": "a6111cef708aa3c9fb0144b49de9f781eef84d2096ba4f8a3de9bb45b1a3774d" + } +} diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py new file mode 100644 index 000000000..e582138bc --- /dev/null +++ b/tests/workflow/test_dag.py @@ -0,0 +1,117 @@ +"""Resolved-job checks for campaign scope, product paths, and PSF custody.""" + +from collections import Counter +from pathlib import Path + +import pytest +from snakemake.exceptions import WorkflowError + +from tests.workflow.harness import Campaign + +BASE_RULES = { + "all", "tile_get_images", "tile_uncompress", "tile_find_exposures", + "exp_get_images", "exp_split", "exp_psf", "clean_exposure", + "tile_exp_forest", "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat", "clean_tile", + "final_cat_merge", +} +PSF_RULES = {"exp_persist", "star_cat_merge"} + + +def test_rule_set_matches_input_mode(campaign, dag): + """A PSF gate cannot remove real-PSF products or add them to fake PSFs.""" + expected = BASE_RULES.copy() + if campaign.psf_model != "fake": + expected |= PSF_RULES + assert dag.rule_names == expected + assert "merge_final_cats" not in dag.declared_rule_names + + +def test_clean_exposure_waits_on_persist_iff_psf(campaign, dag): + """Reclamation waits for persistence and exactly its in-scope readers.""" + jobs = dag.jobs_for("clean_exposure") + assert {job.wildcards.exp for job in jobs} == set(campaign.exposures) + for job in jobs: + exp = job.wildcards.exp + expected = [ + campaign.tile_manifest(tile, "tile_vignets") + for tile, exposures in campaign.ready.items() if exp in exposures + ] + if campaign.psf_model != "fake": + expected.append(campaign.persist_manifest(exp)) + assert Counter(map(str, job.input)) == Counter(map(str, expected)), ( + "clean-exposure-waits-on-persist-iff-psf", exp, list(job.input) + ) + + +def test_final_cat_merge_reads_every_ready_tile(campaign, dag): + """A merge cannot drop a ready tile or pull one from outside this batch.""" + jobs = dag.jobs_for("final_cat_merge") + assert len(jobs) == 1 + assert Counter(map(str, jobs[0].input)) == Counter( + str(campaign.final_cat(tile)) for tile in campaign.ready + ) + + +def test_products_use_products_dir_and_run_name(campaign, dag): + """Neither scratch nor a directory basename can name durable products.""" + expected = { + "tile_make_cat": { + campaign.final_cat(tile) for tile in campaign.ready + }, + "final_cat_merge": { + campaign.products_dir / f"final_cat_{campaign.name}.hdf5" + }, + "star_cat_merge": set(), + "exp_persist": set(), + } + if campaign.psf_model != "fake": + expected["star_cat_merge"] = { + campaign.products_dir / f"full_starcat_{campaign.name}.hdf5" + } + expected["exp_persist"] = { + campaign.persist_manifest(exp) for exp in campaign.exposures + } + output_names = { + "tile_make_cat": "final_cat", "final_cat_merge": "merged", + "star_cat_merge": "star_cat", "exp_persist": "manifest", + } + for rule, paths in expected.items(): + actual = { + Path(getattr(job.output, output_names[rule])) + for job in dag.jobs_for(rule) + } + assert actual == paths, (rule, actual, paths) + assert all( + path.is_relative_to(campaign.products_dir) for path in actual + ) + for job in dag.jobs_for("exp_persist"): + assert Path(job.params.dest) == ( + campaign.products_dir / "exp" / job.wildcards.exp[:2] + / job.wildcards.exp / "psf" + ) + for rule in ("final_cat_merge", "star_cat_merge"): + for job in dag.jobs_for(rule): + assert job.params.campaign == campaign.name + assert f"--campaign '{campaign.name}'" in job.shellcmd + assert dag.namespace["CAMPAIGN"] == campaign.name + assert Path(dag.namespace["INDEX_DB"]) == campaign.index_db + + +def test_missing_run_fails_during_parse(campaign, resolve_dag): + """Explicit paths cannot bypass the required campaign name diagnostic.""" + campaign.omit_run() + with pytest.raises(WorkflowError, match=( + r"for machine='candide', input_type=" + r".*: run\. Set them in your run config \(SP_RUN_CONFIG\)\." + )): + with resolve_dag(campaign): + pytest.fail("a campaign without run: must fail at parse time") + + +def test_mccd_is_refused_during_parse(tmp_path, resolve_dag): + """MCCD products are unreadable to persistence and the star merge.""" + campaign = Campaign(tmp_path / "campaign", "data", "mccd") + with pytest.raises(WorkflowError, match=r"psf_model=mccd: PSF persistence"): + with resolve_dag(campaign): + pytest.fail("psf_model=mccd must be refused at parse time") diff --git a/tests/workflow/test_params_pin.py b/tests/workflow/test_params_pin.py new file mode 100644 index 000000000..faeb43158 --- /dev/null +++ b/tests/workflow/test_params_pin.py @@ -0,0 +1,38 @@ +"""Campaign-boundary checks on the shell and params.pre rerun surface.""" + +import json +from pathlib import Path + +from tests.workflow.harness import Campaign +from tests.workflow.params import params_pin + +PIN = Path(__file__).with_name("params_pin.json") +BOUNDARY_MESSAGE = ( + "params.pre is a rerun trigger under both profiles; this change reruns " + "every finished unit of a resumed campaign — land it at a campaign " + "boundary and update the pin" +) + + +def test_unit_pre_changes_at_campaign_boundary(psfex_dag, pytestconfig): + """A shared prologue or per-rule shell edit must move the reviewed pin.""" + actual = params_pin(psfex_dag) + if pytestconfig.getoption("--update-params-pin"): + PIN.write_text(json.dumps(actual, indent=2, sort_keys=True) + "\n") + assert PIN.is_file(), BOUNDARY_MESSAGE + assert actual == json.loads(PIN.read_text()), BOUNDARY_MESSAGE + + +def test_params_pin_ignores_fixture_root(tmp_path, resolve_dag): + """A different temporary campaign directory cannot require a new pin.""" + pins = [] + for directory in ("first-root", "another-root"): + campaign = Campaign(tmp_path / directory, "data", "psfex") + with resolve_dag(campaign) as dag: + pins.append(params_pin(dag)) + assert pins[0] == pins[1], "normalise fixture paths before hashing" + + +def test_params_is_a_rerun_trigger_under_both_profiles(profile): + """Neither cluster profile can silently disable the protected trigger.""" + assert "params" in profile["rerun-triggers"], BOUNDARY_MESSAGE diff --git a/workflow/CONTRACTS b/workflow/CONTRACTS new file mode 100644 index 000000000..9538e78cf --- /dev/null +++ b/workflow/CONTRACTS @@ -0,0 +1,55 @@ +Scientific contracts governing workflow/ and everything beneath it. + +Format: `@sc [meta] id`, then one paragraph of prose. `sc-list ` prints +every contract governing a path; the Snakefile and rules/*.smk carry theirs here +because sc-list does not parse Snakemake. Where a check enforces a contract, the +prose names it. + +@sc [label:provenance] campaign-name-is-run +A campaign's name is the run config's `run:` and nothing else. The Snakefile +binds it once, `CAMPAIGN = config["run"]`, and every product that carries a +campaign name takes it from there: `final_cat_.hdf5` and its +`patches/` group, `full_starcat_.hdf5`, and the `$run` in the +machine defaults' `products_dir`/`index_db`. No rule or script reads a +`campaign` key or derives a name from a directory (`PRODUCTS_DIR.name`); two +sources that can disagree would file one campaign's merge under another's name. +Every campaign product path is rooted in `PRODUCTS_DIR`. Enforced by +tests/unit/test_campaign_lineage.py and +tests/workflow/test_dag.py::test_products_use_products_dir_and_run_name. + +@sc [label:schema] final-cat-param-is-exact-allow-list +Each input type's `config/*/final_cat.param` is the merged catalogue's exact +column list, in order: `final_cat_merge` writes those columns and no others, +and the reader raises on a listed column a tile lacks. So a name belongs there +only if `make_cat` writes it on EVERY tile. A per-epoch family (`EXP_ID_n`, +`CCD_n`, `HSM_*_PSF_n`) qualifies only with a fixed slot count, since +`make_cat` otherwise sizes it from the tile's own maximum `N_EPOCH`. Enforced +by tests/unit/test_final_cat_merge_invariants.py (the merge) and +tests/module/test_psf_grammar_properties.py (the shipped names). + +@sc [label:hazard] unit-pre-changes-at-campaign-boundary +Every line `unit_pre()` emits is part of each rule's `params.pre`, and both +profiles run with the `params` rerun trigger, so any change to its output +replans every finished unit of the campaign. Mid-campaign that rerun is +unsatisfiable for tiles whose exposure stores were reclaimed, and the failed +tile group's cleanup deletes those finished tiles' `final_cat`. Change +`unit_pre` output, or anything else in a rule's `params`, only at a campaign +boundary on a fresh root. The rendered `unit_pre`, per-job `params.pre`, and +shell strings are pinned by +tests/workflow/test_params_pin.py::test_unit_pre_changes_at_campaign_boundary; +test_params_is_a_rerun_trigger_under_both_profiles checks the profile premise. +Regenerate tests/workflow/params_pin.json only at that boundary with +`pytest tests/workflow/test_params_pin.py --update-params-pin`. + +@sc [label:custody] clean-exposure-waits-on-persist-iff-psf +`clean_exposure` waits on the exposure's persisted PSF products exactly when +`PERSISTS_PSF` (`psf_model != "fake"`, which is `psfex`: `mccd` is refused at +parse time by `refuse_unpersistable_psf`): on the `exp_persist` manifest for a +live store, and on the tar (a leaf), or nothing, for a reclaimed one. Dropping +the edge under psfex lets reclamation delete the scratch store before its PSF +products reach `products_dir`; keeping it under `fake` makes every clean wait on +a rule that is not in the DAG; naming the manifest for a reclaimed store lets a +`persist_exp:` edit rebuild the exposure from VOS. Enforced by +tests/workflow/test_dag.py::test_clean_exposure_waits_on_persist_iff_psf, +which also checks that the consumer edges are exactly the in-scope vignets +manifests. diff --git a/workflow/README.md b/workflow/README.md index ca28a85f0..6319346aa 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -22,16 +22,16 @@ uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, inputs.tiles/exposures, outputs.run_dir, -# outputs.products_dir/index_db, and container. +# Write a run config (see Run configuration below) that sets at least `run:`, +# the campaign's name; workflow/config.yaml's machines: table supplies the rest. # `psf_model` is `psfex` or `mccd`. psfex is exercised by smk-g4 through smk-g6; mccd has run the full chain on # an image-sim star tile (one focal-plane model per exposure, ~1.5 CPU-hours each). # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. -workflow/bin/sp run # bring products on disk up to date with the tile list -workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp run -c my_run.yaml # bring products on disk up to date with the tile list +workflow/bin/sp report -c my_run.yaml # emit run_report.json now (mid-run is fine) workflow/bin/sp cancel # scancel this workflow's jobs workflow/bin/sp container status # which image the jobs will run ``` @@ -61,6 +61,8 @@ each overlay file to its `cfis/` original confined to input naming. `psf_model: fake` is the simulations' true PSF: the exposure stage runs only SExtractor (for the background maps the vignets read), and `tile_vignets` runs `fake_interp_runner`, which writes the `galaxy_psf` product from `psf_dict`. +With no PSF model there is nothing to persist per exposure, so `exp_persist` and +`star_cat_merge` do not run and `clean_exposure` does not wait on them. Simulations that contain stars can run `psfex` or `mccd` exactly as the data do. One campaign per shear branch, each with its own run config: @@ -78,13 +80,18 @@ the jobs read.) `SP_PROFILE` (default `nibi`, or `machine:` in the run config, w agree with it) and `input_type:` then select an entry of the `machines:` table, which supplies `tile_list`, `retrieve` (`symlink` or `vos`), `inputs`, `outputs` and `container` for any of these the run config leaves unset (`$base_dir` expands -to that machine's `base_dir`, `$run` to the run config's `run:`). A value of `TBD` stops the run at parse time +to that machine's `base_dir`, and `$name` or `${name}` to any top-level scalar +of the run config, e.g. `$run` to `run:`; write `${name}` when word characters +follow). `run:` is +required: it also names the campaign's merged catalogues, and config.yaml leaves +it unset. An unset required key, or a value of `TBD`, stops the run at parse time until it is set. A run config therefore only needs what differs, e.g. for one SKiLLS shear branch on candide: ```yaml machine: candide input_type: image_sims +run: 1z2z_grid_3 psf_model: fake psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle tile_list: /path/to/tiles.txt @@ -98,7 +105,11 @@ outputs: ``` sp_validation's image-simulation workflow drives these campaigns and measures m -from their final catalogues. +from their final catalogues. A simulation campaign ends in the same merged +catalogue as a data campaign, written by the same `final_cat_merge` rule: +`/final_cat_.hdf5`, with the columns of +`config/cfis_image_sims/final_cat.param` and the tile count as the file's +`n_tiles` attribute (there is no `n_tiles_final.txt`). On candide, the node-local tile store (bound from the node's 31 GB `/tmp`) does not hold several dense image-sim tiles at once. Set `tile_store_root:` in the run @@ -168,7 +179,9 @@ and the run fails if either phase failed. ## The launch code snapshot `sp run` copies the code it is about to launch — `workflow/` (config symlinks -dereferenced), `src/` and the profile — into `/code`, records HEAD +dereferenced), `src/`, the repo's `scripts/` (`final_cat_merge` loads +`scripts/python/create_final_cat.py` by path) and the profile — into +`/code`, records HEAD plus a dirty flag in `/code/snapshot.json`, and runs the campaign entirely out of that copy. It matters because a campaign is not one process: the SLURM executor re-invokes snakemake on every job's node, so jobs re-parse the @@ -217,15 +230,18 @@ workflow/ bin/sp committed launcher (module load + /project venv + launch code snapshot + run/report/container/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures - exposure.smk per-exposure: get_images, split, psf (no temp()) - tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat + exposure.smk per-exposure: get_images, split, psf, persist (no temp()); campaign star_cat_merge + tile.smk per-tile: exp forest, merge_headers, detect, vignets, ngmix, merge, make_cat; campaign final_cat_merge scripts/ - sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count check) build_index.py prepare-phase run_index.sqlite builder (plain script) build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) container.py image layers + the resolution order behind `sp container` (stdlib-only) + persist_exp.py ONE exposure's keepable PSF products -> one tar on products_dir (the exp_persist rule) + hdf5_reconcile.py bring an hdf5 catalogue into agreement with a campaign (shared by both merges) + merge_star_cat.py ALL exposures' validation_psf, out of the tars -> full_starcat_.hdf5 + merge_final_cat.py ALL tiles' final_cat -> final_cat_.hdf5 (the final_cat_merge rule) clean_exposure.py ONE exposure's store + manifests + logs -> tombstone (the clean_exposure rule) profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` @@ -301,6 +317,99 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee trigger reads that cut as a reason to rerun the very tiles it protects. Know the consequence — `--forcerun` on a tile whose `final_cat` exists will not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **PSF products leave scratch before the purge does.** `exp_persist` packs + the products named by `persist_exp:` in `config.yaml` from the exposure's + scratch store into ONE uncompressed tar, + `/exp///psf/.tar` (inodes, not bytes, bind + on /project), and writes ONE manifest beside it recording the patterns, the + members and their sizes. The + threat it answers is the /scratch purge, not `clean_exposure` — the store goes + in 60 days whether or not the workflow reclaimed it — so it runs even with + `clean: false`, requested directly by `rule all`. `clean_exposure` takes its + manifest as an input, so reclamation can never overtake the copy. It is a + rule of its own rather than a `cp` on the end of `exp_psf` because the keep + list rides on `params`: adding a pattern reruns seconds of packing, not four + hours of PSF fitting per exposure. A pattern that matches nothing is a + recorded warning (setools rejects sparse CCDs); matching nothing at all is a + failure. A `localrule`, by the same arithmetic as `clean_exposure`. +- **The star catalogue's inputs are always kept; `persist_exp:` is what you + keep on top.** `exp_persist` packs `psf_validation` — the psfex_interp + validation catalogue, one per CCD — for every exposure whatever the config + says, because `star_cat_merge` stacks exactly those into the campaign's + `full_starcat`. They are that catalogue's provenance, and they are what keeps + appending a tile next month cheap rather than a rebuild from VOS. About 2 MB + per exposure: ~40 GB and ~40k inodes at DR6 scale, against a ~1 M-inode group + quota. `persist_exp:` is purely additive, and an empty list is legal — the tar + then holds the merge's inputs and nothing else. +- **The keep list names products, not globs.** Entries are names from a + catalogue in `workflow/scripts/persist_exp.py`, which is the single source of + truth for what each one means and what keeping it buys + ([#844](https://github.com/CosmoStat/shapepipe/issues/844)); `config.yaml`'s + block is that catalogue rendered, and `persist_exp.py --list-products` prints + it. Sizes are per exposure, 40 CCDs, measured on smk-m2. + + | product | glob | per exposure | what it buys | + |---|---|---|---| + | `psf_model` | `*.psf` | 2.8 MB | re-interpolate the PSF anywhere later, no rebuild | + | `psfex_cat` | `psfex_cat-*.cat` | unmeasured | which stars PSFEx's outlier rejection clipped | + | `star_selection` | `star_selection-*.fits` | 24.5 MB | which stars the selection cuts rejected, and why | + | `star_train` | `star_split_ratio_80-*.fits` | 19.9 MB | the 80% sample PSFEx fitted | + | `star_test` | `star_split_ratio_20-*.fits` | 7.1 MB | the 20% sample `psf_validation` corresponds to | + | `star_stats` | `star_stat-*.txt` | unmeasured | setools' per-CCD counts, density and FWHM cuts | + + The default is `psf_model`. `psf_validation` is in the catalogue too but needs + no naming; naming it anyway is harmless. **Retention is additive**: an + existing tar is a floor, so shrinking the list adds nothing and removes + nothing. Dropping a product is a deliberate act on `products_dir`, not a + config edit — otherwise editing a config would delete products from the + backed-up filesystem whose scratch originals are long gone. A raw glob is still accepted as an + escape hatch — anything with a glob metacharacter or a dot is read as one — + and an unknown *name* is a parse-time error listing the valid ones. The list + is exposure-side only; tile-side retention is #844 follow-up. +- **The campaign ends in two merged catalogues, and the workflow makes both.** + Everything above is per unit; the two products downstream analysis actually + opens are per *campaign*, and until these rules existed each was a manual pass + after the run. + `star_cat_merge` collects every exposure's every CCD's `psf_validation` into + `/full_starcat_.hdf5`, one dataset per exposure at + `exposures/` — the rho/tau statistics input. It reads the members + straight out of the per-exposure tars (`tarfile`; unpacking ~800k files to + merge them would defeat the tar's whole purpose), keeps their native dtypes, + and stores `CCD_NB` as an int. sp_validation still opens the old flat FITS + name, `full_starcat-0000000.fits`; its readers move to this file under + [sp_validation#340](https://github.com/CosmoStat/sp_validation/issues/340), + the same migration that retires the `patches/` key on the tile side. The rule + exists whenever the campaign has a persisted exposure. + **Two writers, one schema.** The module runner still emits the flat FITS + table through `MergeStarCatPSFEX`, and this rule emits the hdf5; they are + separate implementations on purpose, because only one of them reads tars, + keeps native dtypes and reconciles. Their 22 COLUMN NAMES must not drift + apart, and nothing else would notice if they did — a column added to one + writer would just be missing from the other's product. `tests/unit/` + `test_star_cat_columns.py` is what holds them together. + `final_cat_merge` collects every ready tile's `final_cat-.fits` into + `/final_cat_.hdf5`: one dataset per tile under a group + named for the campaign, the `final_cat.param` columns, an `n_tiles` attribute. + That schema is what sp_validation's reader opens, so it is fixed; the column + extraction reuses `scripts/python/create_final_cat.py` while the file is + written here, because that script's own discovery walks a directory layout + this workflow does not have. The run config's `run:` names both files and the + group. + BOTH RECONCILE, through one shared module (`hdf5_reconcile.py`) so the + campaign's two products cannot disagree about what an output owes its inputs. + Each adds the units that have no dataset, drops datasets whose unit left the + campaign, re-reads one whose source changed (every dataset records its + source's size and mtime) or whose column set moved (a digest on the file's + root), and leaves the rest unread — because re-reading a campaign to add one + unit is ~800 GB of IO at DR6 scale. The *content* is still a function of the + input set; the byte layout is not, and a no-op leaves the file untouched + rather than rewritten. + Both rerun when the set changes: the unit ids' fingerprint rides on `params`. + Neither is a `localrule` — one job over ~20k units is real work — and neither + puts its input paths in its shell, which is not fastidiousness: ~20k paths is + an order of magnitude over Linux's 128 KiB `MAX_ARG_STRLEN` for a single argv + entry, so each job is handed the tile list and the run index and derives the + same set from them. - **A dead tile can be told to stop pinning exposures.** An exposure is cleanable only once every consuming tile has its vignets, so one permanently-failed tile holds its ~80 exposures for the life of the diff --git a/workflow/Snakefile b/workflow/Snakefile index 2befdb5ec..913b84543 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -31,6 +31,7 @@ the manifest says "this stage succeeded", the log says "here is what happened" (the contract is argued in completeness.py's docstring). """ +import fnmatch import functools import hashlib import json @@ -40,6 +41,10 @@ import sys from pathlib import Path from snakemake.exceptions import WorkflowError +# Explicit rather than relying on the name snakemake injects into this +# namespace: the one place we log at parse time is a branch that only a +# non-default keep list reaches, and a NameError there would be found by a user. +from snakemake.logging import logger # Resolved relative to THIS file, not the working directory: snakemake runs with # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project. @@ -80,7 +85,8 @@ run_config.apply_machine_defaults(config) _unresolved = run_config.unresolved(config) if _unresolved: raise WorkflowError( - f"Unset or {run_config.PLACEHOLDER!r} for machine={MACHINE!r}, " + f"Unset, {run_config.PLACEHOLDER!r}, or holding an unexpanded " + f"$variable for machine={MACHINE!r}, " f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " f"your run config (SP_RUN_CONFIG).") @@ -102,7 +108,8 @@ container: _image # `fake` is the image-simulation true PSF: no exposure PSF fit; the tile's # galaxy_psf comes from `psf_dict` via fake_interp_runner (read by the -# configs as ${SP_PSF}_interp_runner). Sims with stars may use psfex/mccd. +# configs as ${SP_PSF}_interp_runner). Sims with stars may use psfex. mccd is +# a valid chain but refused by refuse_unpersistable_psf below. PSF_MODELS = {"psfex", "mccd", "fake"} PSF_MODEL = config.get("psf_model", "psfex") if PSF_MODEL not in PSF_MODELS: @@ -141,15 +148,27 @@ INPUTS = config["inputs"] OUTPUTS = config["outputs"] RUN_DIR = Path(OUTPUTS["run_dir"]) # Defaults to RUN_DIR so a scratch-only run (a fixture, a smoke test) needs no -# second path: one root, exactly the pre-D5 layout. +# second path. Such a one-root run must set clean: false and clean_tiles: +# false; refuse_one_root_cleaners enforces it. PRODUCTS_DIR = Path(OUTPUTS.get("products_dir") or RUN_DIR) INDEX_DB = Path(OUTPUTS["index_db"]) +# The campaign's NAME is the run config's `run:` — the same name `$run` expands +# to in the paths above. The two campaign-level merges label their output with +# it (`final_cat_.hdf5` and the group inside it holding the per-tile +# datasets, `full_starcat_.hdf5`). REQUIRED in run_config.py, so an unset +# `run:` has already refused above. +CAMPAIGN = config["run"] SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There # is no separate `config_src` setting; input_type picks between the two dirs. CONFIG_DIR = Path(workflow.basedir) / "config" / { "data": "cfis", "image_sims": "cfis_image_sims"}[INPUT_TYPE] +# `sp run`'s code snapshot (bin/sp), a sibling of workflow.basedir inside +# $STATE_DIR/code. Read by the campaign-level merges so the products they write +# carry the code that produced them; absent when this Snakefile is driven +# outside `sp run`, which the merges tolerate (hdf5_reconcile.code_provenance). +SNAPSHOT_JSON = Path(workflow.basedir).parent / "snapshot.json" sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 @@ -176,8 +195,14 @@ if PHASE not in ("prepare", "compute", "passthrough"): raise WorkflowError( f"SP_PHASE={PHASE!r} is not one of prepare, compute, passthrough.") +# DEDUPED, order preserved. The tile list is appended to by hand across a +# campaign, so a tile can appear twice — harmless for a per-tile target, which +# is the same path requested twice, but not for the campaign-level merges: their +# fingerprint counts what is in this list and the job derives a deduped set from +# the same file (build_index.campaign_tiles), so a duplicate line would make the +# two disagree about a set they must name identically. with open(config["tile_list"]) as f: - TILES = [ln.strip() for ln in f if ln.strip()] + TILES = list(dict.fromkeys(ln.strip() for ln in f if ln.strip())) # --- ngmix scatter (D4) ---------------------------------------------------- # Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one @@ -286,9 +311,14 @@ else: "SELECT tile_id FROM tile_exposures WHERE exp_id = ? ORDER BY rowid", (exp,))] -# Tiles this run can actually compute: declared AND indexed. The index spans the -# campaign, so it is intersected with the declared list, not used as it. -TILES_READY = [t for t in TILES if tile_exposures(t)] +# Tiles this run can actually compute: declared AND ready in the index +# (build_index.ready_tiles, as the merges read it). The index spans the +# campaign, so it is intersected with the declared list, not used as it. A tile +# whose exposure list went missing keeps its edges for clean_consumers but is +# not ready. +_READY_INDEXED = (build_index.ready_tiles(INDEX_DB) if INDEX_DB.exists() + else set()) +TILES_READY = [t for t in TILES if t in _READY_INDEXED] READY_SET = set(TILES_READY) # A compute invocation with nothing to compute is never a success. Without this, @@ -316,6 +346,7 @@ EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") # The persistent root mirrors the scratch one, shard for shard, so the two trees # read as the same campaign seen from two filesystems. PROD_TILE_DIR = str(PRODUCTS_DIR / "tiles" / "{shard}" / "{tile}") +PROD_EXP_DIR = str(PRODUCTS_DIR / "exp" / "{shard}" / "{exp}") def tile_dir(tile): return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" @@ -329,6 +360,18 @@ def tile_manifest(tile, stage): def exp_manifest(exp, stage): return f"{exp_dir(exp)}/manifests/{stage}.json" +def prod_exp_dir(exp): + """The exposure's dir on the PERSISTENT root — where exp_persist writes. + + Sharded identically to the scratch one, so the two trees read as the same + campaign seen from two filesystems, exposure side as well as tile side.""" + return f"{PRODUCTS_DIR}/exp/{exp[:2]}/{exp}" + +def prod_exp_manifest(exp, stage): + """A manifest that must SURVIVE reclamation, so it is not in the exposure's + scratch manifests/ dir (clean_exposure deletes that wholesale).""" + return f"{prod_exp_dir(exp)}/manifests/{stage}.json" + def forest_dir(tile): return f"{tile_dir(tile)}/exp_forest" @@ -361,12 +404,52 @@ def unit_num(unit): # gets its own hash, on the forest rule only. def script_hash(name): """The 12-hex fingerprint of one script under workflow/scripts/.""" - return hashlib.md5((SCRIPTS / name).read_bytes()).hexdigest()[:12] + return path_hash(SCRIPTS / name) + + +def path_hash(path): + """The 12-hex fingerprint of any file the rules depend on but do not own. + + A rule whose behaviour comes from more than its own script needs all of it + in one trigger. final_cat_merge is the case: what it writes is decided by + scripts/python/create_final_cat.py (the column extraction) and by + CONFIG_DIR's final_cat.param (which columns), and NEITHER is under + workflow/scripts/ nor a declared input. Without them in the hash, this PR's + own edits to both would have left every finished campaign's hdf5 untouched + and nothing would have said so. + + A MISSING FILE IS NOT A PARSE ERROR. This runs at module level, so raising + here kills EVERY invocation of the workflow — `sp --unlock`, `sp report`, + a dry run — over a file only one rule needs. Worse, it kills them with a + bare FileNotFoundError, which is exactly the diagnosis merge_final_cat.py + already carries and would print if the job were allowed to reach it. So the + hash degrades to a sentinel and says so once; the parse survives, the rule + still exists, and the job fails with the message written for it. + """ + path = Path(path) + try: + return hashlib.md5(path.read_bytes()).hexdigest()[:12] + except OSError: + if workflow.is_main_process: + logger.warning( + f"missing: {path} — it is part of a rule's rerun trigger, so " + f"that rule cannot tell whether it is out of date. The job " + f"that needs the file will say so when it runs.") + return "missing" SCRIPT_HASH = script_hash("completeness.py") FOREST_HASH = script_hash("build_forest.py") CLEAN_HASH = script_hash("clean_exposure.py") CLEAN_TILE_HASH = script_hash("clean_tile.py") +PERSIST_HASH = script_hash("persist_exp.py") +MERGE_STAR_HASH = script_hash("merge_star_cat.py") +# Three files, one trigger: the rule's script, the column extraction it calls, +# and the parameter file that says which columns (path_hash argues why). +MERGE_FINAL_HASH = ":".join(( + script_hash("merge_final_cat.py"), + path_hash(Path(workflow.basedir).parent / "scripts" / "python" + / "create_final_cat.py"), + path_hash(CONFIG_DIR / "final_cat.param"))) # ngmix_range.py earns a hash for a stronger reason than the others. What it # emits is not a stale RESULT but a stale BOUNDARY, and a tile's eight chunks are # a PARTITION of its object IDs: resume a tile across an edit to the split and @@ -484,6 +567,462 @@ def clean_targets(): out.append(tombstone(exp)) return sorted(out) +# --- persisted exposure products (D5) -------------------------------------- +# The keep list is config, not a rule input, and it is READ HERE so that exactly +# one place converts it into the form the rule carries. +# +# OPTIONAL RETENTION, and only that. What star_cat_merge needs — every CCD's +# psf_validation — is packed by exp_persist whatever this list says +# (persist_exp.py's ALWAYS argues why: provenance for the merged catalogue, and +# a cheap tile append later). So an EMPTY list is a coherent instruction and not +# a switch that turns persistence off: the tar then holds the star catalogue's +# inputs and nothing else, and exp_persist still runs for every exposure. +PERSIST_EXP = list(config.get("persist_exp") or []) + +# Whether exposures have PSF products to persist at all. psf_model=fake (the +# image-simulation true PSF) fits no model: exp_psf runs SExtractor only and +# writes no validation_psf-*.fits, so exp_persist would find nothing to pack +# and star_cat_merge nothing to stack. Under fake both drop out of the DAG, +# and clean_exposure stops waiting on exp_persist; final_cat_merge is +# unaffected. psf_exposures() below is where this gate acts. +PERSISTS_PSF = PSF_MODEL != "fake" + + +def refuse_unpersistable_psf(psf_model): + """Refuse a PSF model whose products the persistence path cannot read. + + exp_persist packs PSFEx products (persist_exp.py maps psf_model to + *.psf) and star_cat_merge reads PSFEx's per-CCD validation table in HDU 2 + (merge_star_cat.py). MCCD writes fitted_model-.npy and one + exposure-wide table in HDU 1, so an MCCD campaign would run every exposure + and then fail at star_cat_merge. Refusing at parse time costs nothing. + """ + if psf_model == "mccd": + raise WorkflowError( + "psf_model=mccd: PSF persistence (workflow/scripts/persist_exp.py) " + "and the star-catalogue merge (workflow/scripts/merge_star_cat.py) " + "read PSFEx products only. An MCCD campaign needs an MCCD-aware " + "persist table and star-catalogue reader first (tracked in the PR).") + + +refuse_unpersistable_psf(PSF_MODEL) + +# The keep list names PRODUCTS (`psf_model`), not globs (`*.psf`); the +# catalogue that maps one to the other lives in persist_exp.py, which is also +# what the rule runs, so there is one definition and not a copy here. +# UNKNOWN NAMES DIE AT PARSE TIME, listing the valid ones — a typo in a keep +# list would otherwise be a silently-empty keep or a per-exposure failure an +# hour into a campaign. +import persist_exp as _persist # noqa: E402 + +for _entry in PERSIST_EXP: + try: + _persist.resolve(_entry) + except KeyError as _exc: + raise WorkflowError(f"config persist_exp: {_exc.args[0]}") + + +def persist_targets(): + """Which exposures this invocation must pack PSF products off scratch for. + + `rule all` requests these DIRECTLY rather than reaching them only through + clean_exposure. Persistence and reclamation are different concerns — the + /scratch purge takes the store whether or not `clean:` is on — and hanging + the copy off the clean rule alone would mean a campaign run with clean:false + persists nothing and loses everything at the purge. + + Scope is the ready tiles' exposures, which `all` already builds through the + tile chain, so nothing new is pulled into the DAG by asking. + + EXCEPT AN EXPOSURE WHOSE STORE IS GONE. Its exp_psf manifest is not there, + so requesting its persist manifest would make the DAG rebuild the whole + exposure chain from VOS — the avalanche tile.smk's reclaimed-edge cut exists + to prevent, arriving through a new target instead. exp_store_reclaimed() + below is that test, and what it does NOT test is the tombstone. + + HEAD PROCESS ONLY, for the same reason as clean_targets() above. + """ + if not workflow.is_main_process: + return [] + return persist_manifests() + + +@functools.lru_cache(maxsize=1) +def persist_manifests(): + """persist_targets() without the head-process guard, memoised. + + The guard on persist_targets() is a cost decision, not a correctness one: + `rule all` reads it at MODULE level, so a job parse would pay a whole + campaign's index walk for a target it can never schedule. star_cat_merge + reads the same list through an INPUT FUNCTION, which snakemake evaluates + only for parses that actually build that job — the head process, and the one + merge job's own re-parse under the slurm executor, which genuinely needs it. + So this half carries no guard and the memo keeps either parse to one walk. + """ + return sorted(prod_exp_manifest(e, "exp_persist") for e in psf_exposures() + if not exp_store_reclaimed(e)) + + +def psf_exposures(): + """The ready tiles' exposures that have PSF products to persist and stack: + all of them, or none under psf_model=fake (PERSISTS_PSF). The one set + exp_persist's targets and star_cat_merge's inputs are drawn from.""" + if not PERSISTS_PSF: + return [] + return sorted({e for t in TILES_READY for e in tile_exposures(t)}) + + +def exp_store_reclaimed(exp): + """True when this exposure's PSF products exist ONLY on the persistent root. + + The one condition both the persist target list and star_cat_merge's edge + choice turn on, and it deliberately does NOT read the tombstone. + + THE TOMBSTONE ALONE IS NOT THE EVIDENCE. It says clean_exposure ran, and + clean_exposure is only one of the two ways a scratch store disappears. + + WHAT THE TEST HAS TO SEPARATE is a store that is GONE from one that has not + been BUILT yet, and no single file says that. This runs at parse time, before + any exp_psf job of a fresh campaign has run, so "the exp_psf manifest is + missing" alone would skip every exposure of a new campaign and persist + nothing at all. The question is therefore: is there evidence this exposure + once had a store? Two files carry it, and either will do: + + * the TOMBSTONE — clean_exposure ran, so the store was built and reclaimed, + and whatever was going to be packed was packed before it went; + * a PERSISTED MANIFEST with no exp_psf manifest beside it — exp_persist + ran, so the store existed, and it is not there now. This is the purge + case, and it is the one the tombstone cannot see: /scratch is purged on + a 60-day window whether or not this workflow reclaimed anything, and it + leaves nothing behind. Keying on the tombstone alone meant that after a + purge, or on any campaign run with `clean: false`, every exposure looked + live, its persist manifest was requested, its exp_psf manifest was not + there, and snakemake rebuilt the entire exposure chain from VOS. + + An exposure with a LIVE store and a manifest is not reclaimed and is still + asked for, which is what lets an edit to `persist_exp:` re-pack in seconds + rather than be silently ignored — the whole reason exp_persist is a rule of + its own. An exposure with neither file is asked for too: either it has not + run yet, or it was purged having saved nothing, and only the DAG can tell + those apart by trying. + """ + return (Path(tombstone(exp)).exists() + or (Path(prod_exp_manifest(exp, "exp_persist")).exists() + and not Path(exp_manifest(exp, "exp_psf")).exists())) + +# --- the campaign-level merges --------------------------------------------- +# Two rules, one job each per campaign, both writing to the persistent root, and +# both the LAST link of a chain whose per-unit half the workflow already had: +# the exposure side ends in one `full_starcat_.hdf5` (every CCD's PSF +# validation catalogue — the rho/tau statistics input) and the tile side in one +# `final_cat_.hdf5` (every tile's final catalogue — the shear +# catalogue sp_validation reads). Until they existed the workflow's product set +# was two files short of what the old `combine_runs.bash` + `create_final_cat.py` +# chain delivered, and every campaign ended with a manual merge. +# +# NEITHER IS A LOCALRULE, and the arithmetic runs the opposite way from +# exp_persist's. Those rules are ~20k jobs of seconds each, so submitting them +# costs more in scheduling latency than the work; these are ONE job each over the +# whole campaign — ~800k catalogues stacked in memory, or ~20k catalogues read +# end to end at DR6 scale. That is a compute job, and it belongs on a node. +# +# NEITHER PUTS ITS INPUT PATHS IN ITS SHELL. `{input}` at DR6 scale is ~20k paths +# in a single argv entry, an order of magnitude over Linux's 128 KiB +# MAX_ARG_STRLEN, and the job would die on exec. So each rule's `input` is the +# DAG EDGE (what must exist first) and each script rediscovers the same set from +# the tile list and the index; what travels is a FINGERPRINT of that set's unit +# ids, on `params`, which is what makes the merge rerun when the set changes and +# not otherwise. +# The scripts' docstrings argue the rediscovery — it is also what lets the merges +# cover exposures whose scratch stores reclamation has since taken. + + +def unit_fingerprint(units): + """A short digest of a set of UNIT IDS, for a merge rule's `params`. + + `params` is a rerun trigger and a set of ids is not: appending a tile grows + the set, moves the digest and reruns the merge, while a rerun over the same + set leaves it alone. Sorted before hashing because the ORDER is not part of + what changed. + + IDS RATHER THAN THE RULE'S `input` PATHS. A path can change while the set + does not — star_cat_merge's edge for one exposure flips from its manifest to + its tar when the store is reclaimed — and a merge that reruns over identical + content on every reclamation pass is a rerun trigger firing on bookkeeping. + The ids are also exactly what the job derives on its own side, so both + halves agree on the set and on how it is named. + """ + joined = "\n".join(sorted(str(u) for u in units)) + return f"{len(units)}:{hashlib.md5(joined.encode()).hexdigest()[:12]}" + + +# NO GATE ON THE KEEP LIST. star_cat_merge used to exist only when +# `persist_exp:` named something validation_psf-shaped, which made the +# campaign's star catalogue an opt-in and a typo away from silently absent. +# exp_persist now packs psf_validation unconditionally, so the merge is +# requested whenever the campaign has a persisted exposure at all, and +# star_cat_targets() below is the only condition left. + + +def full_starcat(): + """The campaign's merged star catalogue — the rho/tau statistics input. + + hdf5, one dataset per exposure, named for the campaign exactly as the shear + catalogue beside it is. The old flat FITS table it replaces was called + `full_starcat-0000000.fits` and sp_validation still opens that name; + CosmoStat/sp_validation#340 moves its readers to this file, the same + migration that retires the `patches/` key on the galaxy side.""" + return f"{PRODUCTS_DIR}/full_starcat_{CAMPAIGN}.hdf5" + + +def final_cat_hdf5(): + """The campaign's merged shear catalogue — sp_validation's galaxy_cat_path.""" + return f"{PRODUCTS_DIR}/final_cat_{CAMPAIGN}.hdf5" + + +def prod_exp_tar(exp): + """The tar exp_persist writes. Not a declared output of anything — see + star_cat_inputs().""" + return f"{prod_exp_dir(exp)}/psf/{exp}.tar" + + +@functools.lru_cache(maxsize=1) +def star_cat_inputs(): + """What star_cat_merge waits for: every exposure of TILES_READY whose PSF + products are on the persistent root, live and reclaimed alike. + + THE SAME SET merge_star_cat.py derives at job time, and that equality is + load-bearing — the fingerprint on `params` is taken over THIS list, so + anything the job stacked that was not in it would be rows no rerun trigger + could see. The job states the rule from its own side: same tile list, same + index, exp_persist manifest present on the persistent root. By the time it + runs, every exposure below has one. + + RECLAIMED EXPOSURES BELONG IN THE STAR CATALOGUE. Carrying their PSF + products off scratch is exactly what exp_persist is for, and a merge that + dropped them would shrink the campaign's star catalogue every time + reclamation ran. But their exp_psf manifest is gone, so REQUESTING their + exp_persist manifest rebuilds the whole exposure chain from VOS — the + avalanche persist_targets() drops them to avoid. + ancient() DOES NOT HELP: it suppresses the timestamp comparison, not the + missing input, and snakemake schedules the chain anyway. Measured on smk-g6 + with one reclaimed exposure given a manifest by hand: the dry run grew + exp_get_images, exp_split, exp_psf and exp_persist jobs. + + So a reclaimed exposure is depended on through its TAR instead. The tar is + not a declared output of any rule (exp_persist declares only its manifest, + deliberately — persist_exp.py says why), so a tar that exists is a DAG leaf: + snakemake requires it and builds nothing. A live exposure keeps its manifest + edge, which is what orders the merge after the packing; its tar does not + exist yet, so it could not serve as the edge. + + An exposure reclaimed by a workflow PREDATING exp_persist has neither tar nor + manifest and is in no set at all. Nothing short of rebuilding its chain from + VOS recovers it; the merge reports how many exposures it found. + """ + live, reclaimed = [], [] + for exp in psf_exposures(): + if not exp_store_reclaimed(exp): + live.append(prod_exp_manifest(exp, "exp_persist")) + elif Path(prod_exp_tar(exp)).exists(): + reclaimed.append(prod_exp_tar(exp)) + return live + reclaimed + + +@functools.lru_cache(maxsize=1) +def star_cat_exposures(): + """The exposure IDs star_cat_merge stacks — what its fingerprint is taken + over. + + THE IDS, NOT THE PATHS, and the difference is a rerun. An exposure's edge + FLIPS from its manifest to its tar the moment its store is reclaimed, so a + fingerprint over paths moves on every reclamation pass and reruns the merge + over content that did not change. The ids move only when the set does, which + is what the trigger is for. It is also what merge_star_cat.py derives on the + job side, so the two agree on the set AND on how it is named. + """ + return [e for e in psf_exposures() + if Path(prod_exp_manifest(e, "exp_persist")).exists() + or not exp_store_reclaimed(e)] + + +# --- sizing the two merges (D4) --------------------------------------------- +# MEASURED, not guessed, and measured as a SLOPE rather than a single number: +# these are the only two rules whose one job's footprint grows with the whole +# campaign, so a constant is wrong by however much the campaign is not the one +# it was tuned on. +# +# Both slopes were measured on this login node, inside the campaign container, +# against synthetic tars for the star side and against smk-g6's real +# catalogues for the tile side. Peak RSS is getrusage(RUSAGE_CHILDREN). +# +# STAR SIDE, AND IT IS FLAT IN THE CAMPAIGN. The merge writes one hdf5 dataset +# per exposure and reads one exposure at a time, so it is sized on the LARGEST +# exposure's members — ~2 MB — not on the campaign's. What follows is the +# history of how that came to be true, because the numbers are the argument. +# +# The FITS full_starcat this replaced was one flat table, so the job held the +# whole campaign. Two fixture points, 20 and 80 exposures of 40 CCDs x 400 stars +# (1.6 MB of members per exposure, against the 2.0 MB measured on smk-m2), +# across the rewrites this PR made to MergeStarCatPSFEX: +# +# input members python lists arrays+concat two passes +# 32.3 MB 383 MB 238 MB 221 MB +# 129.0 MB 1313 MB 740 MB 661 MB +# slope 10.1x 5.5x 4.8x +# +# The tenfold was one python float object (32 bytes) plus a list pointer (8) per +# 4 bytes of float32 payload. Arrays per catalogue removed that; counting rows +# from the headers and filling a preallocated array removed the rest. What +# remained at 4.8x was the OUTPUT: file_io writes every float column as FITS 1D, +# so float32 became a float64 table astropy then buffered. +# +# Per-exposure hdf5 removes the term entirely rather than shrinking it — and +# with it the ~240 GB a DR6-scale flat table would have wanted. Those +# improvements stay upstream regardless: the module runner still merges to one +# FITS table, and they are its fix. +# +# TILE SIDE, and it is the reassuring one. Two points against real smk-g6 +# catalogues, 2 tiles (73.9 MB in, largest 39.6 MB) and 6 tiles (235.5 MB in, +# largest 47.7 MB): peak RSS 129 MB and 139 MB. FLAT IN THE NUMBER OF TILES — +# the merge holds one catalogue at a time — so it is sized on the LARGEST tile, +# not the total, at ~3x it plus the interpreter. +# THE CEILING ON ANY REQUEST, and it is not a formatting nicety: a mem_mb above +# the partition maximum is a job SLURM will never schedule and snakemake will +# never diagnose — it sits PENDING with a reason nobody reads while the campaign +# looks alive. The two merge formulas grow with the campaign, so at some size +# they WILL cross it; capping turns "silently never runs" into "runs on the +# biggest node there is, and possibly dies with a diagnosable OOM". +# +# Nibi's standard compute node is 766 GB (192 cores, 4 GB/core); 750000 leaves +# room for the OS and the slurm accounting overhead. Override with `max_mem_mb:` +# for a cluster with smaller nodes, or to reserve headroom. +MAX_MEM_MB = int(config.get("max_mem_mb", 750_000)) +_capped_warned = set() + + +def capped_mem(mb, rule): + """min(mb, MAX_MEM_MB), and say so ONCE at parse time when it bites.""" + mb = int(mb) + if mb > MAX_MEM_MB: + if rule not in _capped_warned and workflow.is_main_process: + _capped_warned.add(rule) + logger.warning( + f"{rule}: sized at {mb} MB, capped to max_mem_mb={MAX_MEM_MB} " + f"(Nibi's standard node is 766 GB). The job will run with less " + f"memory than the measurement says it wants — expect an OOM, " + f"and split the campaign or fix the merge rather than raising " + f"this number past what a node has.") + return MAX_MEM_MB + return mb + + +STAR_MEM_BASE_MB = 500 # interpreter + astropy + h5py, rounded up +STAR_MEM_FACTOR = 6 # x the LARGEST exposure's members +FINAL_MEM_BASE_MB = 800 +FINAL_MEM_FACTOR = 4 # x the LARGEST tile; ~3 measured +# What one unit costs when its product is not on disk yet to be measured — a +# fresh campaign sizes its merge before anything has been packed or made. The +# exposure figure is psf_validation's alone (the only members the star merge +# reads), not a whole tar's; both are the measured medians in config.yaml's +# persist_exp block and the D5 notes. +EXP_BYTES_DEFAULT = 2_000_000 +# The product whose members star_cat_merge stacks — named once, here and in +# merge_star_cat.py, and resolved through persist_exp.py's catalogue. +STAR_CAT_PRODUCT = _persist.ALWAYS +STAR_CAT_PATTERN = _persist.resolve(_persist.ALWAYS) +TILE_BYTES_DEFAULT = 46_000_000 + + +def _size(path, default): + """Bytes on disk, or the documented per-unit default if it is not there.""" + try: + return Path(path).stat().st_size + except OSError: + return default + + +def star_cat_max_bytes(): + """The LARGEST exposure's psf_validation members — what sizes the merge. + + The star merge holds ONE exposure at a time now that its output is hdf5 + with a dataset per exposure, so its memory is flat in the campaign exactly + as the tile side's is. Sizing on the total would ask a node for a campaign's + worth of memory to hold ~2 MB. + """ + return max(_star_cat_exposure_bytes() or [EXP_BYTES_DEFAULT]) + + +def star_cat_bytes(): + """Total bytes of the members the star merge will actually read. + + THE TAR'S SIZE IS THE WRONG NUMBER, and increasingly wrong as the keep list + grows: the merge reads the psf_validation members and nothing else, while + the tar also holds whatever `persist_exp:` retains. With the default + retention that is 2.4x too much, and with the star_* products on it is ~36x + — a memory request that misses by more than an order of magnitude, and one + that would jump the moment an exposure got packed, since an unpacked one + contributed the per-exposure default instead. So the MANIFEST is read and + only the psf_validation members are counted; persist_exp records the product + each member came from, exactly so this is answerable without opening a tar. + + One json parse per exposure at DAG build, and only for the parse that builds + this job. An exposure not yet packed has no manifest and contributes the + measured default, which is the psf_validation figure and not the tar's. + """ + return sum(_star_cat_exposure_bytes()) + + +@functools.lru_cache(maxsize=1) +def _star_cat_exposure_bytes(): + """Per exposure, the bytes of the members the star merge will read.""" + out = [] + for exp in star_cat_exposures(): + manifest = Path(prod_exp_manifest(exp, "exp_persist")) + if not manifest.exists(): + out.append(EXP_BYTES_DEFAULT) + continue + try: + body = json.loads(manifest.read_text()) + # By product name or, for a manifest written before that field + # existed or by a raw-glob keep list, by file name — the same test + # merge_star_cat.is_member() applies, so the sizing counts exactly + # the members the job will read. + out.append(sum(f["bytes"] for f in body["files"] + if f.get("product") == STAR_CAT_PRODUCT + or fnmatch.fnmatch(f["name"], STAR_CAT_PATTERN))) + except (OSError, ValueError, KeyError): + out.append(EXP_BYTES_DEFAULT) + return out + + +def final_cat_max_bytes(): + """The LARGEST tile catalogue the hdf5 merge will read — what sizes it.""" + return max([_size(final_cat(t), TILE_BYTES_DEFAULT) for t in TILES_READY] + or [TILE_BYTES_DEFAULT]) + + +def star_cat_targets(): + """`full_starcat` when there is anything to stack into it, else nothing. + + One way to get nothing, and it is a state rather than an error: every + exposure in scope is already tombstoned — a + campaign resumed after reclamation, whose exposures were cleaned by a + workflow that predates exp_persist and therefore left neither tar nor + manifest to read. A rule with an empty input list would still be a JOB, and + it would write an empty star catalogue over a good one. + """ + if not workflow.is_main_process: + return [] + return [full_starcat()] if star_cat_inputs() else [] + + +def final_cat_targets(): + """The merged hdf5, whenever this campaign has a tile to put in it.""" + if not workflow.is_main_process or not TILES_READY: + return [] + return [final_cat_hdf5()] + # --- tile reclamation (D5) -------------------------------------------------- # A separate flag from `clean:` (config.yaml carries the full # argument): exposure reclamation costs nothing but a rebuild if a tile is @@ -495,6 +1034,30 @@ def clean_targets(): CLEAN_TILES = flag(config.get("clean_tiles", False)) and PHASE == "compute" +def refuse_one_root_cleaners(products_dir, run_dir, clean, clean_tiles): + """Refuse reclamation when products and scratch share one root. + + With products_dir == run_dir the per-tile final_cat sits inside the tile + store clean_tile prunes, and the exp_persist manifests sit inside the + exposure tree clean_exposure deletes, so reclamation would destroy the + products it exists to protect. A one-root run is for fixtures and smoke + tests, which keep everything anyway. + """ + if Path(products_dir).resolve() != Path(run_dir).resolve(): + return + if clean or clean_tiles: + raise WorkflowError( + f"outputs.products_dir is outputs.run_dir ({run_dir}): a one-root " + "run is for fixtures and smoke tests only, and must run with " + "clean: false and clean_tiles: false, since clean_tile would " + "delete final_cat and clean_exposure the persisted manifests. Set " + "both false, or give products_dir its own root.") + + +refuse_one_root_cleaners(PRODUCTS_DIR, RUN_DIR, flag(config.get("clean")), + flag(config.get("clean_tiles"))) + + def tile_tombstone(tile): """The clean_tile output. Beside manifests/ and logs/, not inside either — same placement and same reason as the exposure tombstone() above.""" @@ -568,9 +1131,9 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, Every line here is part of each rule's ``params.pre`` and so of the ``params`` rerun trigger: a line added for every rule reruns every finished - unit of a campaign on its next ``sp run``. The image-simulation lines are - therefore conditional, and a data run's prologue is byte-identical to what - it was before input_type existed. + unit of a campaign on its next ``sp run``, so lines are added only at a + campaign boundary. The image-simulation lines are conditional, so a data + prologue does not carry them. ``tile_numbers.txt`` carries the number in the form the INPUT files are named with, because get_images substitutes it verbatim into @@ -670,61 +1233,22 @@ include: "rules/tile.smk" # localrule would: a local job cannot be fused into a submitted group. The old # star-catalogue rules were exactly that, and they are gone with the internal # mask generation.) -localrules: all, prepare_all_tiles, clean_exposure, clean_tile - -# --- merged catalogue (image sims only) ------------------------------------ -# The sp_validation side starts from ONE hdf5 per branch, not 39 FITS tiles, so -# the campaign is not finished until they are merged. Image sims only: the data -# path's patches are merged separately, on a different naming convention. # -# Paths are derived from products_dir rather than from `run:`, so they hold -# whatever the run config calls things. create_final_cat.py takes a root to -# scan (-i) and a patch name (-P) that is BOTH the directory to match under it -# and the hdf5 group, so the patch directory is the branch dir and the root is -# its parent -- exactly the `-i .. -P ` the old sp_validation rule ran -# from inside the branch. -MERGE_PATCH_DIR = PRODUCTS_DIR.parent # the branch dir, holding product/tiles -MERGE_ROOT = MERGE_PATCH_DIR.parent # -i -MERGE_PATCH = MERGE_PATCH_DIR.name # -P, and the hdf5 group -MERGED_CAT = PRODUCTS_DIR / f"final_cat_{MERGE_PATCH}.hdf5" -CREATE_FINAL_CAT = Path(workflow.basedir).parent / "scripts" / "python" / "create_final_cat.py" - - -def merge_targets(): - """The merged catalogue, once every ready tile has published.""" - return [str(MERGED_CAT)] if INPUT_TYPE == "image_sims" and TILES_READY else [] - - -rule merge_final_cats: - """Merge the per-tile final catalogues into one hdf5 (image sims). - - Reads the PRODUCTS root, not the run dir: clean_tile deletes the run-dir - copies, so on a reclaimed campaign the published ones are all that is left. - Ordering needs no edge to clean_tile for the same reason. - """ - input: - [final_cat(t) for t in TILES_READY], - output: - cat = str(MERGED_CAT), - n_tiles = str(PRODUCTS_DIR / "n_tiles_final.txt"), - params: - root = str(MERGE_ROOT), - patch = MERGE_PATCH, - param_file = str(CONFIG_DIR / "final_cat.param"), - threads: 1 - resources: - mem_mb = lambda wc, attempt: 8000 * attempt, - runtime = 120 - shell: - f"python {CREATE_FINAL_CAT} -I" - " -m {output.cat} -i {params.root} -p {params.param_file}" - " -P {params.patch} -o {output.n_tiles} -v" +# exp_persist joins them for the same arithmetic — one tar of a few MB per +# exposure, ~20k of them at DR6 scale, each far shorter than the scheduling +# latency that would submit it (exposure.smk argues the placement in full). It +# sits mid-chain between exp_psf and clean_exposure, but both of those are +# outside every group already (exp_psf is heavy, clean_exposure is local), so it +# adds no new grouping constraint. +localrules: all, prepare_all_tiles, clean_exposure, clean_tile, exp_persist rule all: input: [final_cat(t) for t in TILES_READY], - merge_targets(), + persist_targets(), + star_cat_targets(), + final_cat_targets(), clean_targets(), clean_tile_targets(), diff --git a/workflow/bin/sp b/workflow/bin/sp index 7d10b988d..944595172 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -34,14 +34,13 @@ # SP_PROFILE=candide workflow/bin/sp run -c ~/my_run.yaml # a campaign # SP_PROFILE=candide workflow/bin/sp run -c ~/my_run.yaml -n # dry run # SP_PROFILE=candide workflow/bin/sp report -c ~/my_run.yaml # status now -# workflow/bin/sp run # config.yaml only # # Run config: -c FILE, also --config-file. Read after workflow/config.yaml and # merged on top; anything still unset comes from the machines: entry for -# SP_PROFILE and input_type (scripts/run_config.py). +# SP_PROFILE and input_type (workflow/scripts/run_config.py). # To check a resolved value without running: # -# python scripts/run_config.py workflow/config.yaml ~/my_run.yaml outputs.run_dir +# python workflow/scripts/run_config.py workflow/config.yaml ~/my_run.yaml outputs.run_dir # # -c is sp's flag, not snakemake's --cores; pass cores as --cores or -j. # @@ -74,7 +73,7 @@ _sp_args=() while [ $# -gt 0 ]; do case "$1" in --) - _sp_args+=("$@"); break ;; + shift; _sp_args+=("$@"); break ;; -c|--config-file) [ $# -ge 2 ] || { echo "sp: $1 needs a file" >&2; exit 2; } RUN_CONFIG="$2"; shift 2 ;; @@ -127,8 +126,17 @@ fi # Snakefile resolves it. cfg() { python "$SCRIPTS/run_config.py" "$HERE/config.yaml" "${SP_RUN_CONFIG:-}" "$1"; } RUN_DIR="$(cfg outputs.run_dir)"; INDEX_DB="$(cfg outputs.index_db)" -if [ "${1:-}" != container ] && { [ -z "$RUN_DIR" ] || [ "$RUN_DIR" = TBD ]; }; then - echo "sp: outputs.run_dir is unset or TBD for this machine/input_type" >&2; exit 2 +# `sp container`, `sp cancel` and the help forms (bare `sp`, -h, --help) need +# neither a campaign nor its state dir; every other verb does. Checked here, before the state dir is created from +# RUN_DIR: with `run:` unset, RUN_DIR still holds a literal `$run`. +case "${1:-}" in container|cancel|""|-h|--help) NEEDS_CAMPAIGN=0 ;; *) NEEDS_CAMPAIGN=1 ;; esac +if [ "$NEEDS_CAMPAIGN" = 1 ]; then + if [ -z "$(cfg run)" ]; then + echo "sp: \`run:\` is unset; the run config (-c FILE) names the campaign" >&2; exit 2 + fi + if [ -z "$RUN_DIR" ] || [ "$RUN_DIR" = TBD ] || [[ "$RUN_DIR" == *'$'* ]]; then + echo "sp: outputs.run_dir is unset, TBD or unexpanded ($RUN_DIR) for this machine/input_type" >&2; exit 2 + fi fi # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO @@ -141,7 +149,8 @@ fi # the placement is right on its own merits.) # --directory only moves state: all data paths are absolute, and the Snakefile # resolves its own configfile. -STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}" +[ "$NEEDS_CAMPAIGN" = 0 ] || mkdir -p "$STATE_DIR" # --- the launch code snapshot ---------------------------------------------- # THE ONE HOME for this concept; everything else points here. @@ -157,8 +166,10 @@ STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and # runs the campaign entirely out of that copy: the Snakefile, the rules, the # scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points -# into example/, and the copy must be self-contained), src/, the profile, and -# the repo's top-level scripts/ (576 KB), which merge_final_cats runs out of. +# into example/, and the copy must be self-contained), src/, the repo's own +# scripts/ (final_cat_merge loads scripts/python/create_final_cat.py by path -- +# it is a script, not an installed module, and the hdf5 layout it defines must +# be pinned to the campaign like everything else here), and the profile. # Every workflow-internal path hangs off `workflow.basedir`, which IS the # snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the # one that cannot (YAML splices nothing) and is rewritten below. @@ -177,10 +188,10 @@ snapshot_code() { mkdir -p "$SNAPSHOT" if command -v rsync >/dev/null 2>&1; then rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ - "$HERE" "$REPO/src" "$REPO/profiles" "$REPO/scripts" "$SNAPSHOT/" + "$HERE" "$REPO/src" "$REPO/scripts" "$REPO/profiles" "$SNAPSHOT/" else rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" - cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$REPO/scripts" "$SNAPSHOT/" + cp -rL "$HERE" "$REPO/src" "$REPO/scripts" "$REPO/profiles" "$SNAPSHOT/" find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + fi @@ -362,6 +373,10 @@ case "$cmd" in | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; + ""|-h|--help) + # snakemake's own usage: parsing the Snakefile would need a campaign. + snakemake --help + ;; *) sm "$@" ;; diff --git a/workflow/config.yaml b/workflow/config.yaml index a220a4706..95a0f3d88 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -21,8 +21,9 @@ input_type: data # shadows that table. The Snakefile falls back to psfex if no entry # supplies one. -# Run name, available as `$run` in the paths below. -run: smk-g6 +# Run name, available as `$run` in the paths below; it also names the +# campaign's merged catalogues. Required, and set by the run config. +# run: smk-g6 # Entries per machine (SP_PROFILE, default nibi; implemented: nibi, candide) # and input_type. A run config may instead state `machine:`, which must @@ -32,7 +33,8 @@ run: smk-g6 # - retrieve: method to get input data, allowed are symlink, vos # - inputs.tiles,, .exposures: path for input tile and exposure # - outputs.run_dir: (scratch) run directory, where tmp files will be stored -# - outputs.products_dir: path to final products +# - outputs.products_dir: path to final products (default run_dir; one root +# needs clean: false and clean_tiles: false) # - outputs.index_db: path to bookkeeping index sqlite file # - container: TBD @@ -75,6 +77,101 @@ machines: psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif +# OPTIONAL per-exposure retention: what to carry onto the persistent root ON TOP +# OF the star catalogue's own inputs, before the scratch store goes +# (`exp_persist`, exposure.smk). +# +# WHAT IS ALWAYS KEPT, AND IS NOT A CHOICE HERE: psf_validation, the psfex_interp +# validation catalogue, one per CCD. `star_cat_merge` stacks every one of them +# into /full_starcat_.hdf5, so they are that +# catalogue's PROVENANCE — a merged star catalogue with no per-exposure inputs +# beside it cannot be audited, re-cut, or recomputed after a purge — and they are +# what keeps APPENDING TILES CHEAP, since a tile added next month brings +# exposures whose catalogues must join the existing stack. ~2 MB per exposure: +# ~40 GB and ~40k inodes at DR6 scale, against a ~1 M-inode group quota. That is +# the price of being able to say where the number came from, and it is paid. +# +# SO THIS LIST IS PURELY ADDITIVE, and an empty one is a coherent instruction: +# the tar then holds the star catalogue's inputs and nothing else. +# +# Entries are PRODUCT NAMES — not globs. The catalogue below is the rendering of +# workflow/scripts/persist_exp.py's PRODUCTS table, which is the single source of +# truth for what each name means and what keeping it buys +# (CosmoStat/shapepipe#844); print it any time with +# +# workflow/bin/sp container exec python workflow/scripts/persist_exp.py --list-products +# +# product glob size/exposure +# -------------- -------------------------- ------------- +# star_selection star_selection-*.fits 24.5 MB +# setools' PRE-SPLIT selection. The only file that answers which +# stars the selection cuts rejected and why; the split samples have +# already lost the rejects. +# star_train star_split_ratio_80-*.fits 19.9 MB +# the 80% TRAINING sample, the stars PSFEx actually fitted. Rows +# duplicate star_selection. +# star_test star_split_ratio_20-*.fits 7.1 MB +# the 20% VALIDATION sample — the positions psf_validation's rows +# correspond to. Rows duplicate star_selection. +# star_stats star_stat-*.txt unmeasured +# setools' per-CCD STAT block: star counts, stars/deg^2, FWHM mode +# and cuts. The selection's summary without its catalogue. +# psf_model *.psf 2.8 MB +# the PSFEx model itself. Keeping it means the PSF can be +# re-interpolated at ANY position later without rebuilding the +# exposure chain — the single most capability-adding entry here. +# psfex_cat psfex_cat-*.cat unmeasured +# PSFEx's own output catalogue (FITS_LDAC): the per-star FLAGS_PSF +# and CHI2_PSF, i.e. WHICH stars outlier rejection clipped. Not +# recoverable from anything else — the .psf header keeps only the +# LOADED/ACCEPTED counts. +# psf_validation validation_psf-*.fits 2.0 MB +# the psfex_interp validation catalogue, one per CCD: the input to +# the rho/tau statistics, and to the star_cat_merge rule that stacks +# them into the campaign's full_starcat. +# +# A RAW GLOB IS STILL ACCEPTED, as an escape hatch for a file the catalogue does +# not name yet: anything carrying a glob metacharacter or a dot is taken as a +# glob rather than a name (`*.psf` is a glob, `psf_model` is the name for it). +# An unknown NAME is a parse-time error listing the valid ones. +# +# Matches are packed, flat, into ONE uncompressed tar per exposure: +# /exp///psf/.tar, with a manifest listing the +# members (and the product each came from) beside it. One tar rather than loose +# copies because inodes, not bytes, bind on /project. FITS members read straight +# from the tar: fits.open(io.BytesIO(tarfile.open(t).extractfile(m).read())). +# +# WHY COPY RATHER THAN EXEMPT THESE FROM CLEANUP. Reclamation is not the threat. +# run_dir is /scratch and is PURGED on a 60-day window whether or not +# clean_exposure ever ran; products_dir is /project, backed up and not purged. +# The only way a per-exposure product outlives its campaign is to leave the +# filesystem. (Ordering is free: clean_exposure takes the exp_persist manifest +# as an input, so a store is never reclaimed before its keepers are written.) +# +# EDITING THIS LIST IS CHEAP. It rides on exp_persist's `params`, so a change +# reruns the packing (seconds) and NOT exp_psf (four hours per exposure). That +# separation is the whole reason exp_persist is a rule of its own. +# +# THE DEFAULT is psf_model (2.8 MB per exposure on top of psf_validation's 2.0): +# the model that lets the PSF be re-interpolated at any position later without +# rebuilding the exposure chain from VOS, which is the single most +# capability-adding thing an exposure can keep. Add psfex_cat for a production +# run if you want to know which stars PSFEx clipped; the star_* products are for +# selection studies and cost an order of magnitude more. +# +# THIS LIST IS EXPOSURE-SIDE ONLY. Tile-side retention is not configurable: the +# only tile product that persists today is final_cat, written by tile_make_cat +# straight to products_dir. A tile keep list is #844 follow-up. +# +# NOTE ON products_dir DEFAULTING TO run_dir (a fixture or smoke test): the tar +# then lands beside the store on the same filesystem and buys nothing, and +# final_cat and the persist manifests sit inside the trees clean_tile and +# clean_exposure delete. A one-root run is for fixtures and smoke tests only and +# must set clean: false and clean_tiles: false; the Snakefile refuses it +# otherwise. +persist_exp: + - psf_model + # Per-exposure store reclamation once every tile reading it has its vignets. clean: true @@ -87,6 +184,16 @@ clean_tiles: true # Their exposures are rebuilt from scratch if the tile is retried. clean_ignore_tiles: [] +# The ceiling on any rule's mem_mb: nibi's, whose standard compute node is +# 766 GB (192 cores at 4 GB/core), so 750000 leaves room for the OS and slurm's +# own overhead. A candide run never reaches it. A request above the partition +# maximum is a job SLURM never schedules and snakemake never diagnoses: it sits +# PENDING while the campaign looks alive. The two +# campaign-level merges size themselves from the campaign's bytes and will cross +# this at survey scale — the cap turns "never runs" into "runs on the biggest +# node there is", with a parse-time warning saying which rule was capped. +max_mem_mb: 750000 + # ngmix chunks per tile. ngmix_chunks: 8 diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 6ebd1aae9..552732201 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -8,11 +8,32 @@ YWIN_WORLD # @sc [decision:catalogue_assembly.tile_overlap_handling] TILE_ID +# SExtractor object number within the tile: sp_validation's extraction joins +# on it, and the per-tile catalogue always carries it. +NUMBER + # flags FLAGS -# @sc [decision:detection.detection_source_mode] -IMAFLAGS_ISO - +# NO IMAFLAGS_ISO, AND NO MASK COLUMN AT ALL — READ THIS BEFORE ADDING ONE. +# The tile-side SExtractor runs with FLAG_IMAGE = False and DOT_PARAM_FILE = +# default_noimaflags.param (config_tile_Sx.ini), so IMAFLAGS_ISO is never +# written into a tile catalogue and asking for it here only made the merge +# fail. Instrument flags reach the pipeline on the EXPOSURE side, where +# exp_split delivers the flag image and SExtractor reads it. +# +# Its intended replacement is make_cat's per-band MASK_ columns, queried +# from the sky-fixed healsparse maps named by MASK_EXT_PATHS. THE WORKFLOW SETS +# NO SUCH PATHS: config_tile_Mc.ini has no MASK_EXT_PATHS entry, so +# save_mask_ext_data is never called, no MASK_ column exists in any tile +# catalogue this workflow has produced, and smk-g6's carry none (checked). +# Naming one here would fail every merge on every campaign. +# +# So the merged catalogue carries NO mask information today, and that is a +# CONFIG gap and not a gap in this file: turning it on is setting +# MASK_EXT_PATHS in config_tile_Mc.ini (`band:path` pairs, the same grammar as +# the commented MASK_PATHS in config_exp_psfex.ini) and adding the matching +# MASK_ names here, in that order. No healsparse map is staged under +# /project/def-mjhudson yet. NGMIX_MCAL_FLAGS # PSF ellipticity (original image PSF) @@ -30,6 +51,119 @@ NGMIX_G2_PSF_ORIG_NOSHEAR N_EPOCH NGMIX_N_EPOCH +# Per-epoch identity and PSF shape, slot n = 1..12 (config_tile_Mc.ini's +# N_EPOCH_SLOTS): EXP_ID_n/CCD_n name the exposure and CCD of epoch n, aligned +# with HSM_*_PSF_n; empty slots carry -1 (ids), -10 (g, M4), 0 (T), 1 (flag), +# -1 (RHO4). The count is fixed at the source so every tile shares this schema. +EXP_ID_1 +EXP_ID_2 +EXP_ID_3 +EXP_ID_4 +EXP_ID_5 +EXP_ID_6 +EXP_ID_7 +EXP_ID_8 +EXP_ID_9 +EXP_ID_10 +EXP_ID_11 +EXP_ID_12 +CCD_1 +CCD_2 +CCD_3 +CCD_4 +CCD_5 +CCD_6 +CCD_7 +CCD_8 +CCD_9 +CCD_10 +CCD_11 +CCD_12 +HSM_G1_PSF_1 +HSM_G1_PSF_2 +HSM_G1_PSF_3 +HSM_G1_PSF_4 +HSM_G1_PSF_5 +HSM_G1_PSF_6 +HSM_G1_PSF_7 +HSM_G1_PSF_8 +HSM_G1_PSF_9 +HSM_G1_PSF_10 +HSM_G1_PSF_11 +HSM_G1_PSF_12 +HSM_G2_PSF_1 +HSM_G2_PSF_2 +HSM_G2_PSF_3 +HSM_G2_PSF_4 +HSM_G2_PSF_5 +HSM_G2_PSF_6 +HSM_G2_PSF_7 +HSM_G2_PSF_8 +HSM_G2_PSF_9 +HSM_G2_PSF_10 +HSM_G2_PSF_11 +HSM_G2_PSF_12 +HSM_T_PSF_1 +HSM_T_PSF_2 +HSM_T_PSF_3 +HSM_T_PSF_4 +HSM_T_PSF_5 +HSM_T_PSF_6 +HSM_T_PSF_7 +HSM_T_PSF_8 +HSM_T_PSF_9 +HSM_T_PSF_10 +HSM_T_PSF_11 +HSM_T_PSF_12 +HSM_FLAG_PSF_1 +HSM_FLAG_PSF_2 +HSM_FLAG_PSF_3 +HSM_FLAG_PSF_4 +HSM_FLAG_PSF_5 +HSM_FLAG_PSF_6 +HSM_FLAG_PSF_7 +HSM_FLAG_PSF_8 +HSM_FLAG_PSF_9 +HSM_FLAG_PSF_10 +HSM_FLAG_PSF_11 +HSM_FLAG_PSF_12 +HSM_M4_1_PSF_1 +HSM_M4_1_PSF_2 +HSM_M4_1_PSF_3 +HSM_M4_1_PSF_4 +HSM_M4_1_PSF_5 +HSM_M4_1_PSF_6 +HSM_M4_1_PSF_7 +HSM_M4_1_PSF_8 +HSM_M4_1_PSF_9 +HSM_M4_1_PSF_10 +HSM_M4_1_PSF_11 +HSM_M4_1_PSF_12 +HSM_M4_2_PSF_1 +HSM_M4_2_PSF_2 +HSM_M4_2_PSF_3 +HSM_M4_2_PSF_4 +HSM_M4_2_PSF_5 +HSM_M4_2_PSF_6 +HSM_M4_2_PSF_7 +HSM_M4_2_PSF_8 +HSM_M4_2_PSF_9 +HSM_M4_2_PSF_10 +HSM_M4_2_PSF_11 +HSM_M4_2_PSF_12 +HSM_RHO4_PSF_1 +HSM_RHO4_PSF_2 +HSM_RHO4_PSF_3 +HSM_RHO4_PSF_4 +HSM_RHO4_PSF_5 +HSM_RHO4_PSF_6 +HSM_RHO4_PSF_7 +HSM_RHO4_PSF_8 +HSM_RHO4_PSF_9 +HSM_RHO4_PSF_10 +HSM_RHO4_PSF_11 +HSM_RHO4_PSF_12 + # Blend flag: coadd seg stamp held a non-central footprint (shapepipe#776) NGMIX_NEIGHBOUR_FLAG @@ -117,5 +251,6 @@ NGMIX_T_PSF_ORIG_NOSHEAR # PSF size measured on reconvolved image # NGMIX_T_PSF_RECONV_NOSHEAR -# ngmix moment failure flag -NGMIX_MOM_FAIL +# ngmix metacalibration type failure flag (renamed from NGMIX_MOM_FAIL in +# f0fca23e; catalogues written before that commit carry the old name) +NGMIX_MCAL_TYPES_FAIL diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 5bb0bdd17..40c7c9faa 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -1,9 +1,8 @@ # Final-catalogue column selection for the image simulations. # -# create_final_cat.py -I selects exactly these columns from each simulated -# tile's make_cat output (tiles///output/run_sp_tile_Mc/...) into -# final_cat_{sim}.hdf5, and sp_validation's extract step reads the same list; -# every entry must exist in that catalogue. +# final_cat_merge selects exactly these columns from each simulated tile's +# final catalogue into /final_cat_.hdf5, and sp_validation's +# extract step reads the same list; every entry must exist in that catalogue. # # A real file in this overlay dir, not a symlink into ../cfis/, because the # real-data list names columns make_cat does not write for the simulations: diff --git a/workflow/config/cfis_image_sims/gauss_3.0_7x7.conv b/workflow/config/cfis_image_sims/gauss_3.0_7x7.conv new file mode 120000 index 000000000..25e80a4b4 --- /dev/null +++ b/workflow/config/cfis_image_sims/gauss_3.0_7x7.conv @@ -0,0 +1 @@ +../cfis/gauss_3.0_7x7.conv \ No newline at end of file diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 637c1dc15..221d10928 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,6 +1,6 @@ """Exposure chain — per exposure, keyed by exp base id (dedup is structural). - exp_get_images -> exp_split -> exp_psf + exp_get_images -> exp_split -> exp_psf -> exp_persist Each in the exposure's own sharded work dir, chained by manifests; every config reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a @@ -19,6 +19,13 @@ opt-in, see ``star_selection.setools``), and ``make_cat`` writes the per-band star catalogue, or a network fetch — hence no ``star_catalogue`` / ``exp_star_cat`` here, and no ``exp_mask``. +``exp_persist`` is the one rule here that writes to the PERSISTENT root: it +packs the PSF products named by `persist_exp:` into one tar per exposure off +/scratch before the purge (or clean_exposure) can take them. It is a separate +rule from exp_psf precisely so that editing that list costs a re-pack and not a +four-hour refit; the full +argument is in workflow/scripts/persist_exp.py. + NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, not over one invocation — reclamation here is clean_exposure's job (S5), driven @@ -115,6 +122,62 @@ rule exp_psf: sp_shell("exp_psf", f"config_exp_{PSF_MODEL}.ini") +# --- persistence (D5) ------------------------------------------------------- +# The counterpart of reclamation, and it must come first in the DAG: this packs +# the exposure's keepable PSF products into one tar on the persistent root, and +# clean_exposure below takes its manifest as an input so the store is never +# reclaimed before the keepers have left /scratch. The purge would take them +# anyway — that, not clean_exposure, is what this rule exists for +# (persist_exp.py's docstring argues both halves, and config.yaml's +# `persist_exp:` block carries the keep list and its candidates). +# +# A LOCALRULE (declared in the Snakefile), by exactly the arithmetic that made +# clean_exposure one: the body is a `tar` of a few MB from one shared filesystem +# to another, seconds of work, and one sbatch per exposure would be ~20k +# submissions at DR6 scale for jobs shorter than the scheduling latency. The +# grouping constraint that binds mid-chain localrules (this file's docstring) +# does not bite here: exp_persist's only neighbours are exp_psf, which is too +# heavy to ever fuse, and clean_exposure, which is local itself. +# +# ONE DECLARED OUTPUT, AND IT IS A MANIFEST, NOT THE TAR OR A directory(). The +# tar is not declared: a directory output would attest that a directory exists, +# where what we want written down is WHICH files were packed and how big each was — +# the provenance a rho-statistics run months from now needs in order to know +# what it is reading. The manifest is byte-stable, so a no-op rerun does not +# move its mtime and does not make clean_exposure look out of date. +# +# THE KEEP LIST RIDES ON params. That is the entire reason this is not three +# lines of tar appended to exp_psf's shell: `params` is a rerun trigger, so +# adding a pattern reruns the packing and leaves the PSF chain alone. +rule exp_persist: + input: + rules.exp_psf.output.manifest + output: + manifest = f"{PROD_EXP_DIR}/manifests/exp_persist.json" + # No `log:`: the script's only failure modes are "nothing matched" and a + # name collision, both of which it reports on stderr and neither of which + # has a per-CCD verdict worth a completeness record. + params: + # Only the OPTIONAL retention list travels: psf_validation is packed + # by persist_exp.py whatever this says. It still rides on params, so + # adding a product re-packs (seconds) rather than re-fitting the PSF. + patterns = " ".join(f"--pattern '{p}'" for p in PERSIST_EXP), + exp_dir = lambda wc: exp_dir(wc.exp), + dest = lambda wc: f"{prod_exp_dir(wc.exp)}/psf", + script_hash = PERSIST_HASH + threads: 1 + retries: 2 + resources: + mem_mb = 2000, + runtime = 10 + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/persist_exp.py" + " --exp-dir '{params.exp_dir}' --exp {wildcards.exp}" + " --dest '{params.dest}' --manifest {output.manifest}" + " {params.patterns}" + + # --- reclamation (D5) ------------------------------------------------------- # The one exception to "no reclamation in this file": clean_exposure OWNS # exposure-level deletion, and it is a real job, not temp() bookkeeping, because @@ -146,7 +209,26 @@ rule clean_exposure: # spatial neighbours. In-scope consumers keep their edge: they may run in # this DAG, so the clean must be ordered after them. lambda wc: [tile_manifest(t, "tile_vignets") - for t in clean_consumers(wc.exp) if t in READY_SET] + for t in clean_consumers(wc.exp) if t in READY_SET], + # The keepers must be off /scratch before the store goes. No keep list + # removes this edge: exp_persist always packs the star catalogue's + # inputs. Only psf_model=fake does, which has no PSF products to keep + # (PERSISTS_PSF, Snakefile). + # + # A LIVE exposure is asked for its exp_persist manifest: the thing to + # build, and what orders this rule after the pack. A RECLAIMED one + # (exp_store_reclaimed, Snakefile) is asked for its TAR if it has one — + # on the persistent root, no rule's declared output, hence a leaf that + # requires nothing — and for nothing if it has none. Naming the + # manifest there reopens the reclaimed chain: a `persist_exp:` edit + # changes exp_persist's params, the manifest reruns, and it sits behind + # exp_psf's manifest, which went with the store, so snakemake rebuilds + # the exposure from VOS. + lambda wc: ([] if not PERSISTS_PSF + else [prod_exp_manifest(wc.exp, "exp_persist")] + if not exp_store_reclaimed(wc.exp) + else [prod_exp_tar(wc.exp)] + if Path(prod_exp_tar(wc.exp)).exists() else []) output: tombstone = f"{EXP_DIR}/cleaned.json" params: @@ -160,3 +242,86 @@ rule clean_exposure: f"python {SCRIPTS}/clean_exposure.py" " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" " --tombstone {output.tombstone} --consumers '{params.consumers}'" + + +# --- the campaign's star catalogue ------------------------------------------ +# ONE job per campaign: every exposure's every CCD's `validation_psf--.fits`, +# collected into `/full_starcat_.hdf5`, one dataset per +# exposure. That file is the rho/tau statistics input; the old bash chain built +# a flat FITS table with `combine_runs.bash psf` + a `merge_starcat_runner` +# pass, and the workflow emitted neither. sp_validation still opens the FITS +# name today — CosmoStat/sp_validation#340 moves its readers to this file, the +# same migration that retires the `patches/` key on the tile side. +# +# ONE DATASET PER EXPOSURE, NOT ONE TABLE, and it is the same decision as the +# tile side's: it makes the file RECONCILABLE. A flat table had to be restacked +# from every exposure the campaign had ever seen to add one — ~40 GB of members +# at DR6 scale to add ~2 MB — and held the whole campaign in memory while it did +# so. Reconciled, an append reads the appended exposures and nothing else, and +# the job holds one exposure at a time. hdf5_reconcile.py is the shared +# machinery; merge_star_cat.py argues the format and the tar reading. +# +# THE INPUT IS star_cat_inputs() (Snakefile): every exposure of TILES_READY whose +# PSF products are on the persistent root — the live ones through the exp_persist +# manifest edge `rule all` already requests, the RECLAIMED ones through their TAR, +# which no rule declares and which therefore requires nothing to be built. That +# asymmetry is not a flourish; requesting a reclaimed exposure's manifest +# rebuilds its whole chain from VOS, and ancient() does not prevent it (measured +# — the Snakefile carries the numbers). Nothing new enters the DAG either way. It +# is read through an INPUT FUNCTION rather than at module level so that only a +# parse which actually builds this job pays for the walk. +# +# THE PATHS DO NOT REACH THE SHELL, and that is not a style choice: ~20k manifest +# paths is an order of magnitude over Linux's 128 KiB MAX_ARG_STRLEN for a single +# argv entry, so `{input}` here would be a job that dies on exec at DR6 scale. +# The job is handed the two small files the Snakefile itself started from — the +# tile list and the index — and derives THE SAME SET from them; `params.inputs` +# carries that set's FINGERPRINT, which is the rerun trigger. The equality is +# the point: a job that stacked anything the fingerprint did not see would be +# rows no rerun trigger could notice, which is what a glob over products_dir +# would have given on a root shared with an earlier, larger tile list. +# +# NOT A LOCALRULE. exp_persist is local because it is 20k jobs of seconds; this +# is one job that reads the campaign's tars end to end. Its MEMORY is flat in +# the campaign (one exposure at a time) and sized on the largest exposure; its +# RUNTIME is the total. +# +# NO JOB AT ALL when every exposure in scope is tombstoned with no tar left +# behind: star_cat_targets() (Snakefile) simply does not request the output. +rule star_cat_merge: + input: + lambda wc: star_cat_inputs() + output: + star_cat = full_starcat() + params: + products_dir = str(PRODUCTS_DIR), + tile_list = str(config["tile_list"]), + index_db = str(INDEX_DB), + campaign = CAMPAIGN, + snapshot = str(SNAPSHOT_JSON), + inputs = unit_fingerprint(star_cat_exposures()), + script_hash = MERGE_STAR_HASH + threads: 1 + resources: + # Sized on the campaign's own member bytes, slope and intercept + # measured (the Snakefile's sizing block carries both points, and the + # ceiling this rule runs into at DR6 scale). Still * attempt, because a + # measured slope on synthetic tars is not a guarantee about real ones. + # Sized on the LARGEST exposure, not the total: the merge holds one + # exposure at a time (the Snakefile's sizing block carries the history). + mem_mb = lambda wc, attempt: capped_mem(attempt * ( + STAR_MEM_BASE_MB + + STAR_MEM_FACTOR * star_cat_max_bytes() // 1_000_000), + "star_cat_merge"), + # ~2 min per GB of members on the measurement above, doubled, over a + # floor that covers the fixed cost of opening ~40 members per exposure. + runtime = lambda wc, attempt: attempt * ( + 30 + 4 * star_cat_bytes() // 1_000_000_000) + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/merge_star_cat.py" + " --products-dir '{params.products_dir}'" + " --tile-list '{params.tile_list}' --index-db '{params.index_db}'" + " --output {output.star_cat}" + " --campaign '{params.campaign}'" + " --snapshot-json '{params.snapshot}'" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index ccbc83fbf..8f2e21f24 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -913,3 +913,79 @@ rule clean_tile: f"python {SCRIPTS}/clean_tile.py" " --tile-dir $(dirname {output.tombstone}) --tile {wildcards.tile}" " --tombstone {output.tombstone}" + + +# --- the campaign's shear catalogue ----------------------------------------- +# ONE job per campaign, the tile-side twin of exposure.smk's star_cat_merge, and +# the same three design calls hold: the input is the list `rule all` already +# requests (every ready tile's final_cat), the paths never reach the shell +# (MAX_ARG_STRLEN), and a fingerprint on `params` is what makes it rerun when a +# tile is appended. The job derives the same set the fingerprint was taken over +# from the tile list and the index rather than globbing products_dir — on a +# products root shared with an earlier, larger tile list a glob would merge tiles +# no rerun trigger ever saw. +# +# THE OUTPUT SCHEMA IS AN INTERFACE, NOT A CHOICE. sp_validation opens this file +# as its `galaxy_cat_path`: one dataset per tile under a named group, the +# columns of CONFIG_DIR's final_cat.param, an `n_tiles` attribute on the +# root. The group is named for the CAMPAIGN, which is the only unit this +# workflow has above the tile. It is the merger for both input types: an +# image-sims campaign reads config/cfis_image_sims/final_cat.param, whose +# columns are what sp_validation's image-sims extract step reads. So the rule reuses +# scripts/python/create_final_cat.py's column extraction rather than restating +# it, and writes the file itself — merge_final_cat.py argues that split, the one +# legacy literal in the schema, and the two places where the reference +# implementation had to be pinned down to be reproducible. +# +# THE INPUT IS final_cat, NOT the tile_make_cat manifest, for the same reason +# clean_tile's is: final_cat on the persistent root IS the campaign's +# tile-finished marker (see final_cat() in the Snakefile), and it is the file +# this rule actually reads. +# +# NOT A LOCALRULE, and here the reason is IO rather than memory: a first build +# reads every tile's catalogue end to end — ~32-46 MB per tile, so ~2 GB for a +# 64-tile campaign and ~800 GB at DR6's 23k tiles. It RECONCILES rather than +# rebuilds or appends: a tile with no dataset is added, a dataset whose tile +# left the campaign is deleted, a dataset whose source catalogue changed is +# re-read, and one that agrees with its source is left alone. So an append +# reads the appended tiles and nothing else, while the file still cannot drift +# from its inputs the way an append-only tool does (merge_final_cat.py argues +# what is and is not a function of the input set here). Memory is one tile's +# catalogue at a time plus the hdf5 write buffer, which is why mem_mb is modest +# where star_cat_merge's is not — and why runtime, which is sized on the whole +# campaign, is the pessimistic first-build case. +rule final_cat_merge: + input: + lambda wc: [final_cat(t) for t in TILES_READY] + output: + merged = final_cat_hdf5() + params: + products_dir = str(PRODUCTS_DIR), + tile_list = str(config["tile_list"]), + index_db = str(INDEX_DB), + param_file = str(CONFIG_DIR / "final_cat.param"), + campaign = CAMPAIGN, + snapshot = str(SNAPSHOT_JSON), + inputs = unit_fingerprint(TILES_READY), + script_hash = MERGE_FINAL_HASH + threads: 1 + resources: + # Sized on the LARGEST tile, not the total: the merge holds one + # catalogue at a time, and the measurement is flat in the tile count + # (the Snakefile's sizing block carries both points). + mem_mb = lambda wc, attempt: capped_mem(attempt * ( + FINAL_MEM_BASE_MB + + FINAL_MEM_FACTOR * final_cat_max_bytes() // 1_000_000), + "final_cat_merge"), + # Runtime, unlike memory, is the TOTAL: every tile is read end to end. + # ~1 min per 10 tiles on the measurement, triply generous, over a floor. + runtime = lambda wc, attempt: attempt * (30 + len(TILES_READY) // 3) + shell: + "set -euo pipefail\n" + f"python {SCRIPTS}/merge_final_cat.py" + " --products-dir '{params.products_dir}'" + " --tile-list '{params.tile_list}' --index-db '{params.index_db}'" + " --output {output.merged}" + " --campaign '{params.campaign}'" + " --param-file '{params.param_file}'" + " --snapshot-json '{params.snapshot}'" diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 2dd191306..ea83e9048 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -125,6 +125,10 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: if tile_id in missing_set: + # Not ready (ready_tiles reads the tiles table), but its edges stay: + # clean_exposure's consumer sets must still see a tile that read an + # exposure. + con.execute("DELETE FROM tiles WHERE tile_id = ?", (tile_id,)) continue ra_dir = tile_id.split(".")[0] exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) @@ -152,6 +156,68 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, "n_missing": len(missing)} +# --- reading it back, for the campaign-level merges ------------------------- +# The Snakefile loads this index into dicts at parse time and derives the +# campaign's unit sets from them (TILES_READY, and the exposures those tiles +# read). A merge JOB has to derive the same two sets, and cannot be handed them +# on its command line — ~20k paths is an order of magnitude over Linux's 128 KiB +# MAX_ARG_STRLEN for a single argv entry. So it is given the two things the +# Snakefile itself started from, the tile list and this database, and rebuilds +# the sets here. Both halves therefore read the schema through one module rather +# than two hand-written queries that could drift apart. + + +def ready_tiles(db_path: Path) -> set[str]: + """Tiles whose exposure list the last build over them found non-empty. + + Readiness is the ``tiles`` row, which a build removes when the list goes + missing; the edges in ``tile_exposures`` outlive it as cleanup consumers, + so they alone do not make a tile ready. ``n_exp`` is the number of edges + the build wrote beside the row. Only the small ``tiles`` table is read, + since every job's parse of the Snakefile calls this. + """ + con = sqlite3.connect(db_path, timeout=60) + ready = {r[0] for r in con.execute( + "SELECT tile_id FROM tiles WHERE n_exp > 0")} + con.close() + return ready + + +def campaign_tiles(tile_list: Path, db_path: Path) -> list[str]: + """The campaign's ready tiles: declared in the list AND ready_tiles(). + + Exactly the Snakefile's TILES_READY, computed the same way from the same two + files — a declared tile with no current exposure list cannot have been + computed, so it has no catalogue to merge. + """ + # DEDUPED, order preserved. The tile list is appended to by hand across a + # campaign, so a tile can appear twice; a merge would then try to write that + # tile's dataset twice and die on the second. Deduping here rather than at + # the call sites keeps the answer the same for every reader of the index. + seen, declared = set(), [] + with open(tile_list) as f: + for line in f: + tile = line.strip() + if tile and tile not in seen: + seen.add(tile) + declared.append(tile) + ready = ready_tiles(db_path) + return [t for t in declared if t in ready] + + +def campaign_exposures(tile_list: Path, db_path: Path) -> list[str]: + """Every exposure the campaign's ready tiles read, sorted. + + Exactly the set the Snakefile's persist_manifests() builds its manifest + paths from. + """ + tiles = set(campaign_tiles(tile_list, db_path)) + con = sqlite3.connect(db_path, timeout=60) + rows = con.execute("SELECT tile_id, exp_id FROM tile_exposures").fetchall() + con.close() + return sorted({e for t, e in rows if t in tiles}) + + def main() -> None: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--tile-list", required=True, type=Path, diff --git a/workflow/scripts/hdf5_reconcile.py b/workflow/scripts/hdf5_reconcile.py new file mode 100644 index 000000000..7bf51c6ae --- /dev/null +++ b/workflow/scripts/hdf5_reconcile.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Bring an hdf5 catalogue into agreement with a campaign, one dataset per unit. + +Shared by the two campaign-level merges — ``merge_final_cat.py`` (one dataset +per tile) and ``merge_star_cat.py`` (one per exposure) — because they want +exactly the same thing of their output and disagreeing about it would be a bug +waiting to happen rather than a difference worth having. + +WHY RECONCILE RATHER THAN REBUILD. The output must be a function of the input +set — that is what makes the rules' fingerprints mean anything — but reading +every unit to add one is ~800 GB of IO at DR6 scale for a few tens of MB of new +data. So the file is brought INTO AGREEMENT with the campaign instead: + + * a unit with no dataset is read and added; + * a dataset whose unit has left the campaign is deleted; + * a dataset whose SOURCE has changed is re-read. Each records its source's + size and mtime as attributes, and a mismatch is what changed means. This is + the only reason a finished unit is read twice, and it is why the file + cannot drift from its inputs the way an append-only tool does; + * a dataset whose column set was written under a DIFFERENT SCHEMA is re-read. + The column set is the one input nothing else can see: it is not a source + file, so no stamp moves when it changes. It travels as a digest on the + file's root. + * a dataset that agrees with its source and its schema is left alone, unread. + +ONE TYPE PER COLUMN. The digest covers names only, because no merge knows its +column types before reading: they come from the sources. So `apply` checks +types as it reads. A unit whose column types differ from the datasets it would +keep means the reader changed under them, and every unit is re-read. If the +sources themselves still disagree, the merge refuses, naming the column and +both types; concatenating them would silently promote one. + +An append therefore READS exactly the appended units. It still WRITES the whole +file: the existing one is copied so the result can be moved into place +atomically, which costs one pass over it and, briefly, twice its size on disk. +That is the cheap half by orders of magnitude — copying a 1 GB hdf5 against +re-reading 800 GB of catalogues — but it is not free, and `apply` refuses rather +than filling the filesystem when the free space is not there. + +WHAT IS AND IS NOT A FUNCTION OF THE INPUT SET. The file's CONTENT is: the same +units with the same sources give the same datasets, the same columns and the +same count attribute, whether they arrived at once or one batch at a time. Its +BYTE LAYOUT is not, because hdf5 lays a group out in the order things were +added. That is the trade for not re-reading the campaign, and it is why the +no-op case compares ACTIONS rather than bytes. + +UNTOUCHED ON A NO-OP, which is stronger than byte-stable and cheaper to +establish. Reconciling is PLANNED against a read-only open; an empty plan never +opens the file for writing, so its mtime cannot move — and mtime is a rerun +trigger, so an unconditional rewrite would make every invocation look like a +change. +""" + +import hashlib +import json +import shutil +import sys +from pathlib import Path + +import h5py + + +def schema_digest(columns) -> str: + """A fingerprint of the COLUMN SET the datasets were written with.""" + return hashlib.md5("\n".join(columns).encode()).hexdigest()[:16] + + +def code_provenance(snapshot_json) -> dict: + """The launch code's identity, to stamp onto the merged file's root. + + ``snapshot_json`` is ``sp run``'s code snapshot (``bin/sp``'s + ``$STATE_DIR/code/snapshot.json``), passed through by the calling rule. A + workflow driven outside ``sp run`` has no such file — the merge still + succeeds, and the caller writes ``code_head = "unknown"`` rather than + failing an otherwise-good build. + """ + if not snapshot_json or not Path(snapshot_json).exists(): + return {"head": "unknown"} + data = json.loads(Path(snapshot_json).read_text()) + out = {k: data[k] for k in ("head", "branch", "dirty", "taken_at") + if k in data} + if data.get("dirty") and data.get("dirty_files"): + out["dirty_files"] = data["dirty_files"] + return out + + +def stamp(path: Path) -> tuple: + """A source's identity, as recorded on the dataset built from it. + + Size and mtime, not a checksum: the question is "did this change since we + read it", which mtime answers for a pipeline that writes a file once. A + campaign that rewrote a source in place with identical size and mtime would + defeat it, and nothing does. + """ + st = Path(path).stat() + return st.st_size, st.st_mtime_ns + + +def column_types(dtype) -> dict: + """``{column: (kind, itemsize, shape)}``: a dataset's schema, as compared + across units. Byte order is left out: FITS sources are big-endian, hdf5 + may hand them back either way, and neither changes a value.""" + return {n: (dtype[n].base.kind, dtype[n].base.itemsize, dtype[n].shape) + for n in dtype.names} + + +def type_conflict(a_unit, a_dtype, b_unit, b_dtype) -> str: + """Name the first column whose type differs between two units' dtypes.""" + a, b = column_types(a_dtype), column_types(b_dtype) + for col in a: + if a[col] != b.get(col): + got = b_dtype[col].base.name if col in b else "absent" + return (f"column {col} is {a_dtype[col].base.name} in {a_unit} " + f"but {got} in {b_unit}") + return f"{b_unit} carries columns {a_unit} does not" + + +class _Retyped(Exception): + """A read unit's types differ from the datasets `apply` would keep.""" + + +class Plan: + """What reconciling requires: three unit lists. + + ``add`` and ``refresh`` are both "read the source and write the dataset"; + they are separate only so the log can say which happened, because a refresh + means a finished unit's source moved under us and that is worth seeing. + """ + + def __init__(self, add, refresh, remove): + self.add, self.refresh, self.remove = add, refresh, remove + + def empty(self): + return not (self.add or self.refresh or self.remove) + + def describe(self): + return (f"{len(self.add)} added, {len(self.refresh)} refreshed, " + f"{len(self.remove)} removed") + + +def plan(output: Path, group_path: str, units: list, digest: str) -> Plan: + """Compare the file on disk with the campaign, WITHOUT writing anything.""" + if not output.exists(): + return Plan([u for u, _ in units], [], []) + + want = {unit for unit, _ in units} + add, refresh = [], [] + with h5py.File(output, "r") as f: + stale_schema = f.attrs.get("param_digest") != digest + have = dict(f[group_path].items()) if group_path in f else {} + present = set(have) + for unit, source in units: + if unit not in present: + add.append(unit) + elif stale_schema: + refresh.append(unit) + else: + attrs = have[unit].attrs + if (int(attrs.get("src_bytes", -1)), + int(attrs.get("src_mtime_ns", -1))) != stamp(source): + refresh.append(unit) + return Plan(add, refresh, sorted(present - want)) + + +# Twice the file, plus a tenth of it again: the copy and the original coexist, +# and hdf5 is not a format to run to the last byte of a filesystem on. +FREE_SPACE_MARGIN = 2.1 + + +def check_free_space(output: Path) -> None: + """Refuse to start a rewrite the filesystem cannot hold. + + A merge that fills /project does not just fail: it fails everything else + writing there at the same time, and it can leave a truncated tmp beside a + catalogue people trust. Cheaper to say so first. + """ + if not output.exists(): + return + size = output.stat().st_size + free = shutil.disk_usage(output.parent).free + if free < size * FREE_SPACE_MARGIN: + sys.exit( + f"hdf5_reconcile: {output.parent} has {free / 1e9:.1f} GB free and " + f"this merge needs about {size * FREE_SPACE_MARGIN / 1e9:.1f} GB — " + f"it rewrites {output.name} ({size / 1e9:.1f} GB) through a tmp " + f"copy beside it. Free space or move products_dir; the existing " + f"catalogue is untouched.") + + +def check_sole_group(output: Path, group_path: str) -> None: + """One file, one campaign — refuse to half-update a file holding two. + + The output path carries `run:`, so renaming a campaign produces a new + file, not a second group in this one. What this guards is a file already + at this path that holds ANOTHER campaign's group (a hand merge, or a copy). + Reconciling would then add a second group beside the first, leave the + first frozen and stale, and set a count attribute describing only one of + them. Nothing downstream reads such a file correctly, and no rule + here means to produce one. Say what is there and stop. + """ + if not output.exists() or "/" not in group_path: + return + parent, leaf = group_path.rsplit("/", 1) + with h5py.File(output, "r") as f: + if parent not in f: + return + others = sorted(k for k in f[parent] if k != leaf) + if others: + sys.exit( + f"hdf5_reconcile: {output} already holds {parent}/" + f"{', '.join(others)} beside {group_path}. One file is one " + f"campaign: reconciling would freeze the other group and count " + f"only this one. Point `run:` back, or write to a new path.") + + +def apply(output: Path, group_path: str, todo: Plan, units: list, read, + digest: str, count_attr: str, provenance: dict | None = None) -> None: + """Carry the plan out; re-read every unit if the column types changed. + + See ``_apply``. When a unit read under the plan has other column types + than the datasets the plan would keep, the plan widens to refresh every + kept unit, so the file keeps one type per column (module docstring). + """ + try: + _apply(output, group_path, todo, units, read, digest, count_attr, + provenance) + except _Retyped as exc: + every = Plan(todo.add, [u for u, _ in units if u not in todo.add], + todo.remove) + print(f"[hdf5_reconcile] {exc}; re-reading all {len(units)} unit(s)") + _apply(output, group_path, every, units, read, digest, count_attr, + provenance) + + +def _apply(output: Path, group_path: str, todo: Plan, units: list, read, + digest: str, count_attr: str, provenance: dict | None) -> None: + """Carry the plan out on a tmp file, then move it into place. + + ``read(unit, source)`` returns the structured array for one unit; it is + called only for the units the plan names, which is what makes an append + cheap. + + ``provenance`` (``code_provenance()``'s return) is stamped onto the file's + root as ``code_head``/``code_branch``/``code_dirty``/``code_snapshot_at``, + plus ``code_dirty_files`` (newline-joined) when the snapshot was dirty. It + is written here, alongside ``count_attr`` and ``param_digest``, rather than + on every no-op invocation: reconciling is planned against a read-only open, + and an empty plan must leave the file's mtime alone (see the module + docstring), so a run that changes no data never touches the file even if + the code that would have produced it has moved on. + + TWO WAYS TO BUILD THE TMP, and which one is used is about SPACE, not speed. + HDF5 never reclaims the space a deleted dataset occupied, so a file that is + copied and then edited in place grows for the life of the campaign — every + refresh of a unit leaks that unit. So: + + * a plan that only ADDS copies the existing file and appends to it. There + is nothing to reclaim, and copying beats rewriting. It is still a pass + over the whole file — an append is cheap in READS, not in writes. + * a plan that removes or refreshes anything builds the tmp FRESH, moving + the datasets it keeps across with h5py's own group copy — a + dataset-level copy inside the library that never reads a row into numpy + — and writing only the units that actually changed. The result is + compact. + + Either way the tmp is moved into place at the end, so a crash mid-merge + leaves the old catalogue intact rather than a half-written one. A SIGKILL + between writing the tmp and renaming it leaves the tmp behind — one file, + beside the catalogue, deleted by the next run before its space check; the + rename itself is atomic, which is the property that matters. + """ + sources = dict(units) + rewrite = bool(todo.remove or todo.refresh) + tmp = output.with_name(output.name + ".tmp") + # A tmp left by a killed run is this run's to delete (one writer per + # output), and deleting it first keeps it from counting against the space + # this run needs. + tmp.unlink(missing_ok=True) + check_free_space(output) + check_sole_group(output, group_path) + written = set(todo.add) | set(todo.refresh) + keep = [u for u, _ in units if u not in written] + try: + if output.exists() and not rewrite: + shutil.copy2(output, tmp) + with h5py.File(tmp, "a") as f: + group = (f[group_path] if group_path in f + else f.create_group(group_path)) + if rewrite and output.exists(): + with h5py.File(output, "r") as src: + for unit in keep: + # File.copy, not Dataset.copy — the latter does not + # exist, and the difference only shows when a plan both + # rewrites and keeps something. + src.copy(f"{group_path}/{unit}", group, name=unit) + # The kept datasets' types are the reference a read unit must + # match; with nothing kept, the first unit read is. Kept datasets + # that already disagree (a file an older merge left) re-read too. + kept = [(u, group[u].dtype) for u in keep if u in group] + ref = kept[0] if kept else None + for unit, dtype in kept[1:]: + if column_types(dtype) != column_types(ref[1]): + raise _Retyped(type_conflict(*ref, unit, dtype)) + for unit in todo.add + todo.refresh: + source = sources[unit] + data = read(unit, source) + if ref is None: + ref = (unit, data.dtype) + elif column_types(data.dtype) != column_types(ref[1]): + conflict = type_conflict(*ref, unit, data.dtype) + if keep: + raise _Retyped(conflict) + sys.exit(f"hdf5_reconcile: {conflict}. One catalogue " + f"holds one type per column; remake the units " + f"whose sources are stale. {output} is " + f"untouched.") + dset = group.create_dataset(unit, data=data, dtype=data.dtype) + # The dataset's own record of what it was read from; this is + # what lets a later invocation leave it alone. + dset.attrs["src_bytes"], dset.attrs["src_mtime_ns"] = \ + stamp(source) + f.attrs[count_attr] = len(group) + f.attrs["param_digest"] = digest + if provenance: + # One record at a time: an add-only merge copied the last + # one's attributes, and a snapshot-less record must not keep + # its branch, dirty flag, dirty files or time. + for attr in [a for a in f.attrs if a.startswith("code_")]: + del f.attrs[attr] + f.attrs["code_head"] = provenance.get("head", "unknown") + for key, attr in (("branch", "code_branch"), + ("dirty", "code_dirty"), + ("taken_at", "code_snapshot_at")): + if key in provenance: + f.attrs[attr] = provenance[key] + if provenance.get("dirty_files"): + f.attrs["code_dirty_files"] = \ + "\n".join(provenance["dirty_files"]) + tmp.replace(output) # atomic: same filesystem + finally: + tmp.unlink(missing_ok=True) diff --git a/workflow/scripts/merge_final_cat.py b/workflow/scripts/merge_final_cat.py new file mode 100644 index 000000000..1ad6fbdff --- /dev/null +++ b/workflow/scripts/merge_final_cat.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Collect the campaign's per-tile final catalogues into ONE hdf5 file. + +Run as the shell of the campaign-level ``final_cat_merge`` rule, never by hand. + +WHAT IT PRODUCES, AND FOR WHOM. ``/final_cat_.hdf5``: +one dataset per tile, carrying the columns named by the input type's +``final_cat.param`` (``workflow/config/cfis/`` for data, +``workflow/config/cfis_image_sims/`` for image sims), plus an ``n_tiles`` attribute on the +file root. sp_validation opens that file as its ``galaxy_cat_path`` +(``sp_validation/catalog.py``), so its SCHEMA is an interface and not a choice — +see ``SPVAL_GROUP`` below for the one legacy literal in it. + +(sp_validation's own ``merge_catalogues`` is a different layer entirely: it +works over already-calibrated ``shape_catalog_comprehensive_*.fits``. It does +not do this merge, and this does not do that one.) + +WHAT IT REUSES, AND WHAT IT DOES NOT. The column extraction is +``create_final_cat.py``'s — ``read_param_file`` for the parameter list, +``read_data`` and ``copy_data`` for pulling those columns out of one catalogue +with their FITS dtypes — so the column grammar keeps exactly one definition. +Those three are REPRODUCIBLE FUNCTIONS, and this PR is what made them so: the +parameter list comes back ordered rather than through a set, ``copy_data`` +allocates the requested columns alone rather than leaving every other column of +the source as uninitialised memory, and a missing column raises with its own +name instead of falling out of a bare ``except:`` as an UnboundLocalError. The +fixes are upstream, in that script, because a hand-run of it deserves them as +much as this rule does. +Its ``process()`` is NOT used and neither is any of its discovery: that function +walks a directory tree the workflow does not have and never will, and it groups +by a unit ShapePipe v2 no longer has. This script walks the workflow's own +products tree instead (``tiles/<2-char prefix>//final_cat-.fits``) and +writes the hdf5 itself. + +WHERE ``create_final_cat.py`` IS FOUND. Beside this workflow, at +``/scripts/python/create_final_cat.py`` — resolved relative to THIS file, +so it follows the launch code snapshot (``bin/sp``) exactly as +``workflow/scripts/*`` does, and a campaign never reads a mid-run edit. It is +loaded by path rather than imported: it is a script, not an installed module, +and the container's ``shapepipe`` install does not carry it. + +IT RECONCILES, IT NEITHER REBUILDS NOR BLINDLY APPENDS, and the machinery for +that is ``hdf5_reconcile.py``, shared with the star side so the campaign's two +products cannot disagree about what an output owes its inputs. That module +carries the argument in full: an append reads the appended tiles, a source that +changed is re-read, a tile that left the campaign is deleted, a column-set +change refreshes everything, and a no-op leaves the file untouched. +``create_final_cat.py``'s own ``process()`` implements only the append-only half +— it skips a tile already in the file, whatever the file on disk now says — +which is right for a hand-driven update and wrong for a DAG output. (Its ``-s`` +single-ID mode implements ``check`` and ``remove``; ``add`` is accepted by the +argument validator and then falls through to the ordinary walk, so it is not a +way to add one tile by hand.) + +WHICH TILES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. The set +is the CAMPAIGN's: every tile both declared in ``tile_list`` and present in the +index, which is exactly the Snakefile's TILES_READY, rebuilt here from the same +two files the Snakefile started from (``--tile-list`` and ``--index-db``, read +through ``build_index.campaign_tiles`` so there is one definition and not two +that can drift). It is derived rather than passed because at DR6 scale the set +is ~20k paths and a shell command reaches ``execve`` as a SINGLE argv entry +capped at 128 KiB by ``MAX_ARG_STRLEN``; the rule's ``input`` is the DAG edge +and its ``params`` carries a fingerprint of that same list, which is the rerun +trigger. + +THE TWO SETS ARE THE SAME SET, which is the point of deriving it this way rather +than globbing ``/tiles``: a products root shared with an earlier, +larger tile list would hand the job tiles the fingerprint never saw and no rerun +trigger would notice. A tile in the derived set whose catalogue is missing is a +hard error here, not a skip — under the DAG it cannot happen, since every one of +them is a declared input of this job. + +@sc [label:selection] never-fit-rows-pass-through +Every row of every tile catalogue reaches the merged file, unchanged, +including objects ngmix never fit. Those carry `NGMIX_N_EPOCH == 0` with +sentinel values (`NGMIX_MCAL_FLAGS == 0`, ellipticities `-10`, `T == 0`), so +`NGMIX_MCAL_FLAGS == 0` is not a validity cut: consumers select fitted objects +with `NGMIX_N_EPOCH > 0`. The merge neither fills these rows nor drops them; +that selection belongs to the consumer. Enforced by +tests/unit/test_final_cat_merge_invariants.py. +""" + +import argparse +import importlib.util +import sys +from pathlib import Path + +# Same directory; the rule invokes this file by path, so it is sys.path[0]. +import build_index +import hdf5_reconcile + +# /scripts/python/create_final_cat.py, from /workflow/scripts/this. +CFC_PATH = (Path(__file__).resolve().parents[2] + / "scripts" / "python" / "create_final_cat.py") + + +def spval_group(campaign: str) -> str: + """The hdf5 group the campaign's per-tile datasets live under. + + ``patches/`` is a LEGACY KEY IN sp_validation's FILE SCHEMA, kept verbatim + only so its reader works unchanged (CosmoStat/sp_validation#340 tracks + removing it); it names nothing in this workflow, which has campaigns and + tiles and no other unit. This is the one place the literal appears — + everything else here says campaign. + """ + return f"patches/{campaign}" + + +def load_create_final_cat(): + """The hdf5 layout's definition, loaded by path (see the module docstring).""" + if not CFC_PATH.exists(): + sys.exit(f"merge_final_cat: {CFC_PATH} is not there — the launch code " + f"snapshot must carry scripts/python/ (see bin/sp).") + spec = importlib.util.spec_from_file_location("create_final_cat", CFC_PATH) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def catalogues(products_dir: Path, tile_list: Path, index_db: Path) -> list: + """``(tile ID, path)`` for the campaign's tiles, in ID order. + + Not a glob over the products root: see the module docstring on why the set + is the campaign's and not the filesystem's. + """ + out, missing = [], [] + for tile in sorted(build_index.campaign_tiles(tile_list, index_db)): + path = (products_dir / "tiles" / tile[:2] / tile + / f"final_cat-{tile}.fits") + if path.exists(): + out.append((tile, path)) + else: + missing.append(tile) + if missing: + sys.exit(f"merge_final_cat: {len(missing)} campaign tile(s) have no " + f"final catalogue: {' '.join(missing[:5])}" + f"{' ...' if len(missing) > 5 else ''}") + return out + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--products-dir", required=True, type=Path, + help="the persistent root; per-tile catalogues are found " + "beneath it") + p.add_argument("--tile-list", required=True, type=Path, + help="the campaign's tile list (config tile_list)") + p.add_argument("--index-db", required=True, type=Path, + help="the campaign's run index (config outputs.index_db)") + p.add_argument("--output", required=True, type=Path) + p.add_argument("--campaign", required=True, + help="names the campaign's group in the output file") + p.add_argument("--param-file", required=True, type=Path, + help="the input type's final_cat.param — the column list") + p.add_argument("--hdu", type=int, default=1) + p.add_argument("--snapshot-json", type=Path, default=None, + help="sp run's code snapshot (bin/sp's " + "$STATE_DIR/code/snapshot.json); absent outside sp run") + args = p.parse_args() + + cfc = load_create_final_cat() + param_list = cfc.read_param_file(str(args.param_file), verbose=False) + if not param_list: + sys.exit(f"merge_final_cat: no columns read from {args.param_file}") + # read_data/copy_data read their knobs out of this dict, exactly as + # create_final_cat.py's own main() builds it. + params = {"hdu_num": args.hdu, "param_list": param_list, "verbose": False} + + tiles = catalogues(args.products_dir, args.tile_list, args.index_db) + if not tiles: + # An empty hdf5 would satisfy every downstream existence check and + # produce an empty shear catalogue. + sys.exit(f"merge_final_cat: no tile in {args.tile_list} is indexed in " + f"{args.index_db}, so there is nothing to merge") + + args.output.parent.mkdir(parents=True, exist_ok=True) + group_path = spval_group(args.campaign) + digest = hdf5_reconcile.schema_digest(param_list) + + def read_tile(tile, path): + """One tile's requested columns, via create_final_cat.py's own reader.""" + extracted, dtype = cfc.read_data(str(path), params) + return cfc.copy_data(params["param_list"], extracted, dtype) + + todo = hdf5_reconcile.plan(args.output, group_path, tiles, digest) + if todo.empty(): + print(f"[merge_final_cat] unchanged: {args.output} " + f"({len(tiles)} tile(s))") + return + hdf5_reconcile.apply(args.output, group_path, todo, tiles, read_tile, + digest, "n_tiles", + hdf5_reconcile.code_provenance(args.snapshot_json)) + print(f"[merge_final_cat] {todo.describe()} -> {args.output} " + f"({len(tiles)} tile(s), {len(param_list)} column(s), " + f"group {group_path})") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/merge_star_cat.py b/workflow/scripts/merge_star_cat.py new file mode 100644 index 000000000..371119546 --- /dev/null +++ b/workflow/scripts/merge_star_cat.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Collect the campaign's per-CCD PSF validation catalogues into ONE hdf5 file. + +Run as the shell of the campaign-level ``star_cat_merge`` rule, never by hand. + +WHAT IT PRODUCES, AND FOR WHOM. ``/full_starcat_.hdf5``: +one dataset per exposure at ``exposures/``, holding that exposure's every +CCD's ``validation_psf--.fits`` rows stacked, with a ``CCD_NB`` column +recording which CCD each row came from. It is the input to the rho/tau +statistics. Historically this was ``combine_runs.bash psf`` plus a +``merge_starcat_runner`` pass producing one flat FITS table, +``full_starcat-0000000.fits``, and sp_validation still opens that name today; +its readers move to this hdf5 under CosmoStat/sp_validation#340, the same +migration that retires the ``patches/`` key on the galaxy side. + +WHY HDF5, AND WHY ONE DATASET PER EXPOSURE. The campaign's two products should +behave the same way, and one flat table cannot: appending a tile meant +restacking every exposure the campaign had ever seen — ~40 GB of members at DR6 +scale to add ~2 MB. Per-exposure datasets make the file RECONCILABLE +(hdf5_reconcile.py carries that argument, and merge_final_cat.py is the same +machinery on the tile side), so an append reads the appended exposures and +nothing else while the file still cannot drift from its inputs. Memory follows: +one exposure at a time, not one campaign. + +NATIVE DTYPES. Columns are written as the validation_psf files store them — +float32 stays float32. The FITS writer this replaces widened every float column +to ``1D``, doubling both the file and the peak memory of the job that wrote it, +for no information. + +CCD_NB IS AN INTEGER. It is parsed out of the member name +(``validation_psf--.fits``), where it is always digits, so a string +buys nothing — and an int column costs 4 bytes a row against the 8 a +two-character fixed-width string does. + +IT READS THE TARS, IT DOES NOT UNPACK THEM. ``exp_persist`` packs each +exposure's keepers into one uncompressed tar on the persistent root +(``/exp///psf/.tar``) precisely because inodes, +not bytes, bind on /project. Unpacking ~20k tars x ~40 members to merge them +would materialise ~800k files on the filesystem that design exists to protect. +Members are read through the archive's own file object — seekable, the tar +being uncompressed by design — so the counting pass costs a header rather than +a member. + +THE OPTIONAL COLUMNS ARE A PER-FILE QUESTION. A pix2wcs-converted catalogue has +no MAG/SNR/ACCEPTED where an ordinary one does, and a campaign can hold both. +Deciding once for the merge is wrong in both directions: it either fails on the +first converted file or silently zeroes the real values of every ordinary one. +Each file is asked for its own schema, and only the files that lack a column are +zero-filled. + +WHICH EXPOSURES — AND WHY THE JOB DERIVES THE SET RATHER THAN BEING TOLD IT. +The set is the CAMPAIGN's: every exposure read by a tile that is both declared +in ``tile_list`` and present in the index, which is the Snakefile's TILES_READY +walked one edge further. This script rebuilds it from the same two files the +Snakefile started from (``--tile-list`` and ``--index-db``, both small, both on +the persistent root, both read through ``build_index.campaign_exposures`` so +there is one query and not two that can drift), and then takes the exposures +whose ``exp_persist`` manifest is on the persistent root. + +It is derived rather than passed because at DR6 scale the set is ~20k paths and +a shell command reaches ``execve`` as a SINGLE argv entry capped at 128 KiB by +``MAX_ARG_STRLEN``. So the rule's ``input`` is the DAG EDGE — what must exist +before this runs — and its ``params`` carries a FINGERPRINT of the same set, +which is what makes the merge rerun when the set changes. A glob over +``/exp`` would NOT be the same set: it would sweep in exposures of +an earlier, larger tile list sharing the products root, stacking rows the +fingerprint never saw and no rerun trigger would notice. + +THE MANIFEST, NOT THE TAR, IS WHAT IT READS FIRST: the manifest records what was +actually packed, member by member, with sizes, sha256 digests and the product +each came from, so this script never guesses at tar contents. +""" + +import argparse +import json +import sys +import tarfile +from fnmatch import fnmatch +from pathlib import Path + +import numpy as np +from astropy.io import fits + +# Same directory; the rule invokes this file by path, so it is sys.path[0]. +import build_index +import hdf5_reconcile +import persist_exp + +# The members this merge consumes, named as the keep list names them and +# resolved through the same catalogue persist_exp packs by — so the glob has one +# definition and adding a product cannot leave the two disagreeing. They are +# always there to find: persist_exp packs this product for every exposure +# whatever `persist_exp:` says, and fails the pack rather than writing a +# manifest without it. +MEMBER_PRODUCT = persist_exp.ALWAYS +MEMBER_PATTERN = persist_exp.resolve(MEMBER_PRODUCT) + +# The group holding the per-exposure datasets. Unlike the galaxy side's +# `patches/`, this name is ours and says what it holds. +GROUP = "exposures" + +# The validation_psf table's HDU: what MergeStarCatPSFEX defaulted to and what +# psfex_interp writes — a SExtractor-style file, empty primary, header-carrying +# image extension, then the table. +HDU = 2 + +# The columns, in the order the FITS full_starcat carried them, which is the +# order every consumer has seen. The optional three are zero-filled per file. +COLUMNS = ("X", "Y", "RA", "DEC", + "HSM_G1_PSF", "HSM_G2_PSF", "HSM_T_PSF", + "HSM_M4_1_PSF", "HSM_M4_2_PSF", "HSM_RHO4_PSF", + "HSM_G1_STAR", "HSM_G2_STAR", "HSM_T_STAR", + "HSM_M4_1_STAR", "HSM_M4_2_STAR", "HSM_RHO4_STAR", + "HSM_FLAG_PSF", "HSM_FLAG_STAR") +# CANONICAL DTYPES, not whatever the first file that carries the column happens +# to use. These three are absent from pix2wcs-converted catalogues, so an +# exposure whose files all lack them would otherwise be allocated a fallback +# dtype while its neighbours got the real one — and datasets under exposures/* +# would then differ in dtype, which np.concatenate refuses and no digest can +# repair, since nothing about the schema CHANGED. Pinning the dtype here is what +# makes every exposure's dataset the same shape whatever its files carry. +OPTIONAL = {"MAG": np.float32, "SNR": np.float32, "ACCEPTED": np.int32} +CCD_COLUMN = "CCD_NB" +ALL_COLUMNS = COLUMNS + tuple(OPTIONAL) + (CCD_COLUMN,) + + +def ccd_number(member_name: str) -> int: + """The CCD this member's rows belong to: ``validation_psf--.fits``. + + Always digits, which is why the column is an int; a member name that does + not carry one is a tar we do not understand, and saying so beats writing a + sentinel into the catalogue. + """ + ccd = member_name.rsplit(".", 1)[0].rsplit("-", 1)[-1] + if not ccd.isdigit(): + sys.exit(f"merge_star_cat: cannot read a CCD number out of member " + f"name {member_name!r}") + return int(ccd) + + +def is_member(entry: dict) -> bool: + """Is this manifest entry one of the members this merge reads? + + BY PRODUCT NAME, OR FAILING THAT BY FILE NAME. persist_exp records the + product every member came from and always packs psf_validation, so the name + is the answer for anything it writes today. The glob is the fallback, and it + earns its place twice over: a tar packed before the product field existed + has no label at all, and a keep list written as a raw glob + (`validation_psf-*.fits` rather than `psf_validation`) labels its members + with the glob. Neither should make the campaign's star catalogue silently + empty. + """ + return (entry.get("product") == MEMBER_PRODUCT + or fnmatch(entry["name"], MEMBER_PATTERN)) + + +def manifests(products_dir: Path, tile_list: Path, index_db: Path) -> list: + """``(exposure, manifest path)`` for the campaign's packed exposures. + + Not a glob over the products root: see the module docstring on why the set + is the campaign's and not the filesystem's. + """ + out = [] + for exp in build_index.campaign_exposures(tile_list, index_db): + path = (products_dir / "exp" / exp[:2] / exp / "manifests" + / "exp_persist.json") + if path.exists(): + out.append((exp, path)) + return out + + +def tars(manifest_paths: list) -> tuple: + """``[(exposure, tar path)]`` for the merge, and the exposures with nothing. + + Every tar is checked for existence HERE, so a products root missing a file + fails before a single row is read rather than an hour in. The tar is also + the unit's SOURCE for reconciling: its size and mtime are what a later + invocation compares against to decide whether this exposure changed. + """ + chosen, empty = [], [] + for exp, man_path in manifest_paths: + man = json.loads(man_path.read_text()) + # MEMBERSHIP IS THE MEMBER NAME, and only the member name. is_member() + # will also accept a manifest's own product LABEL, which is the right + # test for "did this exposure keep the product" — but a label is not + # what read_exposure() selects on, and a mislabeled entry whose name + # does not match would put this exposure in the merge and then abort + # the whole campaign when the tar turned out to hold nothing selectable. + # So the two agree by construction: both ask the name. + if not any(fnmatch(f["name"], MEMBER_PATTERN) for f in man["files"]): + if any(is_member(f) for f in man["files"]): + # Labelled as the product, named as something else. Worth one + # line — it means a manifest we did not write, or a keep list + # whose glob does not match the member it matched. + print(f"[merge_star_cat] {exp}: manifest labels a " + f"{MEMBER_PRODUCT} member whose name does not match " + f"{MEMBER_PATTERN}; not merging it") + empty.append(exp) + continue + tar_path = Path(man["tar"]) + if not tar_path.exists(): + sys.exit(f"merge_star_cat: {man_path} names a tar that is not " + f"there: {tar_path}") + chosen.append((exp, tar_path)) + return chosen, empty + + +def read_exposure(exp: str, tar_path: Path) -> np.ndarray: + """One exposure's every CCD, stacked, as a structured array. + + TWO PASSES over the tar's members, and neither holds the exposure twice: + the first reads only each member's FITS HEADER — NAXIS2, the row count — + and the second allocates the columns once at their exact final length and + fills them slice by slice. Members are visited in sorted name order, so the + row order is a function of the tar's contents alone. + + NOTE ON WHEN THIS IS CALLED AGAIN. The unit's source is the TAR, so adding a + retention product re-packs it, moves its mtime, and refreshes this exposure + even though its validation members are byte-for-byte what they were. Reading + one exposure is seconds and the alternative — stamping the members rather + than the archive — buys a rarely-taken shortcut for a per-member bookkeeping + cost on every exposure. Not worth it. + """ + try: + tf = tarfile.open(tar_path) + except tarfile.TarError as exc: + sys.exit(f"merge_star_cat: cannot read {tar_path}: {exc}. That tar is " + f"this exposure's only copy of its PSF products — do not " + f"delete it; re-pack the exposure if its scratch store is " + f"still there, and treat the exposure as lost if it is not.") + with tf: + names = sorted(n for n in tf.getnames() + if fnmatch(n, MEMBER_PATTERN)) + if not names: + sys.exit(f"merge_star_cat: {tar_path} holds no {MEMBER_PATTERN}") + + # --- pass 1: row counts and dtypes, from headers alone -------------- + counts, dtypes, n_total = [], None, 0 + for name in names: + with fits.open(tf.extractfile(name), memmap=False, + ignore_missing_simple=True) as hdul: + hdu = hdul[HDU] + counts.append(hdu.header["NAXIS2"]) + # ColDefs.dtype describes the table without reading it. NOTE: + # it is the RAW storage dtype and ignores TSCAL/TZERO, so a + # scaled column would be allocated narrower than the values + # .data returns. Latent, not live: no validation_psf column is + # scaled. Read the dtype off .data if one ever is. + if dtypes is None: + dtypes = hdu.columns.dtype + n_total += counts[-1] + + fields = [(c, dtypes[c]) for c in COLUMNS] + # The optional three take their CANONICAL dtype, not one file's (see + # OPTIONAL): every exposure's dataset must have the same dtype whether + # or not its files carry the column. + fields += list(OPTIONAL.items()) + fields += [(CCD_COLUMN, np.int32)] + data = np.empty(n_total, dtype=np.dtype(fields)) + + # --- pass 2: fill --------------------------------------------------- + at = 0 + for name, n_rows in zip(names, counts): + with fits.open(tf.extractfile(name), memmap=False, + ignore_missing_simple=True) as hdul: + rows = hdul[HDU].data + have = set(rows.dtype.names or ()) + sl = slice(at, at + n_rows) + for col in COLUMNS: + data[col][sl] = rows[col] + for col in OPTIONAL: + # THIS file's schema, not the exposure's. + data[col][sl] = rows[col] if col in have else 0 + data[CCD_COLUMN][sl] = ccd_number(name) + at += n_rows + + if at != n_total: + raise ValueError(f"merge_star_cat: {tar_path}: pass 1 counted " + f"{n_total} rows, pass 2 filled {at}") + return data + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--products-dir", required=True, type=Path, + help="the persistent root; exp_persist manifests and tars " + "are found beneath it") + p.add_argument("--tile-list", required=True, type=Path, + help="the campaign's tile list (config tile_list)") + p.add_argument("--index-db", required=True, type=Path, + help="the campaign's run index (config outputs.index_db)") + p.add_argument("--output", required=True, type=Path) + p.add_argument("--campaign", required=True, + help="named in the log; the group name is fixed") + p.add_argument("--snapshot-json", type=Path, default=None, + help="sp run's code snapshot (bin/sp's " + "$STATE_DIR/code/snapshot.json); absent outside sp run") + args = p.parse_args() + + manifest_paths = manifests(args.products_dir, args.tile_list, args.index_db) + chosen, empty = tars(manifest_paths) + if not chosen: + # Not a no-op: an empty star catalogue would pass every downstream + # existence check and produce meaningless rho statistics. + sys.exit(f"merge_star_cat: no {MEMBER_PRODUCT} member in any of " + f"{len(manifest_paths)} exp_persist manifest(s) for this " + f"campaign. persist_exp packs {MEMBER_PRODUCT} for every " + f"exposure, so this means the manifests are not what we think " + f"they are.") + if empty: + print(f"[merge_star_cat] {len(empty)} exposure(s) persisted no " + f"{MEMBER_PRODUCT}: {', '.join(sorted(empty)[:5])}" + f"{' ...' if len(empty) > 5 else ''}") + + args.output.parent.mkdir(parents=True, exist_ok=True) + digest = hdf5_reconcile.schema_digest(ALL_COLUMNS) + todo = hdf5_reconcile.plan(args.output, GROUP, chosen, digest) + if todo.empty(): + print(f"[merge_star_cat] unchanged: {args.output} " + f"({len(chosen)} exposure(s))") + return + hdf5_reconcile.apply(args.output, GROUP, todo, chosen, read_exposure, + digest, "n_exposures", + hdf5_reconcile.code_provenance(args.snapshot_json)) + print(f"[merge_star_cat] {todo.describe()} -> {args.output} " + f"({len(chosen)} exposure(s), {len(ALL_COLUMNS)} column(s), " + f"campaign {args.campaign})") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/persist_exp.py b/workflow/scripts/persist_exp.py new file mode 100644 index 000000000..3e6267552 --- /dev/null +++ b/workflow/scripts/persist_exp.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +"""Pack ONE exposure's keepable PSF products into a tar off scratch, and record what went. + +Run as the shell of the in-DAG ``exp_persist`` rule, never by hand. + +WHY A COPY AND NOT AN EXEMPTION FROM CLEANUP. The obvious alternative — teach +``clean_exposure`` to spare these files — does not work, because reclamation is +not what threatens them. The exposure store lives on ``run_dir``, which is +/scratch: a 60-day purge takes everything there whether or not this workflow +ever cleaned it. ``products_dir`` is /project, backed up and not purged. So the +only way a per-exposure product outlives its campaign is to LEAVE THE +FILESYSTEM, and that is a copy. Reclamation ordering then falls out for free: +``clean_exposure`` takes this rule's manifest as an input, so the store is never +deleted before its keepers have been written elsewhere. + +WHY A SEPARATE RULE AND NOT A ``cp`` APPENDED TO ``exp_psf``. The list of what +to keep is a decision that will be revisited — rho statistics want one file +today, a residual study may want three tomorrow — and ``exp_psf`` is four hours +per exposure. The list rides on this rule's ``params``, so editing it makes +snakemake rerun THIS rule (seconds of cp) and leaves the PSF chain alone. Folded +into ``exp_psf``, the same edit would re-derive every PSF model in the campaign. + +WHAT IT SEARCHES. ``/output/run_sp_exp_SxSePsf/*/output/`` — the four +module output dirs of the PSF config (sextractor, setools, psfex, psfex_interp) +— RECURSIVELY. The recursion is not laziness: setools does not write flat, it +writes into ``mask/``, ``rand_split/``, ``new_cat/``, ``plot/`` and ``stat/`` +beneath its own output dir, so a caller who wrote ``star_split_ratio_80-*.fits`` +meaning "the training star sample" would match nothing under a non-recursive +glob. Patterns are therefore plain FILE names and the layout is ours to know, +not the config author's. + +RETENTION IS ADDITIVE, AND THAT IS A SAFETY PROPERTY. The keep list rides on +the rule's ``params``, so SHRINKING it reruns this script — and a naive rerun +would rewrite the tar without the products that were dropped, deleting them +from the backed-up filesystem because someone edited a config, with the scratch +store they came from usually long gone. An existing tar is therefore a FLOOR: +its members are carried into the new one whatever the current list says, and a +config change can only ever add. Removing a product is a deliberate act on +products_dir, not a config edit. + +THE KEEP LIST IS WHAT THE CAMPAIGN KEEPS ON TOP OF THE MERGE'S INPUTS. +``psf_validation`` is packed unconditionally (see ALWAYS below); ``persist_exp:`` +is purely optional retention, and an EMPTY one is a coherent instruction — the +tar then holds the star catalogue's inputs and nothing else. + +ZERO MATCHES FOR ONE PATTERN IS A WARNING, NOT A FAILURE. setools rejects sparse +CCDs (~0.2% attrition, tolerated by exp_psf's own count floor), so per-CCD +counts are not fixed, and a pattern naming an optional diagnostic may legitimately +find nothing. ZERO FILES IN TOTAL IS A FAILURE: it means the store was not what +we think it is, and writing a green manifest over that would let +``clean_exposure`` delete an exposure whose products were never saved. + +The manifest lists every member (name, pattern, source path, bytes, sha256), +so a reader knows what the tar holds without opening it. + +ONE UNCOMPRESSED TAR PER EXPOSURE, ``/.tar``, NOT LOOSE COPIES. +Inodes, not bytes, are what bind on /project: the group quota is ~1 M files, +and loose per-CCD copies are ~200 per exposure with all candidates on — ~25k for +a 64-tile campaign, ~2 M at DR6 scale, against ~7 GB of bytes. A tar collapses +that to one inode per exposure and costs nothing to read: FITS members go +``tarfile.open(t).extractfile(m).read()`` -> ``fits.open(io.BytesIO(...))``, +which is why a tar rather than a multi-HDU FITS bundle (the keep list mixes +FITS, ``.psf`` and ``.txt``; a FITS container could not hold the last two). +Uncompressed because FITS barely compresses and a plain tar is seekable. + +Members are FLAT — file name only, no module subtree — because the module a +file came from is already in its name and the consumer globs member names. A +name collision between two modules is therefore a hard error rather than a +silent overwrite; nothing in the current config can produce one, and if a +future one can we want to hear about it. + +The tar is written DETERMINISTICALLY (ownership zeroed, members in sorted +order, source mtimes kept), tmp-then-``cmp``-then-``mv``: a rerun over an +unchanged store produces a byte-identical tar and leaves the existing one's +mtime alone. + +The manifest is the rule's ONLY declared output, and it lives on the persistent +root beside the tar (``/exp///manifests/``, beside the tar's ``psf/``), NOT in +the exposure's scratch ``manifests/`` dir which ``clean_exposure`` deletes +wholesale. It is deliberately NOT a ``directory()`` output: what was copied, and +how big each file was, is provenance we want written down, and a directory +output attests only that some directory exists. + +It carries no timestamp and is written tmp-then-``cmp``-then-``mv`` (the pattern +``clean_exposure`` uses), so a rerun that packs the same files leaves the mtime +alone — mtime is a rerun trigger, and an unconditional rewrite would make every +downstream ``clean_exposure`` look out of date once per invocation. + +@sc [label:safety] persist-exp-additive-and-always-validation +`persist_exp:` only ever adds: an existing tar is a floor whose members are +carried into the rewrite whatever the current keep list says, and +`psf_validation` is packed on every run whether or not the list names it. A +keep-list edit reruns this script, so a subtractive rewrite would delete +products from `products_dir` after their scratch store is gone, and a pack +without `psf_validation` would leave `star_cat_merge` short an exposure. +Enforced by tests/unit/test_persist_exp_props.py. +""" + +import argparse +import filecmp +import hashlib +import json +import sys +import tarfile +from pathlib import Path + +# The PSF chain's run dir: RUN_NAME in config_exp_psfex.ini AND in +# config_exp_mccd.ini, which carry the same name on purpose so nothing +# downstream of exp_psf branches on the PSF model. Hardcoded rather than passed: +# this rule persists the PSF stage's products and nothing else, and a knob here +# would be a knob for "persist some other stage", which is a different rule. +# tests/unit/test_workflow_run_names.py holds this equal to the configs. +RUN_NAME = "run_sp_exp_SxSePsf" + +# --- the product catalogue (CosmoStat/shapepipe#844) ------------------------ +# THE SINGLE SOURCE OF TRUTH for what an exposure can keep. `persist_exp:` in +# config.yaml names PRODUCTS, not globs: `psf_model`, not `*.psf`. The glob is +# an implementation detail of the module that writes the file, and a keep list +# written in globs is a keep list nobody can read — the argument that produced +# #844 and the 2026-09-08 call's request to keep the PSF model, which had to be +# spelled `*.psf` to be said at all. +# +# Each entry is (glob, per-exposure size, what keeping it buys). Sizes are for +# 40 CCDs, measured on smk-m2 (127 exposures, 64 tiles); "?" means not yet +# measured. `persist_exp.py --list-products` renders this table, and +# config.yaml's block is that rendering rather than a second copy of it. +# +# ORDER IS THE ORDER OF THE CHAIN — sextractor, setools, psfex, psfex_interp — +# so the table reads as the pipeline runs. +# THE STAR CATALOGUE'S INPUTS ARE NOT A USER CHOICE. star_cat_merge stacks +# every CCD's psf_validation into the campaign's full_starcat, so exp_persist +# ALWAYS packs it, whatever `persist_exp:` says. Two reasons, and neither is +# about taste. It is the merged catalogue's PROVENANCE: a full_starcat with no +# per-exposure inputs beside it cannot be audited, re-cut or recomputed after a +# purge. And it is what keeps APPENDING TILES CHEAP: a tile added next month +# brings exposures whose validation catalogues must join the existing stack, and +# if the earlier ones are gone the merge either shrinks or rebuilds their chains +# from VOS. ~2 MB per exposure, so ~40 GB and ~40k inodes at DR6 scale, against +# a group quota of ~1 M inodes — the cost of being able to say where the number +# came from. +ALWAYS = "psf_validation" + +PRODUCTS = { + "star_selection": ( + "star_selection-*.fits", 24_500_000, + "setools' PRE-SPLIT selection. The only file that answers which stars " + "the selection cuts rejected and why; the split samples have already " + "lost the rejects."), + "star_train": ( + "star_split_ratio_80-*.fits", 19_900_000, + "the 80% TRAINING sample, the stars PSFEx actually fitted. Rows " + "duplicate star_selection."), + "star_test": ( + "star_split_ratio_20-*.fits", 7_100_000, + "the 20% VALIDATION sample — the positions psf_validation's rows " + "correspond to. Rows duplicate star_selection."), + "star_stats": ( + "star_stat-*.txt", None, + "setools' per-CCD STAT block: star counts, stars/deg^2, FWHM mode and " + "cuts. The selection's summary without its catalogue."), + "psf_model": ( + "*.psf", 2_800_000, + "the PSFEx model itself. Keeping it means the PSF can be " + "re-interpolated at ANY position later without rebuilding the exposure " + "chain — the single most capability-adding entry here."), + "psfex_cat": ( + "psfex_cat-*.cat", None, + "PSFEx's own output catalogue (FITS_LDAC): the per-star FLAGS_PSF and " + "CHI2_PSF, i.e. WHICH stars outlier rejection clipped. Not recoverable " + "from anything else — the .psf header keeps only the LOADED/ACCEPTED " + "counts."), + "psf_validation": ( + "validation_psf-*.fits", 2_000_000, + "the psfex_interp validation catalogue, one per CCD: the input to the " + "rho/tau statistics, and to the star_cat_merge rule that stacks them " + "into the campaign's full_starcat."), +} + +# PSFEx residual/check images and its XML diagnostics are deliberately absent: +# the committed default.psfex sets CHECKIMAGE_TYPE NONE and WRITE_XML N, so +# nothing is emitted to match. They are a config change first, a catalogue +# entry second. + +# A raw glob is still accepted, as an escape hatch for a file the catalogue does +# not name yet. The test is syntactic and deliberately cheap: a product name is +# a bare identifier, so anything carrying a glob metacharacter or a dot is a +# glob. That makes `*.psf`, `star_stat-*.txt` and `default.psfex` globs, and +# `psf_model` a name, with no ambiguity a user could stumble into. +_GLOBBY = set("*?[]. ") + + +def is_glob(entry: str) -> bool: + """True when this keep-list entry is a raw glob rather than a product name.""" + return any(ch in _GLOBBY for ch in entry) + + +def resolve(entry: str) -> str: + """The file-name glob for one keep-list entry, name or raw glob.""" + if is_glob(entry): + return entry + try: + return PRODUCTS[entry][0] + except KeyError: + raise KeyError( + f"unknown persist_exp product {entry!r}; the products are " + f"{', '.join(PRODUCTS)} (or write a raw glob such as '*.psf')" + ) from None + + +def product_of(entry: str) -> str: + """The NAME to record for an entry — the entry itself for a raw glob.""" + return entry + + +def render_products() -> str: + """The catalogue as a table, for --list-products and for config.yaml.""" + width = max(len(n) for n in PRODUCTS) + lines = [f"{'product'.ljust(width)} {'glob'.ljust(26)} size/exposure", + f"{'-' * width} {'-' * 26} -------------"] + for name, (glob, size, why) in PRODUCTS.items(): + size_s = "unmeasured" if size is None else f"{size / 1e6:.1f} MB" + lines.append(f"{name.ljust(width)} {glob.ljust(26)} {size_s}") + for i, chunk in enumerate(_wrap(why, 66)): + lines.append(f"{' ' * width} {chunk}") + return "\n".join(lines) + + +def _wrap(text: str, width: int) -> list: + out, line = [], "" + for word in text.split(): + if line and len(line) + 1 + len(word) > width: + out.append(line) + line = word + else: + line = f"{line} {word}".strip() + if line: + out.append(line) + return out + + +def collect(exp_dir: Path, patterns: list) -> tuple: + """Matched files per ENTRY, in a stable order, plus the entries that matched + nothing. Entries are product names or raw globs; resolve() takes either.""" + root = exp_dir / "output" / RUN_NAME + found, empty = {}, [] + for entry in patterns: + pat = resolve(entry) + # One glob per module output dir, recursive beneath it (see the module + # docstring on setools' subdirectories). sorted() over the union keeps + # the manifest byte-stable across filesystem readdir order. + hits = sorted({p for mod in sorted(root.glob("*/output")) + for p in mod.rglob(pat) if p.is_file()}) + if hits: + found[entry] = hits + else: + empty.append(entry) + return found, empty + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", type=Path, + help="the exposure's scratch store") + p.add_argument("--exp") + p.add_argument("--dest", type=Path, + help="/exp///psf; the tar is " + "/.tar") + p.add_argument("--manifest", type=Path) + p.add_argument("--pattern", action="append", default=[], + help=f"repeatable; a product name (see --list-products) or " + f"a raw file-name glob. {ALWAYS} is packed whether or " + f"not it is named — star_cat_merge needs it") + p.add_argument("--list-products", action="store_true", + help="print the product catalogue and exit") + args = p.parse_args() + + # --list-products is a QUERY, not a run: it answers "what can I keep?" and + # needs no exposure, so the run arguments are optional at the parser and + # required here instead. + if args.list_products: + print(render_products()) + return + missing = [f"--{n.replace('_', '-')}" for n in + ("exp_dir", "exp", "dest", "manifest") + if getattr(args, n) is None] + if missing: + p.error(f"the following arguments are required: {', '.join(missing)}") + + # The merge's input first and always, then whatever the campaign chose to + # keep on top of it (see ALWAYS). Deduped, so naming it explicitly in + # persist_exp: is harmless rather than a repeated pattern. + entries = [ALWAYS] + [e for e in args.pattern if e != ALWAYS] + + for entry in entries: # loud, and before any work + try: + resolve(entry) + except KeyError as exc: + sys.exit(f"persist_exp: {exc.args[0]}") + + found, empty = collect(args.exp_dir, entries) + # THE MERGE'S INPUTS ARE NOT ALLOWED TO BE MISSING, and this is a harder + # rule than "something matched". An exposure whose psfex_interp failed but + # whose PSFEx model landed has a non-empty match set under the default + # retention list, so it used to get a green manifest — and clean_exposure + # takes that manifest as its go-ahead and deletes the store, taking the + # stars with it. There is no recovering them afterwards short of rebuilding + # the chain from VOS, so a missing psf_validation fails the job here, while + # the store is still on disk. Retention products that match nothing stay + # warnings: they are optional by construction. + if ALWAYS not in found: + sys.exit(f"persist_exp: {args.exp}: nothing matched {ALWAYS} " + f"({resolve(ALWAYS)}) under {args.exp_dir}/output/{RUN_NAME}. " + f"That is the star catalogue's input and it is not optional — " + f"refusing to write a manifest that would let clean_exposure " + f"reclaim this store.") + if not found: + sys.exit(f"persist_exp: {args.exp}: no file matched any of " + f"{entries} under {args.exp_dir}/output/{RUN_NAME}") + + args.dest.mkdir(parents=True, exist_ok=True) + tar_path = args.dest / f"{args.exp}.tar" + # A file matched by TWO patterns is one file, not a collision. Keep lists + # overlap on purpose — `validation_psf-*.fits` alongside `*.fits` is a + # perfectly ordinary way to say "the validation catalogues, and everything + # else FITS while we are here" — and treating the second match as a name + # clash failed every exposure in the campaign. What must still be fatal is + # two DIFFERENT paths landing on one flat member name, which would silently + # overwrite; that is a same-name/different-source test, and the first + # pattern to match a file is the one recorded for it. + seen, files = {}, [] + for pat, hits in found.items(): + for src in hits: + if src.name in seen: + if seen[src.name][0] == src: + continue # same file, a second matching pattern + sys.exit(f"persist_exp: {args.exp}: two source files are both " + f"named {src.name} ({seen[src.name][0]} and {src}); tar " + f"members are flat, so this would silently overwrite") + seen[src.name] = (src, pat) + files.append({"name": src.name, "product": pat, + "pattern": resolve(pat), + "src": str(src), "bytes": src.stat().st_size}) + + # --- RETENTION IS ADDITIVE: an existing tar is a FLOOR, never a draft ---- + # Shrinking `persist_exp:` used to rerun this rule (the list rides on + # params, which is the whole point of the rule) and overwrite the tar with + # a smaller one — deleting products from the BACKED-UP filesystem because + # someone edited a config. The scratch store they came from is usually gone + # by then, so nothing could put them back. Whatever is already in the tar + # therefore stays in it: a config change can only ever ADD. + # + # Removing a product is consequently not a config edit. It is a deliberate + # act on products_dir, and it should look like one. + carried, prior_products = [], {} + if tar_path.exists(): + prior = args.manifest + if prior.exists(): + try: + prior_products = {f["name"]: f.get("product", "?") + for f in json.loads(prior.read_text())["files"]} + except (OSError, ValueError, KeyError): + pass # a damaged manifest loses only labels + try: + old_read = tarfile.open(tar_path) + except tarfile.TarError as exc: + sys.exit(f"persist_exp: {args.exp}: cannot read the existing " + f"{tar_path}: {exc}. Refusing to write a new one — the " + f"old tar is left exactly as it is, and it may still hold " + f"products nothing else has. Move it aside deliberately " + f"if you have decided it is lost.") + with old_read as tf: + for ti in tf.getmembers(): + if ti.name in seen or not ti.isfile(): + continue # a live source supersedes it + carried.append(ti.name) + files.append({"name": ti.name, + "product": prior_products.get(ti.name, "?"), + "pattern": None, "src": None, "bytes": ti.size}) + + files.sort(key=lambda f: f["name"]) + + def anonymous(ti: tarfile.TarInfo) -> tarfile.TarInfo: + # Ownership is the one thing that would differ between two writes of + # the same files from different accounts/nodes; drop it. mtime stays: + # it is the product's, and it is stable while the store is. + ti.uid = ti.gid = 0 + ti.uname = ti.gname = "" + return ti + + # tmp-then-cmp-then-mv, and the tmp NEVER outlives a failure: an orphaned + # .tmp on /project is an inode nothing revisits — the leak this whole tar + # design exists to avoid, one per failed attempt at DR6 scale. + tmp = tar_path.with_name(tar_path.name + ".tmp") + try: + # One pass in sorted member order, taking each member from whichever + # side has it: a live source on disk, or the existing tar. Members are + # copied across with their own TarInfo, so a carried member is + # byte-for-byte what it was and a rerun that changes nothing still + # produces an identical archive. + with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: + # Already proven readable above, where the members were listed. + old_tar = (tarfile.open(tar_path) if carried else None) + try: + for f in files: + if f["name"] in seen: + tf.add(seen[f["name"]][0], arcname=f["name"], + filter=anonymous) + else: + ti = anonymous(old_tar.getmember(f["name"])) + tf.addfile(ti, old_tar.extractfile(f["name"])) + finally: + if old_tar is not None: + old_tar.close() + if tar_path.exists() and filecmp.cmp(tmp, tar_path, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(tar_path) # atomic: no half-written archive + finally: + tmp.unlink(missing_ok=True) + + # Each member's sha256, read back from the tar as published. The manifest + # is the DAG edge star_cat_merge waits on: a refit that changes values but + # no sizes must change it, and a rerun over the same bytes must not. + with tarfile.open(tar_path) as tf: + for f in files: + f["sha256"] = hashlib.file_digest( + tf.extractfile(f["name"]), "sha256").hexdigest() + + body = { + "stage": "exp_persist", "level": "exp", "unit": args.exp, + "status": "complete", + "tar": str(tar_path), + "products": entries, + "patterns": [resolve(e) for e in entries], + # The warning the docstring argues for: named patterns that matched + # nothing. Present as a key even when empty, so a reader never has to + # wonder whether an old manifest predates the field. + "patterns_unmatched": empty, + "n_files": len(files), + "bytes": sum(f["bytes"] for f in files), + "files": files, + } + args.manifest.parent.mkdir(parents=True, exist_ok=True) + tmp = args.manifest.with_name(args.manifest.name + ".tmp") + try: + tmp.write_text(json.dumps(body, indent=2, sort_keys=True) + "\n") + if args.manifest.exists() and filecmp.cmp(tmp, args.manifest, shallow=False): + tmp.unlink() # unchanged: leave the mtime alone + else: + tmp.replace(args.manifest) + finally: + tmp.unlink(missing_ok=True) + + warn = (f" ({len(empty)} retention product(s) matched nothing: {empty})" + if empty else "") + if carried: + warn += f" ({len(carried)} member(s) carried from the existing tar)" + print(f"[persist_exp] {args.exp}: {len(files)} file(s), " + f"{body['bytes'] / 1e6:.1f} MB -> {tar_path}{warn}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py index 50cb9a417..831147986 100644 --- a/workflow/scripts/run_config.py +++ b/workflow/scripts/run_config.py @@ -23,7 +23,10 @@ # here must not carry a top-level default as well. MACHINE_KEYS = ("tile_list", "retrieve", "container", "inputs", "outputs", "psf_model", "psf_dict") -REQUIRED = ("tile_list", "inputs.tiles", "inputs.exposures", +# `run` is required in its own right, not only through `$run` in the paths: +# it names the campaign's merged catalogues, so a run config whose paths +# never mention `$run` must still set it. +REQUIRED = ("run", "tile_list", "inputs.tiles", "inputs.exposures", "outputs.run_dir", "outputs.index_db") @@ -72,7 +75,7 @@ def apply_machine_defaults(config): A key already in `config` wins; for `inputs`/`outputs` the merge is per sub-key. In all of these, `$base_dir` expands to machines[machine].base_dir - and `$run` to the top-level `run:`. + and `$name` / `${name}` to any top-level scalar (`$run` to `run:`). """ machine = config.get("machine") or os.environ.get("SP_PROFILE", "nibi") entry = (config.get("machines") or {}).get(machine) or {} @@ -116,11 +119,24 @@ def get(config, dotted): return value +def _dollar_keys(value, prefix): + """Dotted keys under `value` whose string still holds a `$`.""" + if isinstance(value, dict): + return [k for key, sub in value.items() + for k in _dollar_keys(sub, f"{prefix}.{key}")] + return [prefix] if isinstance(value, str) and "$" in value else [] + + def unresolved(config): """REQUIRED keys that are unset, the placeholder, or hold an unexpanded - $variable (e.g. `$run` with no `run:` set).""" - return [k for k in REQUIRED - if get(config, k) in (None, "", PLACEHOLDER) or "$" in str(get(config, k))] + $variable, then every MACHINE_KEYS value (recursively through + inputs/outputs) that still holds one (e.g. `$run` with no `run:` set, or + a misspelt name).""" + missing = [k for k in REQUIRED + if get(config, k) in (None, "", PLACEHOLDER) + or "$" in str(get(config, k))] + dollar = [k for key in MACHINE_KEYS for k in _dollar_keys(config.get(key), key)] + return missing + [k for k in dollar if k not in missing] def load(config_yaml, run_config=None): diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 24c07670b..11cdcffe4 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -61,6 +61,13 @@ "tile_ngmix", "tile_merge_cats", "tile_make_cat"] EXP_STAGES = ["exp_get_images", "exp_split", "exp_psf"] +# exp_persist is DELIBERATELY NOT in that list. This report disk-scans the +# scratch run_dir, and exp_persist's manifest is the one exposure manifest that +# lives on products_dir instead — that placement is what makes it survive +# clean_exposure. Listed here it would read as "not run" for every exposure in +# the campaign. Reporting on the persisted products means scanning the second +# root, which is a report this one does not yet do. + # The manifests clean_tile leaves on disk (workflow/scripts/clean_tile.py names # the mechanism that owns each). Their presence is therefore NOT evidence that a # tile's chain was rebuilt, which absorb_tombstones needs to know