Record the pipeline's scientific decisions in astra.yaml, linked to the code by @sc tags

CLAUDE.md

@@ -118,4 +118,40 @@ keep in their own stores outside it. A `.felt/` directory (a markdown "fiber" no
 store used with the `felt` CLI) is **not tracked here**: it's gitignored, and where
 it exists it's a machine-local symlink into a private, separately git-synced store,
 so a fresh clone won't have one. Record durable decisions in the PR, issue, or docs
-where the change lives.
+where the change lives — and *scientific* decisions in `astra.yaml`, below.
+
+## Scientific decisions live in `astra.yaml`
+
+`astra.yaml` at the repo root is the pipeline's decision record: every
+consequential scientific choice embedded in the code and committed configs,
+with its rationale, alternatives and selected default. `universes/committed.yaml`
+pins the option the committed configuration selects for every decision.
+`@sc [decision:<id>]` tags live at implementing code/config sites; optional
+`Values:` sentences assert committed values within those tagged sites. The
+rationale stays in ASTRA, while local `@sc` contracts hold site-specific
+constraints. `tests/unit/test_decisions.py` checks tag syntax, bidirectional
+decision/site coverage, values, and test `decision` markers. To inspect tags at
+an implementation site, run `python -m tests.helpers.decisions <path>[:<line>]`;
+`--decision <id>` lists its sites. The format is ASTRA.
+
+`uvx astra-tools@0.2.17 guide` is the briefing and
+`uvx astra-tools@0.2.17 spec` the field reference. The file's header states
+its conventions, including `[HARDCODED]` and `[LINT]`.
+
+Membership test: a different defensible choice would change which objects enter
+the shear catalogue, or the numbers attached to them. Detection thresholds,
+masking, star-selection cuts, PSF model degree, ngmix priors and seeding, flag
+semantics, completeness gates: in. Workflow policy (manifests, chunking,
+allocation, failure reporting, provenance) is out; it lives in the PR and in the
+PRD, CosmoStat/shapepipe#848.
+
+**A scientific change is not finished until the record is.** When a change moves
+what the pipeline measures, amend `astra.yaml` in the same PR (add the decision,
+or edit its rationale, options, Values and site tags), pin the selected option
+in `universes/committed.yaml`, and say so in the PR description.
+`tests/unit/test_decisions.py` runs in CI; a scientific change that breaks its
+site/value checks or leaves the record stale is unfinished. Before committing:
+
+```bash
+uvx astra-tools@0.2.17 validate
+```

pyproject.toml

@@ -147,4 +147,5 @@ markers = [
     "slow: heavy compute (minutes); excluded from the fast inner loop.",
     "candide: needs the candide cluster and/or its real data; auto-skipped elsewhere.",
     "unions: uses UNIONS-specific survey layout or data.",
+    "decision(*ids): the astra.yaml decisions this test protects.",
 ]

src/shapepipe/modules/find_exposures_package/find_exposures.py

@@ -64,6 +64,10 @@ def get_exposure_list(self):
         Return list of exposure file used for the tile in process, from tiles
         FITS header.
 
+        @sc [decision:preparation.epoch_provenance_from_tile_history,label:convention] epochs-from-tile-history
+        Apply ``EXP_PREFIX`` only at the start of the parsed name; do not strip
+        matching text from the middle or remove the trailing ``p`` here.
+
         Returns
         -------
         list

src/shapepipe/modules/make_cat_package/__init__.py

@@ -50,6 +50,12 @@
     retained as the extension point for a future estimator family)
 SAVE_PSF_DATA : bool, optional
     Save PSF information if ``True``; default value is ``False``
+N_EPOCH_SLOTS : int, optional
+    Number of slots written for each per-epoch PSF column family
+    (``HSM_*_PSF_n``, ``EXP_ID_n``, ``CCD_n``) when ``SAVE_PSF_DATA`` is
+    ``True``; unfilled slots hold the family's sentinel, and an object with
+    more epochs than slots raises an error. A fixed count gives every tile
+    the same schema. Default is the tile's maximum ``N_EPOCH`` plus one
 
 """
 

src/shapepipe/modules/make_cat_package/make_cat.py

@@ -111,6 +111,7 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True):
     int
         Number of objects saved
 
+    @sc [decision:catalogue_assembly.tile_overlap_handling]
     """
     sexcat_file = file_io.FITSCatalogue(sexcat_path, SEx_catalogue=True)
     sexcat_file.open()
@@ -175,6 +176,7 @@ def save_sm_data(
     -------
     int
         Number of objects saved
+    @sc [decision:catalogue_assembly.star_galaxy_classification]
     """
     final_cat_file.open()
 
@@ -250,6 +252,10 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log):
     The lookup itself is ``shapepipe.utilities.mask_query.query_map``,
     shared with the ``mask_query`` module: one primitive, two consumers.
 
+    @sc [decision:masking.sky_mask_application,label:scope] mask-columns-verbatim
+    Query ``XWIN_WORLD`` and ``YWIN_WORLD`` in catalogue order and pass the
+    ``query_map`` result to ``MASK_<BAND>`` unchanged.
+
     Parameters
     ----------
     final_cat_file : file_io.FITSCatalogue
@@ -299,6 +305,7 @@ def process(
         mode="",
         cat_path=None,
         moments=False,
+        n_epoch_slots=None,
     ):
         """Process Catalogue.
 
@@ -310,6 +317,9 @@ def process(
             Path to input catalogue
         moments : bool
             Option to run ``ngmix`` mode with moments
+        n_epoch_slots : int, optional
+            Number of per-epoch slots in ``psf`` mode; if ``None``, the
+            tile's ``max(N_EPOCH) + 1``
 
         Returns
         --------
@@ -326,7 +336,7 @@ def process(
         if mode == "ngmix":
             err_msg = self._save_ngmix_data(cat_path, moments)
         elif mode == "psf":
-            self._save_psf_data(cat_path)
+            self._save_psf_data(cat_path, n_epoch_slots)
         else:
             err_msg = (
                 f"Invalid process mode ({mode}) for "
@@ -412,6 +422,13 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False):
         moments : bool, optional
             If True, write the parallel ``NGMIXm_*`` (moments-branch) columns.
 
+        @sc [decision:catalogue_assembly.failure_sentinels,label:coupling] failure-sentinel-cut-semantics
+        Missing-row ``T``, ``SNR``, flux, magnitude, PSF-size and flag values
+        initialize to 0, which is in range and cannot identify failure. Use
+        ``NGMIX_N_EPOCH == 0`` for that cut; the -10 ellipticity, -1
+        flux/magnitude-error and 1e30 size-error sentinels are out of range and
+        can also identify missing fits.
+
         """
         self._key_ends = ["1M", "1P", "2M", "2P", "NOSHEAR"]
 
@@ -614,10 +631,14 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False):
 
         return None
 
-    def _save_psf_data(self, galaxy_psf_path):
+    def _save_psf_data(self, galaxy_psf_path, n_epoch_slots=None):
         """Save PSF data.
 
-        Save the PSF catalogue into the final one.
+        Save the PSF catalogue into the final one, as per-epoch column
+        families ``HSM_*_PSF_n`` (``psf_shape_cols``), ``EXP_ID_n`` and
+        ``CCD_n``. Slot ``n`` of every
+        family refers to the same epoch; slots no epoch fills keep the
+        family's sentinel.
 
         @sc [label:schema] psf-epoch-slot-columns
         The per-epoch families are ``psf_shape_cols`` plus ``EXP_ID``/``CCD``,
@@ -629,11 +650,23 @@ def _save_psf_data(self, galaxy_psf_path):
         ----------
         galaxy_psf_path : str
             Path to the PSF catalogue to save
+        n_epoch_slots : int, optional
+            Number of slots written per family; if ``None``, the tile's
+            ``max(N_EPOCH) + 1``
+
+        Raises
+        ------
+        ValueError
+            If an object has more epochs than ``n_epoch_slots``
 
         """
         galaxy_psf_cat = SqliteDict(galaxy_psf_path)
 
-        max_epoch = np.max(self._final_cat_file.get_data()["N_EPOCH"]) + 1
+        n_epoch = self._final_cat_file.get_data()["N_EPOCH"]
+        if n_epoch_slots is None:
+            n_slots = np.max(n_epoch) + 1
+        else:
+            n_slots = n_epoch_slots
         n_obj = len(self._obj_id)
 
         # Per-epoch PSF shape columns copied from the producer's SHAPES dict:
@@ -657,17 +690,26 @@ def _save_psf_data(self, galaxy_psf_path):
         self._output_dict = {
             f"{name}_{idx + 1}": np.full(n_obj, fill, dtype=dtype)
             for name, fill, dtype in psf_shape_cols + epoch_id_cols
-            for idx in range(max_epoch)
+            for idx in range(n_slots)
         }
 
         for idx, id_tmp in enumerate(self._obj_id):
 
-            if galaxy_psf_cat[str(id_tmp)] == "empty":
+            obj_epochs = galaxy_psf_cat[str(id_tmp)]
+            if obj_epochs == "empty":
                 continue
 
-            for epoch, key in enumerate(galaxy_psf_cat[str(id_tmp)].keys()):
+            if len(obj_epochs) > n_slots:
+                galaxy_psf_cat.close()
+                raise ValueError(
+                    f"Object {id_tmp} has {len(obj_epochs)} PSF epochs"
+                    + f" (N_EPOCH={n_epoch[idx]}), more than the {n_slots}"
+                    + f" per-epoch slots (N_EPOCH_SLOTS={n_epoch_slots})"
+                )
+
+            for epoch, (key, gpc_data) in enumerate(obj_epochs.items()):
 
-                shapes = galaxy_psf_cat[str(id_tmp)][key]["SHAPES"]
+                shapes = gpc_data["SHAPES"]
 
                 # `key` is "<exp>-<ccd>"; reading it in the enumeration that
                 # assigns `epoch` aligns EXP_ID_n/CCD_n with HSM_*_PSF_n by

src/shapepipe/modules/make_cat_runner.py

@@ -37,7 +37,12 @@ def make_cat_runner(
     module_config_sec,
     w_log,
 ):
-    """Define The Make Catalogue Runner."""
+    """Define The Make Catalogue Runner.
+
+    @sc [decision:catalogue_assembly.star_galaxy_classification]
+
+    @sc [decision:masking.sky_mask_application]
+    """
     # Set input file paths
     if len(input_file_list) == 3:
         # No spread model input
@@ -82,6 +87,10 @@ def make_cat_runner(
         save_psf = config.getboolean(module_config_sec, "SAVE_PSF_DATA")
     else:
         save_psf = False
+    if config.has_option(module_config_sec, "N_EPOCH_SLOTS"):
+        n_epoch_slots = config.getint(module_config_sec, "N_EPOCH_SLOTS")
+    else:
+        n_epoch_slots = None
 
     # Set final output file
     final_cat_file = make_cat.prepare_final_cat_file(
@@ -135,7 +144,9 @@ def make_cat_runner(
             w_log.info(err_msg)
 
     if save_psf:
-        err_msg = sc_inst.process("psf", galaxy_psf_path)
+        err_msg = sc_inst.process(
+            "psf", galaxy_psf_path, n_epoch_slots=n_epoch_slots
+        )
 
     # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38):
     # add one MASK_<BAND> column per band, queried at each object's world

src/shapepipe/modules/mask_query_runner.py

@@ -26,7 +26,15 @@ def mask_query_runner(
     module_config_sec,
     w_log,
 ):
-    """Define The Mask Query Runner."""
+    """Define The Mask Query Runner.
+
+    @sc [decision:masking.psf_star_mask_veto,label:scope] mask-query-carries-not-cuts
+    Without MASK_PATHS the catalogue passes through with no MASK_EXT column;
+    with it, MASK_EXT is carried for measurement and nothing here removes a
+    star. The PSF-star veto is a setools edit (``MASK_EXT == 0`` beside
+    IMAFLAGS_ISO), not a change here.
+
+    """
     sexcat_path = input_file_list[0]
 
     # Get file prefix (optional)

src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py

@@ -249,6 +249,8 @@ def mccd_fit_pipeline(
 
     Fit the MCCD model to the Observations.
 
+    @sc [decision:star_selection_psf.psf_modelling_software]
+
     Parameters
     ----------
     trainstar_path : str

src/shapepipe/modules/merge_headers_package/merge_headers.py

@@ -35,6 +35,7 @@ def merge_headers(input_file_list, output_dir, tile_number=None):
     TypeError
         For invalid ``output_dir`` type
 
+    @sc [decision:preparation.astrometric_solution_source]
     """
     if not isinstance(output_dir, str):
         raise TypeError(