Wire the UNIONS external healsparse masks into the workflow (MASK_n* columns)

astra.yaml

@@ -222,9 +222,9 @@ analyses:
         description: >-
           Exposure detection catalogues carrying IMAFLAGS_ISO (and MASK_EXT when
           maps are configured), the flag stamps ngmix reads, and the final
-          catalogue's optional MASK_<band> columns.
+          catalogue's MASK_n<bit> columns.
         inputs: [exposure_flags, sky_masks]
-        decisions: [pixel_mask_source, psf_star_mask_veto, sky_mask_application]
+        decisions: [pixel_mask_source, psf_star_mask_veto, sky_mask_application, mask_default_cut]
     decisions:
       pixel_mask_source:
         label: Only the instrument flag image reaches pixels
@@ -291,27 +291,74 @@ analyses:
               Halos flag objects for the final catalogue; a star inside another
               star's halo is not thereby a bad PSF sample.
       sky_mask_application:
-        label: Object-level sky masking deferred downstream
+        label: Sky masks written as catalogue columns; the cut is deferred downstream
         rationale: >-
-          The final catalogue ships every detected object. When MASK_EXT_PATHS
-          lists band:path pairs, make_cat queries each healsparse map at the
+          The final catalogue ships every detected object. MASK_EXT_PATHS lists
+          label:path pairs, and make_cat queries each healsparse map at the
           object's windowed position and writes the map value verbatim into a
-          MASK_<band> column; an object off coverage gets the map's sentinel
+          MASK_<label> column; an object off coverage gets the map's sentinel
           (False for boolean maps, reading as unmasked; typically -1 for
-          integer maps). The committed config sets no MASK_EXT_PATHS, so no
-          mask column is written and all mask cuts happen downstream.
-        default: deferred_downstream
+          integer maps). The committed config names the UNIONS band-combined
+          ugriz mask ladder, one boolean map per bit (nside 131072, True =
+          masked), labelled n<value>: eleven MASK_n* columns, matched line for
+          line in final_cat.param. The flag value identifies the column;
+          source labels such as faint star halos (n1, bit 0), bright star halos
+          (n2, bit 1), HSC z (n256) and Pan-STARRS z2 (n2048) describe
+          the maps. Nothing in the pipeline cuts on them
+          (mask_default_cut).
+        default: catalogue_columns
         options:
           deferred_downstream:
             label: No mask columns; all objects shipped
           catalogue_columns:
-            label: Per-band MASK_<band> columns from MASK_EXT_PATHS, no cut
+            label: Per-bit MASK_<label> columns from MASK_EXT_PATHS, no cut
           pipeline_cut:
             label: Drop masked objects inside the pipeline
             excluded: true
             excluded_reason: >-
               Location flags are analysis decisions; a pipeline cut would fix
               one mask version into the catalogue.
+      mask_default_cut:
+        label: Default external-mask cut — the six r-mask bits of the MASK_n* ladder
+        rationale: >-
+          make_cat writes one boolean MASK_n<value> column per map in
+          MASK_EXT_PATHS (11 bits: faint star halos n1, bright star halos n2,
+          star body n4, manual n8,
+          per-band no-data footprints n16..n256 (n256 is HSC z), MaxiMask
+          n1024, Pan-STARRS z-band footprint n2048 (z2)) and cuts nothing;
+          the cut is the consumer's
+          (sp_validation). The default is n1|n2|n4|n8|n64|n1024 because it
+          reproduces the published 2D-cosmic-shear mask
+          (/n17data/UNIONS/WL/masks/mask_r_nside131072.hsp), so the
+          catalogue's footprint matches the 2D analysis'. Measured on candide
+          against the Feb-2025 r ladder beside it: pixel-exact on three whole
+          interior coverage granules; on 4M random footprint pixels the only
+          difference is 0.46% of area at footprint-edge granules where mask_r
+          has no coverage and reads unmasked. Every bit is needed: dropping
+          any one leaves mask_r-only area (n2 most, 4.4%). The halo names
+          follow the producer schema: n1 (bit 0, value 1) is faint star halos
+          and n2 (bit 1, value 2) is bright star halos. Both enter the default
+          selection. The DR6 ladder MASK_EXT_PATHS points at is not
+          yet measured against the Feb-2025 one.
+        default: r_mask_bits
+        options:
+          r_mask_bits:
+            label: "OR of n1|n2|n4|n8|n64|n1024 — reproduces mask_r"
+          defects_only:
+            label: "OR of the defect bits n1|n2|n4|n8|n1024"
+            excluded: true
+            excluded_reason: >-
+              Omits the r-footprint bit n64 (no r data) and so misses 9% of
+              mask_r's masked area; the footprint no longer matches the 2D
+              analysis'.
+          all_bits:
+            label: OR of all 11 MASK_n* columns
+            excluded: true
+            excluded_reason: >-
+              The u/g/i/HSC-z footprint bits and Pan-STARRS z2 bit n2048 flag
+              missing data in other
+              filters, not r-band defects; OR-ing them masks whole regions
+              with no u or Pan-STARRS z2 coverage.
     prior_insights:
       farrens22_pipeline_masks:
         claim: >-

src/shapepipe/modules/make_cat_package/make_cat.py

@@ -190,6 +190,16 @@ def save_mask_ext_data(final_cat_file, band_paths, w_log):
     Query ``XWIN_WORLD`` and ``YWIN_WORLD`` in catalogue order and pass the
     ``query_map`` result to ``MASK_<BAND>`` unchanged.
 
+    @sc [decision:masking.mask_default_cut,label:convention] mask-ext-ladder-columns
+    The labels in ``MASK_EXT_PATHS`` are the UNIONS mask ladder's bit names
+    (``n1`` ... ``n2048``), and this function writes ``MASK_<label>`` verbatim.
+    The eleven ``MASK_n*`` lines in ``workflow/config/cfis/final_cat.param``
+    must match those labels line for line: the post-processing merge fails
+    every tile on a name this function did not write. Nothing here cuts; the
+    catalogue's default cut, applied by the consumer, is the OR of the six
+    r-mask bits ``n1|n2|n4|n8|n64|n1024`` (astra decision
+    ``masking.mask_default_cut``), not the OR of every column.
+
     Parameters
     ----------
     final_cat_file : file_io.FITSCatalogue

src/shapepipe/pipeline/file_io.py

@@ -1568,7 +1568,7 @@ def _get_fits_col_type(self, col_data):
             col_type = "K"
         elif type(col_data[0]) in [float, np.float16, np.float32, np.float64]:
             col_type = "D"
-        elif type(col_data[0]) is bool:
+        elif type(col_data[0]) in [bool, np.bool_]:
             col_type = "L"
         elif type(col_data[0]) in [str, np.str_]:
             col_type = "A"

tests/cluster/test_r_mask_bits.py

@@ -0,0 +1,64 @@
+"""The OR of the six r-ladder bits is the published 2D-cosmic-shear r mask.
+
+Decision ``mask_default_cut`` (astra.yaml, catalogue_assembly) makes the
+catalogue's default cut ``n1|n2|n4|n8|n64|n1024``, on the claim that this OR
+reproduces ``mask_r_nside131072.hsp``, the mask the 2D analysis used. This
+test checks that claim pixel-exactly on three whole coverage granules
+(nside_cov 128, 2^20 nside-131072 pixels each) lying fully inside the v1.6.x
+footprint. The granules sit in the interior on purpose: at the footprint
+edge ``mask_r`` has no coverage and reads unmasked where the ladder masks, a
+difference of order half a percent of area that carries no r data.
+
+It reads the Feb-2025 r-band ladder beside the published mask. The DR6 ladder
+that MASK_EXT_PATHS points at is not what this checks. Enforces contract
+``mask-ext-ladder-columns``; skips when the maps are not readable.
+"""
+
+import os
+
+import numpy as np
+import pytest
+
+
+pytestmark = [pytest.mark.slow, pytest.mark.candide]
+
+MASK_DIR = "/n17data/UNIONS/WL/masks"
+R_MASK = f"{MASK_DIR}/mask_r_nside131072.hsp"
+R_BITS = ["n1", "n2", "n4", "n8", "n64", "n1024"]
+# Whole nside_cov=128 granules fully inside coverage_v1.6.x.
+INTERIOR_GRANULES = [32131, 29770, 43619]
+
+
+def _paths():
+    return [R_MASK] + [
+        f"{MASK_DIR}/mask_r_nside131072_{bit}.hsp" for bit in R_BITS
+    ]
+
+
+@pytest.mark.parametrize("granule", INTERIOR_GRANULES)
+def test_six_bit_or_reproduces_mask_r(granule):
+    """OR of the six r-ladder bits equals mask_r on a whole granule."""
+    unreadable = [p for p in _paths() if not os.access(p, os.R_OK)]
+    if unreadable:
+        pytest.skip(f"mask maps not readable: {unreadable}")
+    hsp = pytest.importorskip("healsparse")
+
+    pixels = granule * (1 << 20) + np.arange(1 << 20)
+
+    def read(path):
+        # A bit map with no coverage here reads its sentinel, False: unmasked.
+        if not hsp.HealSparseCoverage.read(path).coverage_mask[granule]:
+            assert path != R_MASK, f"mask_r has no coverage in {granule}"
+            return np.zeros(pixels.size, dtype=bool)
+        sub = hsp.HealSparseMap.read(path, pixels=[granule])
+        return sub.get_values_pix(pixels, nest=True).astype(bool)
+
+    mask_r = read(R_MASK)
+    combined = np.logical_or.reduce([read(p) for p in _paths()[1:]])
+
+    assert mask_r.any(), f"granule {granule}: mask_r masks nothing"
+    n_diff = int(np.count_nonzero(mask_r != combined))
+    assert n_diff == 0, (
+        f"mask_default_cut: granule {granule}: OR of {R_BITS} differs from "
+        f"mask_r on {n_diff} of {pixels.size} pixels"
+    )

tests/module/test_make_cat_mask_ext.py

@@ -16,8 +16,8 @@
 
 healsparse = pytest.importorskip("healsparse")
 
-from shapepipe.modules.make_cat_package import make_cat
-from shapepipe.pipeline import file_io
+from shapepipe.modules.make_cat_package import make_cat  # noqa: E402
+from shapepipe.pipeline import file_io  # noqa: E402
 
 
 class _NullLogger:
@@ -49,7 +49,7 @@ def _make_map(value, dtype=np.int16, sentinel=-1):
 
 
 def _write_final_cat(path):
-    """Write a synthetic final_cat FITS with a RESULTS ext of known positions."""
+    """Write a final_cat FITS with a RESULTS extension of known positions."""
     data = np.empty(
         len(RA),
         dtype=[
@@ -127,3 +127,36 @@ def test_mask_ext_absent_is_noop(tmp_path):
 
     assert cols == {"NUMBER", "XWIN_WORLD", "YWIN_WORLD"}
     assert not any(name.startswith("MASK_") for name in cols)
+
+
+def test_boolean_halo_maps_preserve_bit_identity(tmp_path):
+    """n1 faint and n2 bright halo columns preserve separate boolean maps."""
+    paths = {}
+    for label, values in (
+        ("n2", [False, True, True]),
+        ("n1", [True, False, True]),
+    ):
+        smap = healsparse.HealSparseMap.make_empty(
+            NSIDE_COVERAGE, NSIDE_SPARSE, np.bool_, sentinel=False
+        )
+        smap.update_values_pos(
+            RA[:3], DEC[:3], np.array(values, dtype=bool), lonlat=True
+        )
+        path = tmp_path / f"mask_{label}.hsp"
+        smap.write(str(path))
+        paths[label] = str(path)
+
+    cat_path = tmp_path / "final_cat-halos.fits"
+    _write_final_cat(cat_path)
+    cat = file_io.FITSCatalogue(
+        str(cat_path), open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite
+    )
+    make_cat.save_mask_ext_data(cat, paths, _NullLogger())
+
+    cat.open()
+    data = cat.get_data()
+    npt.assert_array_equal(data["MASK_n1"], [True, False, True, False])
+    npt.assert_array_equal(data["MASK_n2"], [False, True, True, False])
+    assert data["MASK_n1"].dtype == np.dtype(bool)
+    assert data["MASK_n2"].dtype == np.dtype(bool)
+    cat.close()

tests/unit/test_mask_ext_columns.py

@@ -0,0 +1,46 @@
+"""The MASK_n* columns final_cat.param asks for are the ones make_cat writes.
+
+make_cat writes one ``MASK_<label>`` column per ``label:path`` pair in
+``MASK_EXT_PATHS`` (config_tile_Mc.ini); the post-processing merge reads
+``final_cat.param`` and fails every tile on a name make_cat did not write.
+Enforces contract ``mask-ext-ladder-columns``.
+"""
+
+import configparser
+import os
+import re
+from pathlib import Path
+
+from shapepipe.modules.make_cat_package import make_cat
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+CONFIG_DIR = REPO_ROOT / "workflow" / "config" / "cfis"
+
+
+def _written_mask_columns(monkeypatch):
+    monkeypatch.setenv("SP_INPUT_MASKS", "/dummy/masks")
+    config = configparser.ConfigParser(interpolation=None)
+    config.read(CONFIG_DIR / "config_tile_Mc.ini")
+    raw = config.get("MAKE_CAT_RUNNER", "MASK_EXT_PATHS")
+    band_paths = make_cat.parse_mask_ext_paths(os.path.expandvars(raw))
+    for path in band_paths.values():
+        assert path.startswith("/dummy/masks/"), path
+    return [f"MASK_{label}" for label in band_paths]
+
+
+def _requested_mask_columns():
+    lines = (CONFIG_DIR / "final_cat.param").read_text().splitlines()
+    return [line.strip() for line in lines if re.match(r"MASK_", line.strip())]
+
+
+def test_final_cat_mask_columns_match_mask_ext_paths(monkeypatch):
+    """final_cat.param's MASK_* lines are MASK_EXT_PATHS's labels, in order."""
+    written = _written_mask_columns(monkeypatch)
+    requested = _requested_mask_columns()
+    assert len(written) == 11, written
+    assert requested == written, (
+        "mask-ext-ladder-columns: final_cat.param's MASK_* lines "
+        f"{requested} != the columns MASK_EXT_PATHS makes make_cat write "
+        f"{written}"
+    )

tests/workflow/params_pin.json

@@ -4,40 +4,40 @@
     "all": "572122d8d1901e12ff591b30adf405f8920be8183129befececbb819f3392ed8",
     "clean_exposure": "22cb76b13a5205d20a02a9bd3b8c8bea5ea2801555e24dd2b84b11145f7e79d9",
     "clean_tile": "a5c07b0461526ed407df36a291deb866d4181524b4c8e0fbc3dd047fd9d28479",
-    "exp_get_images": "71e76ff7f96af1c5d2b85697cc5819e2271911f177a2075253f3b8cfa268c1a9",
+    "exp_get_images": "a9e9eca0a99a348e43e0cd44d0c0aae0f890773d03e35007fb77f886b7d21f5f",
     "exp_persist": "302e2837542bc1102430c27c81c600b7cda32e8bddcb5fd60d33950987609fff",
-    "exp_psf": "2c4f6d00f1939ccbf05b4982a0202a0ff92a4373a727f4e00f3aaab4eba03352",
-    "exp_split": "6e954f8f3d06f44d3f164675912ce27d6216648855d04968f9168bd7d0f2c4fa",
+    "exp_psf": "87b9fd799a54130eb3dd9c78bb5124838854c23f365df7a991a53f4c6ac4c3ff",
+    "exp_split": "8fc97dd48e76cd6c7fdf9ee388ba1c1a3dd63428cbe1961641fbebe76b05786e",
     "final_cat_merge": "f6222c98be9777cbab4131d8857bd3817719df9302d0a321c6cd99f67d5bdc76",
     "prepare_all_tiles": "b8f872a22adf014e25a7fa5198f49b71a6fe9e56042ed82b682bc8763970a844",
     "star_cat_merge": "6277450958474af5270982fa35360f2f237a29f7533c526ee9265dfd5acc07a0",
-    "tile_detect": "1b4b0871bf28296f8e5d6855ae47ff2cb84542c93593a5ae5c0ffc3c84c45479",
+    "tile_detect": "f3a65e49a46c01451a8f6240bccd7d4136d7bc6585b08928669a2627224df9ec",
     "tile_exp_forest": "7447ab4a1049de5f0b5c81e5f9ed2a8644c7bdab85cb0a060bfde81261a89b28",
-    "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184",
-    "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c",
-    "tile_get_images": "331a67e747f211ebf4c14b946a7af7f9fc9f55243d69d2791d74aecc3ca228c3",
-    "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23",
-    "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb",
-    "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111",
-    "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415",
-    "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3",
-    "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91"
+    "tile_find_exposures": "c883495a7ec196c61ef770507865f43834b08523793d337e8ddeaef0ad62f117",
+    "tile_get_catalogue": "84ffd03c4bde248c06aa58a0f4b7f399390a0ff3ecbcf07e03d718df4fe0cd7f",
+    "tile_get_images": "96e235dad9cdcde8ecf11f4516bc2303dbba1c8115ad343cdb26d3d1513da671",
+    "tile_make_cat": "8c0a8834f39a7e77d81c0d9ca338d4b6ff343c30a2c9b6cce18a0b960ddf8fee",
+    "tile_merge_cats": "0783c93deeb7f90bdb9772bbc4117851dff1a671518c81f67edd4c7c79cc7e6a",
+    "tile_merge_headers": "3e674f961fd5bb93ce1f3b0e2bb072a25a421f2828315a3db45a960c44a42111",
+    "tile_ngmix": "35b1c19ff7157349b1b51a810c12b60dd3c022fd781bc491fc6d5561b3aa5fe4",
+    "tile_uncompress": "4a013cdd04bc3e3c1082e477161d2b03e0eefe280b1aabb9af25c149a977f55b",
+    "tile_vignets": "b9853b2a1830840a21d8ec1f0877e780868a07da57a29f531df47f26fe8692ac"
   },
   "schema": 1,
-  "sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e",
+  "sha256": "7aaed3b051ecd0434ff7fc13614fd86a4988e98087497f6f63ed5612a84f0386",
   "unit_pre": {
-    "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c",
-    "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec",
-    "exp_split": "358fa8bbe59680d4f9839e007b343dd25fed733cf3156dccd30d305d33ac9480",
-    "tile_detect": "adad5d671fa65dd04433e1c82a725b845635e9d2e368a1e70833ed990df2d55a",
-    "tile_find_exposures": "c5922fb507f6fd040a179b53fba0818697661016c9688984b6ce249186dc6986",
-    "tile_get_catalogue": "b124d12252617abb8df8a98d6234ddfd6cc503e19fa72c9a0c48c21559d6f48c",
-    "tile_get_images": "45b47c44bfeb34ac973b89d4e752c8c82e78028f9f0da379c44627005b45e279",
-    "tile_make_cat": "7579a52e32e76523c0b77d75c468a47d7f2e2ca0c55865f40e9628a7481cd79d",
-    "tile_merge_cats": "69cfe94ba941d2ba7b1ce24883961b47b191b80aff381da1865c6bc38de0c354",
-    "tile_merge_headers": "8590cfa3281c88d43eb8c4fe5760176a13e928a419cbea9bea3a8f88b0634b42",
-    "tile_ngmix": "b6b75b553a62df54aea6c337dcde1e0cdcd0fe30000e86880c1a4689e42c4a72",
-    "tile_uncompress": "91a15538491e53ee2b0d52b0472909374270918e4b4c80880c4f5a8329e40161",
-    "tile_vignets": "a6111cef708aa3c9fb0144b49de9f781eef84d2096ba4f8a3de9bb45b1a3774d"
+    "exp_get_images": "dac6685a207dae3ea81d296ce53e9ba75d4636cf1f81e6399f2dda452680c992",
+    "exp_psf": "87fc8ea1153a709ab7ba56542cfacc041f5f8b7f6cf2f38a9bc9d8004d1cecb4",
+    "exp_split": "922a62360969a7d8775172debdb77469ab8552d8031fc559d92d63fc30443122",
+    "tile_detect": "cdd19095ce9e33847284825ed013aa0a35011e9877a3d89d6b8dc0349c6b7707",
+    "tile_find_exposures": "89f0816fcfb513154aa81783cc361add768b0101b4cce19426bdf4b43db0cf71",
+    "tile_get_catalogue": "2030933b663be3d229d328ee6266eda5a63c98589cafa1151a203e88c8dc733a",
+    "tile_get_images": "482acaad46e677ec9d67105c48b5d56116c7dff2d77f4a62bda73061721a7103",
+    "tile_make_cat": "af03252466961e9613fe800e42dff32ff9a3f476726012a1fe8d1c18067ee95d",
+    "tile_merge_cats": "9c6cfbe27fa9618df2418953c056c8ad7cdf64d4b6b0d8a73a32168c39b82114",
+    "tile_merge_headers": "6bd3343d183e5413cedb6b82493445602cc139d4b56fde39281c59085a0efa8d",
+    "tile_ngmix": "47abe4514dc948fb2115d244853da1f792c9f7eea51a207f64de3ee563dad76c",
+    "tile_uncompress": "bf181cb759b56143c7f086f614c559d4cececa54ae7f4c55b78fd8bfad096ba5",
+    "tile_vignets": "348a047b4c2e22d0d68b819a1b0ac3c9677c1ee7914557ac257139290548127a"
   }
 }

universes/committed.yaml

@@ -9,7 +9,8 @@ analyses:
     decisions:
       pixel_mask_source: instrument_flags_only
       psf_star_mask_veto: instrument_flags_only
-      sky_mask_application: deferred_downstream
+      sky_mask_application: catalogue_columns
+      mask_default_cut: r_mask_bits
   detection:
     decisions:
       tile_detection: unions_catalogue