From 8e91bdd3c65ae027f7364b88ed7d14e48d1df878 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 03:50:23 +0200 Subject: [PATCH 1/5] fix(make_cat): return actual saved count from save_sm_data save_sm_data returned the n_obj argument it was given instead of the number of rows it actually saved, so the runner's SExtractor-vs- spread-model size check compared a value with itself and could never detect a mismatch. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014bvNTrAmZxcfb1ee83ApPK --- src/shapepipe/modules/make_cat_package/make_cat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 062ac5719..de7711262 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -207,7 +207,7 @@ def save_sm_data( final_cat_file.close() - return n_obj + return len(sm) def parse_mask_ext_paths(paths_str): From 81ed2d0debd60d2924fb4064b7251a34dcc0ee30 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 03:50:30 +0200 Subject: [PATCH 2/5] fix(make_cat): fix crashing size-mismatch log and honest spread-model logging - The size-mismatch warning called w_log(...) directly; w_log is a logger, not a callable, so a real mismatch crashed instead of being reported. Use w_log.warning(...), matching other runners, and fix the malformed message. - With no spread-model input, the runner logged "setting spread model to 99" but wrote no SPREAD_MODEL column at all. The log now says what actually happens; behaviour for the committed three-input path (SM_DO_CLASSIFICATION off) is unchanged. - SM_DO_CLASSIFICATION = True with no spread-model input is now a config error (there is nothing to classify on), raised clearly instead of being silently ignored. - A missing SM_STAR_THRESH/SM_GAL_THRESH under classification now raises a clear ValueError instead of a bare configparser.NoOptionError. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014bvNTrAmZxcfb1ee83ApPK --- src/shapepipe/modules/make_cat_runner.py | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 176341757..6633622e4 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -62,6 +62,18 @@ def make_cat_runner( "SM_DO_CLASSIFICATION", ) if do_classif: + if sexcat_sm_path is None: + raise ValueError( + "SM_DO_CLASSIFICATION is True but no spread-model " + "catalogue input was provided; star/galaxy classification " + "requires a spread-model input to classify on" + ) + for key in ("SM_STAR_THRESH", "SM_GAL_THRESH"): + if not config.has_option(module_config_sec, key): + raise ValueError( + f"SM_DO_CLASSIFICATION is True but {key} is not set " + "in the config" + ) star_thresh = config.getfloat(module_config_sec, "SM_STAR_THRESH") gal_thresh = config.getfloat(module_config_sec, "SM_GAL_THRESH") else: @@ -96,7 +108,10 @@ def make_cat_runner( # Save spread-model data if sexcat_sm_path is None: - w_log.info("No sm cat input, setting spread model to 99") + w_log.info( + "No sm cat input, spread model will not be written to the " + "final catalogue" + ) else: w_log.info("Save spread-model data") cat_size_sm = make_cat.save_sm_data( @@ -109,10 +124,10 @@ def make_cat_runner( ) if cat_size_sextractor != cat_size_sm: - w_log( - f"Warnign: SExtractor catalogue {tile_sexcat_path} has different size" - + f" ({cat_size_sextractor} than spread_model catalogue" - + f" {sexcat_sm_path} ({cat_size_sm})" + w_log.warning( + f"SExtractor catalogue {tile_sexcat_path} has different" + f" size ({cat_size_sextractor}) than spread_model" + f" catalogue {sexcat_sm_path} ({cat_size_sm})" ) # Save shape data From e22bad13f0a8cd209d960c6a2903e2a04911a361 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 03:50:35 +0200 Subject: [PATCH 3/5] test(make_cat): cover spread-model path defects in save_sm_data/make_cat_runner One test per defect fixed in make_cat_runner/save_sm_data, each verified to fail against the pre-fix code and pass after: save_sm_data returning its true saved count, the size-mismatch warning logging without crashing, honest no-input logging with no SPREAD_MODEL column written (matching the committed three-input path), refusing SM_DO_CLASSIFICATION without a spread-model input, and a clear error for a missing SM_STAR_THRESH/SM_GAL_THRESH. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014bvNTrAmZxcfb1ee83ApPK --- tests/module/test_make_cat.py | 302 ++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index 34a1958e8..db068ea52 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -16,11 +16,16 @@ import numpy as np import numpy.testing as npt +import pytest from astropy.io import fits from sqlitedict import SqliteDict +from shapepipe.modules.make_cat_package import make_cat from shapepipe.modules.make_cat_package.make_cat import SaveCatalogue +from shapepipe.modules.make_cat_runner import make_cat_runner from shapepipe.modules.ngmix_package.ngmix import Ngmix +from shapepipe.pipeline import file_io +from shapepipe.pipeline.config import CustomParser class _NullLogger: @@ -448,3 +453,300 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): assert out[col][0] == -1, col for col in ("EXP_ID_1", "CCD_1", "EXP_ID_2", "CCD_2", "EXP_ID_3", "CCD_3"): assert out[col][1] == -1, col + + +# --- make_cat_runner / save_sm_data: spread-model path defects (fix/makecat) --- + + +class _RecordingLogger: + """Minimal stand-in for the pipeline's ``w_log``, recording calls.""" + + def __init__(self): + self.infos = [] + self.warnings = [] + + def info(self, msg, *_args, **_kwargs): + self.infos.append(msg) + + def warning(self, msg, *_args, **_kwargs): + self.warnings.append(msg) + + +def _write_sex_like_cat(path, data): + """Write a minimal SExtractor-format FITS (data lives at HDU index 2).""" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.BinTableHDU(name="LDAC_IMHEAD"), + fits.BinTableHDU(data, name="LDAC_OBJECTS"), + ] + ).writeto(str(path), overwrite=True) + + +def test_save_sm_data_returns_actual_saved_count(tmp_path): + """``save_sm_data`` must report how many rows it wrote, not its ``n_obj``. + + The runner compares this count against the SExtractor catalogue's size + to catch a mismatch. If ``save_sm_data`` merely echoed back the ``n_obj`` + it was handed (as it did before this fix), the two sides of that + comparison are the same value by construction and the check can never + fire. + """ + final_cat_path = tmp_path / "final_cat-0.fits" + final_cat_file = file_io.FITSCatalogue( + str(final_cat_path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + final_cat_file.save_as_fits( + np.array([(1,), (2,), (3,)], dtype=[("NUMBER", "i8")]), + ext_name="RESULTS", + ) + + sm_path = tmp_path / "sexcat_sm.fits" + _write_sex_like_cat( + sm_path, + np.array( + [(0.001, 0.0001), (0.002, 0.0002)], + dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], + ), + ) + + n_saved = make_cat.save_sm_data( + final_cat_file, + str(sm_path), + do_classif=False, + n_obj=3, + ) + + # Two rows were actually written to the spread-model catalogue, not the + # three the (mismatched) SExtractor catalogue carried. + assert n_saved == 2 + + +def _write_matching_ngmix_cat(path, obj_ids): + """Reuse the module's ngmix-catalogue writer for full-runner tests.""" + _write_ngmix_cat(path, obj_ids) + + +def _run_make_cat_runner(tmp_path, obj_ids, sm_obj_ids, do_classif=False): + """Drive ``make_cat_runner`` end to end with synthetic 4-input data.""" + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat( + tile_sexcat_path, + np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), + ) + + sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" + _write_sex_like_cat( + sexcat_sm_path, + np.array( + [(0.001, 0.0001) for _ in sm_obj_ids], + dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], + ), + ) + + ngmix_path = tmp_path / "ngmix-350-100.fits" + _write_matching_ngmix_cat(ngmix_path, obj_ids) + + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" # unused: no SAVE_PSF_DATA + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": str(do_classif), + "SHAPE_MEASUREMENT_TYPE": "ngmix", + **( + {"SM_STAR_THRESH": "0.003", "SM_GAL_THRESH": "0.01"} + if do_classif + else {} + ), + } + } + ) + + w_log = _RecordingLogger() + result = make_cat_runner( + [ + str(tile_sexcat_path), + str(sexcat_sm_path), + str(galaxy_psf_path), + str(ngmix_path), + ], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + w_log, + ) + return result, w_log + + +def test_make_cat_runner_warns_on_size_mismatch_without_crashing(tmp_path): + """A SExtractor/spread-model size mismatch is logged, not fatal. + + The runner used to call ``w_log(...)`` directly -- but ``w_log`` is a + logger, not a callable -- so a genuine size mismatch crashed instead of + being reported. With three SExtractor objects and only two spread-model + rows, the run must complete and the mismatch must show up as a warning. + """ + obj_ids = [1, 2, 3] + result, w_log = _run_make_cat_runner( + tmp_path, obj_ids=obj_ids, sm_obj_ids=[1, 2] + ) + + assert result == (None, None) + assert any( + "3" in msg and "2" in msg and "different" in msg + for msg in w_log.warnings + ), w_log.warnings + + +def test_make_cat_runner_no_sm_input_logs_honestly_and_writes_no_column( + tmp_path, +): + """With no spread-model input, the log must match reality: no column. + + The three-input path (no spread-model catalogue) used to log "setting + spread model to 99" while writing no ``SPREAD_MODEL`` column at all. + The log message must now describe what actually happens, and the + column must still be absent -- this is the committed, currently-run + configuration (``SM_DO_CLASSIFICATION = False``), so its output must + not change. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat( + tile_sexcat_path, + np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), + ) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + _write_matching_ngmix_cat(ngmix_path, obj_ids) + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": "False", + "SHAPE_MEASUREMENT_TYPE": "ngmix", + } + } + ) + w_log = _RecordingLogger() + + result = make_cat_runner( + [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + w_log, + ) + + assert result == (None, None) + assert not any("99" in msg for msg in w_log.infos) + + final_cat = file_io.FITSCatalogue( + str(make_cat.get_output_name(str(tmp_path), "-350-100")) + ) + final_cat.open() + assert "SPREAD_MODEL" not in final_cat.get_data().dtype.names + final_cat.close() + + +def test_make_cat_runner_refuses_classification_without_sm_input(tmp_path): + """``SM_DO_CLASSIFICATION = True`` with no spread-model input is a config error. + + Classifying stars/galaxies needs real spread-model values; with no + spread-model catalogue there is nothing to classify on, so this must + raise clearly rather than silently skip classification (the prior + behaviour) or crash deep inside the classifier. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat( + tile_sexcat_path, + np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), + ) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": "True", + "SM_STAR_THRESH": "0.003", + "SM_GAL_THRESH": "0.01", + "SHAPE_MEASUREMENT_TYPE": "ngmix", + } + } + ) + w_log = _RecordingLogger() + + with pytest.raises(ValueError, match="SM_DO_CLASSIFICATION"): + make_cat_runner( + [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + w_log, + ) + + +def test_make_cat_runner_raises_clear_error_for_missing_threshold_config( + tmp_path, +): + """A missing ``SM_STAR_THRESH``/``SM_GAL_THRESH`` fails clearly, not deep in configparser. + + Before this fix, a config with classification on but a missing + threshold key surfaced as a bare ``configparser.NoOptionError`` from + inside ``config.getfloat``. It must instead raise with a message that + names the missing key. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat( + tile_sexcat_path, + np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), + ) + sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" + _write_sex_like_cat( + sexcat_sm_path, + np.array( + [(0.001, 0.0001) for _ in obj_ids], + dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], + ), + ) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": "True", + "SM_STAR_THRESH": "0.003", + # SM_GAL_THRESH deliberately omitted + "SHAPE_MEASUREMENT_TYPE": "ngmix", + } + } + ) + w_log = _RecordingLogger() + + with pytest.raises(ValueError, match="SM_GAL_THRESH"): + make_cat_runner( + [ + str(tile_sexcat_path), + str(sexcat_sm_path), + str(galaxy_psf_path), + str(ngmix_path), + ], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + w_log, + ) From e26b2f6efebc41d2caa2371baa615a793b1a9be1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 04:09:02 +0200 Subject: [PATCH 4/5] test(make_cat): fix fixture defects flagged in review - The classification-without-input test never wrote its ngmix catalogue, so on the pre-fix code it failed with catalogueFileNotFound instead of exercising the intended assertion. It now writes a matching ngmix catalogue and fails with "DID NOT RAISE ValueError" against the pre-fix runner, confirmed locally. - Single-field structured arrays (NUMBER only) get collapsed by FITSCatalogue._save_to_fits into one row with a vector-valued column whenever save_sextractor_data re-saves them, so every SExtractor-like fixture now carries a second scalar field (_numbered_data) and tests assert the NUMBER column comes back as scalar ids matching the input. - Present tense throughout: docstrings describe what each test protects, not the prior buggy behaviour; the section heading drops the branch-name reference. - New tests grouped under TestSpreadModelPathDefects so a concurrent append to this file's end (PR #854) merges as one class-sized block. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_014bvNTrAmZxcfb1ee83ApPK --- tests/module/test_make_cat.py | 391 ++++++++++++++++++---------------- 1 file changed, 206 insertions(+), 185 deletions(-) diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index db068ea52..fcbede7c5 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -455,7 +455,13 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): assert out[col][1] == -1, col -# --- make_cat_runner / save_sm_data: spread-model path defects (fix/makecat) --- + + +# --- make_cat_runner / save_sm_data: spread-model path --- +# +# New tests live in TestSpreadModelPathDefects below, appended as a single +# class so a concurrent append to this file's end merges as one block +# rather than interleaving with these functions line by line. class _RecordingLogger: @@ -483,45 +489,21 @@ def _write_sex_like_cat(path, data): ).writeto(str(path), overwrite=True) -def test_save_sm_data_returns_actual_saved_count(tmp_path): - """``save_sm_data`` must report how many rows it wrote, not its ``n_obj``. +def _numbered_data(obj_ids): + """A ``NUMBER`` + dummy second field structured array, one row per id. - The runner compares this count against the SExtractor catalogue's size - to catch a mismatch. If ``save_sm_data`` merely echoed back the ``n_obj`` - it was handed (as it did before this fix), the two sides of that - comparison are the same value by construction and the check can never - fire. + A structured array with a single field gets collapsed by + ``file_io.FITSCatalogue._save_to_fits`` into one row holding a + vector-valued column (``len(names) == 1`` triggers a + ``data = np.array([data])`` wrap), so every SExtractor-like fixture + that ``save_sextractor_data`` re-saves through ``save_as_fits`` needs a + second field to keep one row per object. """ - final_cat_path = tmp_path / "final_cat-0.fits" - final_cat_file = file_io.FITSCatalogue( - str(final_cat_path), - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - ) - final_cat_file.save_as_fits( - np.array([(1,), (2,), (3,)], dtype=[("NUMBER", "i8")]), - ext_name="RESULTS", + return np.array( + [(oid, 0.0) for oid in obj_ids], + dtype=[("NUMBER", "i8"), ("X_IMAGE", "f8")], ) - sm_path = tmp_path / "sexcat_sm.fits" - _write_sex_like_cat( - sm_path, - np.array( - [(0.001, 0.0001), (0.002, 0.0002)], - dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], - ), - ) - - n_saved = make_cat.save_sm_data( - final_cat_file, - str(sm_path), - do_classif=False, - n_obj=3, - ) - - # Two rows were actually written to the spread-model catalogue, not the - # three the (mismatched) SExtractor catalogue carried. - assert n_saved == 2 - def _write_matching_ngmix_cat(path, obj_ids): """Reuse the module's ngmix-catalogue writer for full-runner tests.""" @@ -531,10 +513,7 @@ def _write_matching_ngmix_cat(path, obj_ids): def _run_make_cat_runner(tmp_path, obj_ids, sm_obj_ids, do_classif=False): """Drive ``make_cat_runner`` end to end with synthetic 4-input data.""" tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat( - tile_sexcat_path, - np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), - ) + _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" _write_sex_like_cat( @@ -582,111 +561,106 @@ def _run_make_cat_runner(tmp_path, obj_ids, sm_obj_ids, do_classif=False): return result, w_log -def test_make_cat_runner_warns_on_size_mismatch_without_crashing(tmp_path): - """A SExtractor/spread-model size mismatch is logged, not fatal. - - The runner used to call ``w_log(...)`` directly -- but ``w_log`` is a - logger, not a callable -- so a genuine size mismatch crashed instead of - being reported. With three SExtractor objects and only two spread-model - rows, the run must complete and the mismatch must show up as a warning. - """ - obj_ids = [1, 2, 3] - result, w_log = _run_make_cat_runner( - tmp_path, obj_ids=obj_ids, sm_obj_ids=[1, 2] - ) - - assert result == (None, None) - assert any( - "3" in msg and "2" in msg and "different" in msg - for msg in w_log.warnings - ), w_log.warnings - +class TestSpreadModelPathDefects: + """Runner/``save_sm_data`` behaviour on the spread-model input path. -def test_make_cat_runner_no_sm_input_logs_honestly_and_writes_no_column( - tmp_path, -): - """With no spread-model input, the log must match reality: no column. - - The three-input path (no spread-model catalogue) used to log "setting - spread model to 99" while writing no ``SPREAD_MODEL`` column at all. - The log message must now describe what actually happens, and the - column must still be absent -- this is the committed, currently-run - configuration (``SM_DO_CLASSIFICATION = False``), so its output must - not change. + Each test guards one defect in that path: a broken size check, a + crashing log call, a log message that misdescribes what is written, + and configuration errors that must surface clearly. """ - obj_ids = [1, 2, 3] - tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat( - tile_sexcat_path, - np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), - ) - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" - ngmix_path = tmp_path / "ngmix-350-100.fits" - _write_matching_ngmix_cat(ngmix_path, obj_ids) - config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": "False", - "SHAPE_MEASUREMENT_TYPE": "ngmix", - } - } - ) - w_log = _RecordingLogger() - - result = make_cat_runner( - [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], - {"output": str(tmp_path)}, - "-350-100", - config, - "MAKE_CAT_RUNNER", - w_log, - ) + def test_save_sm_data_returns_actual_saved_count(self, tmp_path): + """``save_sm_data`` reports how many rows it wrote, not its ``n_obj``. + + The runner compares this count against the SExtractor catalogue's + size to catch a mismatch; if ``save_sm_data`` echoed back the + ``n_obj`` it was handed instead of its own row count, the two sides + of that comparison would always be equal and the check could never + fire. + """ + final_cat_path = tmp_path / "final_cat-0.fits" + final_cat_file = file_io.FITSCatalogue( + str(final_cat_path), + open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, + ) + final_cat_file.save_as_fits(_numbered_data([1, 2, 3]), ext_name="RESULTS") + + sm_path = tmp_path / "sexcat_sm.fits" + _write_sex_like_cat( + sm_path, + np.array( + [(0.001, 0.0001), (0.002, 0.0002)], + dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], + ), + ) - assert result == (None, None) - assert not any("99" in msg for msg in w_log.infos) + n_saved = make_cat.save_sm_data( + final_cat_file, + str(sm_path), + do_classif=False, + n_obj=3, + ) - final_cat = file_io.FITSCatalogue( - str(make_cat.get_output_name(str(tmp_path), "-350-100")) - ) - final_cat.open() - assert "SPREAD_MODEL" not in final_cat.get_data().dtype.names - final_cat.close() + # Two rows were actually written to the spread-model catalogue, not + # the three the (mismatched) SExtractor catalogue carried. + assert n_saved == 2 + def test_make_cat_runner_warns_on_size_mismatch_without_crashing( + self, tmp_path + ): + """A SExtractor/spread-model size mismatch is logged, not fatal. + + With three SExtractor objects and only two spread-model rows, the + run completes and the mismatch shows up as a warning, naming both + sizes. + """ + obj_ids = [1, 2, 3] + result, w_log = _run_make_cat_runner( + tmp_path, obj_ids=obj_ids, sm_obj_ids=[1, 2] + ) -def test_make_cat_runner_refuses_classification_without_sm_input(tmp_path): - """``SM_DO_CLASSIFICATION = True`` with no spread-model input is a config error. + assert result == (None, None) + assert any( + "3" in msg and "2" in msg and "different" in msg + for msg in w_log.warnings + ), w_log.warnings - Classifying stars/galaxies needs real spread-model values; with no - spread-model catalogue there is nothing to classify on, so this must - raise clearly rather than silently skip classification (the prior - behaviour) or crash deep inside the classifier. - """ - obj_ids = [1, 2, 3] - tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat( - tile_sexcat_path, - np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), - ) - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" - ngmix_path = tmp_path / "ngmix-350-100.fits" + final_cat = file_io.FITSCatalogue( + str(make_cat.get_output_name(str(tmp_path), "-350-100")) + ) + final_cat.open() + npt.assert_array_equal(final_cat.get_data()["NUMBER"], obj_ids) + final_cat.close() - config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": "True", - "SM_STAR_THRESH": "0.003", - "SM_GAL_THRESH": "0.01", - "SHAPE_MEASUREMENT_TYPE": "ngmix", + def test_make_cat_runner_no_sm_input_logs_honestly_and_writes_no_column( + self, tmp_path + ): + """With no spread-model input, the log matches reality: no column. + + This is the committed, currently-run configuration + (``SM_DO_CLASSIFICATION = False``, three inputs): the log message + must describe what actually happens, and no ``SPREAD_MODEL`` column + is written. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + _write_matching_ngmix_cat(ngmix_path, obj_ids) + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": "False", + "SHAPE_MEASUREMENT_TYPE": "ngmix", + } } - } - ) - w_log = _RecordingLogger() + ) + w_log = _RecordingLogger() - with pytest.raises(ValueError, match="SM_DO_CLASSIFICATION"): - make_cat_runner( + result = make_cat_runner( [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], {"output": str(tmp_path)}, "-350-100", @@ -695,58 +669,105 @@ def test_make_cat_runner_refuses_classification_without_sm_input(tmp_path): w_log, ) + assert result == (None, None) + assert not any("99" in msg for msg in w_log.infos) -def test_make_cat_runner_raises_clear_error_for_missing_threshold_config( - tmp_path, -): - """A missing ``SM_STAR_THRESH``/``SM_GAL_THRESH`` fails clearly, not deep in configparser. - - Before this fix, a config with classification on but a missing - threshold key surfaced as a bare ``configparser.NoOptionError`` from - inside ``config.getfloat``. It must instead raise with a message that - names the missing key. - """ - obj_ids = [1, 2, 3] - tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat( - tile_sexcat_path, - np.array([(oid,) for oid in obj_ids], dtype=[("NUMBER", "i8")]), - ) - sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" - _write_sex_like_cat( - sexcat_sm_path, - np.array( - [(0.001, 0.0001) for _ in obj_ids], - dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], - ), - ) - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" - ngmix_path = tmp_path / "ngmix-350-100.fits" - - config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": "True", - "SM_STAR_THRESH": "0.003", - # SM_GAL_THRESH deliberately omitted - "SHAPE_MEASUREMENT_TYPE": "ngmix", + final_cat = file_io.FITSCatalogue( + str(make_cat.get_output_name(str(tmp_path), "-350-100")) + ) + final_cat.open() + data = final_cat.get_data() + assert "SPREAD_MODEL" not in data.dtype.names + npt.assert_array_equal(data["NUMBER"], obj_ids) + final_cat.close() + + def test_make_cat_runner_refuses_classification_without_sm_input( + self, tmp_path + ): + """``SM_DO_CLASSIFICATION = True`` with no spread-model input is a config error. + + Classifying stars/galaxies needs real spread-model values; with no + spread-model catalogue there is nothing to classify on, so this + must raise clearly rather than silently skip classification or + crash deep inside the classifier. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + _write_matching_ngmix_cat(ngmix_path, obj_ids) + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": "True", + "SM_STAR_THRESH": "0.003", + "SM_GAL_THRESH": "0.01", + "SHAPE_MEASUREMENT_TYPE": "ngmix", + } } - } - ) - w_log = _RecordingLogger() + ) + w_log = _RecordingLogger() + + with pytest.raises(ValueError, match="SM_DO_CLASSIFICATION"): + make_cat_runner( + [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + w_log, + ) - with pytest.raises(ValueError, match="SM_GAL_THRESH"): - make_cat_runner( - [ - str(tile_sexcat_path), - str(sexcat_sm_path), - str(galaxy_psf_path), - str(ngmix_path), - ], - {"output": str(tmp_path)}, - "-350-100", - config, - "MAKE_CAT_RUNNER", - w_log, + def test_make_cat_runner_raises_clear_error_for_missing_threshold_config( + self, tmp_path + ): + """A missing ``SM_STAR_THRESH``/``SM_GAL_THRESH`` fails clearly. + + With classification on but a threshold key missing, the error + names the missing key rather than surfacing as a bare + ``configparser.NoOptionError`` from inside ``config.getfloat``. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) + sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" + _write_sex_like_cat( + sexcat_sm_path, + np.array( + [(0.001, 0.0001) for _ in obj_ids], + dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], + ), ) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + + config = CustomParser() + config.read_dict( + { + "MAKE_CAT_RUNNER": { + "SM_DO_CLASSIFICATION": "True", + "SM_STAR_THRESH": "0.003", + # SM_GAL_THRESH deliberately omitted + "SHAPE_MEASUREMENT_TYPE": "ngmix", + } + } + ) + w_log = _RecordingLogger() + + with pytest.raises(ValueError, match="SM_GAL_THRESH"): + make_cat_runner( + [ + str(tile_sexcat_path), + str(sexcat_sm_path), + str(galaxy_psf_path), + str(ngmix_path), + ], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + w_log, + ) From 6c2fc6f998f20b732246c929ccde4e3728e9b099 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 28 Sep 2026 17:39:27 +0200 Subject: [PATCH 5/5] Remove the spread model from ShapePipe The spread-model star/galaxy classifier did not work reliably when last run and separates stars from galaxies worse than the downstream (sp_validation, size-based) classification, so ShapePipe no longer computes or carries it. - Delete spread_model_runner and spread_model_package. - make_cat_runner takes exactly three inputs (tile sexcat, galaxy PSF, ngmix); save_sm_data and the SM_DO_CLASSIFICATION / SM_STAR_THRESH / SM_GAL_THRESH keys are gone, along with the tests of that path. - Drop SM_DO_CLASSIFICATION from the committed and image-sims make_cat configs and the SPREAD_* columns from final_cat.param and example/unions_800/cat_matched.param; tidy docs and refs.bib. - astra.yaml: star_galaxy_classification is now [HARDCODED]; the spread_model_inline option is excluded, with the reasons. - A make_cat_runner end-to-end test checks every detection is kept with no classification column. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 23 +- docs/source/pipeline_tutorial.md | 9 +- docs/source/refs.bib | 27 -- .../cfis_image_sims/config_tile_Mc_psfex.ini | 6 - example/unions_800/cat_matched.param | 7 - .../modules/make_cat_package/__init__.py | 21 +- .../modules/make_cat_package/make_cat.py | 72 ---- src/shapepipe/modules/make_cat_runner.py | 75 +---- .../modules/spread_model_package/__init__.py | 34 -- .../spread_model_package/spread_model.py | 312 ------------------ src/shapepipe/modules/spread_model_runner.py | 68 ---- tests/module/test_make_cat.py | 294 ++--------------- workflow/config/cfis/config_tile_Mc.ini | 6 +- workflow/config/cfis/final_cat.param | 7 - 14 files changed, 44 insertions(+), 917 deletions(-) delete mode 100644 src/shapepipe/modules/spread_model_package/__init__.py delete mode 100644 src/shapepipe/modules/spread_model_package/spread_model.py delete mode 100644 src/shapepipe/modules/spread_model_runner.py diff --git a/astra.yaml b/astra.yaml index 87c844909..face7aa5b 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1907,16 +1907,11 @@ analyses: star_galaxy_classification: label: Star/galaxy separation deferred out of the pipeline rationale: >- - Classification is off and no spread-model input is wired, so the - catalogue ships every object and separation happens downstream. The - dormant make_cat classifier computes class = sm + 2 sm_err, with - stars at |class| < SM_STAR_THRESH and galaxies at class > - SM_GAL_THRESH, both read from config (the function defaults, 0.003 - and 0.01, are bypassed); the committed config sets neither key, so - enabling classification alone raises. Unlike Guinot+22 - (guinot22_spread_model_cut), it applies no s > 0 or magnitude cut. - Values: - MAKE_CAT_RUNNER.SM_DO_CLASSIFICATION = False. + [HARDCODED] ShapePipe does no star/galaxy classification: make_cat + keeps every detection, and the catalogue carries no classification + column. Separation happens downstream, in sp_validation's size-based + selection. The spread-model classifier Guinot+22 used + (guinot22_spread_model_cut) is removed from the pipeline. default: deferred_downstream options: deferred_downstream: @@ -1924,10 +1919,10 @@ analyses: spread_model_inline: label: spread_model classification in make_cat insights: [guinot22_spread_model_cut] - description: >- - Not wired; needs spread_model_runner after the PSF - interpolation, its output added to make_cat's inputs, and both - thresholds set. + excluded: true + excluded_reason: >- + It did not work reliably when last run, and it separates stars + from galaxies worse than the downstream classification. tile_overlap_handling: label: Tile-overlap duplicates neither removed nor flagged rationale: >- diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index b7ae7985c..ff1059e2d 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -304,8 +304,7 @@ job_sp TILE_ID -j 16 is the selection of galaxies as extended objects compared to the PSF. First, the PSF model is interpolated to galaxy positions, according to the PSF model with `psfex_interp` or `mccd_interp`. Next, postage stamps around galaxies -of the weights maps are created via `vignetmaker`. Then, the spread model -is computed by the `spread_model` module. Finally, postage stamps +of the weights maps are created via `vignetmaker`. Finally, postage stamps around galaxies of single-exposure data is extracted with another call to `vignetmaker`. @@ -313,8 +312,8 @@ The output directory is - `run_sp_MiViSmVi` if the PSF model is `mccd`; - `run_sp_tile_PsViSmVi` for the `PSFEx` PSF model. -This corresponds to the MCCD/PSFex interpolation (`Mi`/`Pi`), `vignetmaker` (`Vi`), `spread_model` (`Sm`), and the -second call to `vignetmaker` (`Vi`). +This corresponds to the MCCD/PSFex interpolation (`Mi`/`Pi`), `vignetmaker` (`Vi`), +and the second call to `vignetmaker` (`Vi`). ## Shape measurement @@ -340,7 +339,7 @@ job_sp TILE_ID -j 64 This task first merges the `NJOB` parallel `ngmix` output files from the previous step into one output file. Then, previously obtained information are pasted into a _final_ shape catalogue via `make_cat`. Included are galaxy detection and basic measurement parameters, the PSF model at -galaxy positions, the spread-model classification, and the shape measurement. +galaxy positions, and the shape measurement. Two output directories are created. The first one is `run_sp_tile_Ms` for the `merge_sep` run. diff --git a/docs/source/refs.bib b/docs/source/refs.bib index c489ac915..6a9375420 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -193,21 +193,6 @@ @article{dalcin:11 keywords = "Python, MPI, PETSc", } -@article{desai:12, - title={THE BLANCO COSMOLOGY SURVEY: DATA ACQUISITION, PROCESSING, CALIBRATION, QUALITY DIAGNOSTICS, AND DATA RELEASE}, - volume={757}, - ISSN={1538-4357}, - url={http://dx.doi.org/10.1088/0004-637X/757/1/83}, - DOI={10.1088/0004-637x/757/1/83}, - number={1}, - journal={The Astrophysical Journal}, - publisher={IOP Publishing}, - author={Desai, S. and Armstrong, R. and Mohr, J. J. and Semler, D. R. and Liu, J. and Bertin, E. and Allam, S. S. and Barkhouse, W. A. and Bazin, G. and Buckley-Geer, E. J. and et al.}, - year={2012}, - month={Sep}, - pages={83} -} - @ARTICLE{ellison:19, author = {{Ellison}, Sara L. and {Viswanathan}, Akshara and {Patton}, David R. and {Bottrell}, Connor and {McConnachie}, Alan W. and {Gwyn}, Stephen and {Cuillandre}, Jean-Charles}, title = "{A definitive merger-AGN connection at z {\ensuremath{\sim}} 0 with CFIS: mergers have an excess of AGN and AGN hosts are more frequently disturbed}", @@ -574,18 +559,6 @@ @INPROCEEDINGS{marmo:08 adsnote = {Provided by the SAO/NASA Astrophysics Data System} } -@article{mohr:12, - title={The Dark Energy Survey data processing and calibration system}, - url={http://dx.doi.org/10.1117/12.926785}, - DOI={10.1117/12.926785}, - journal={Software and Cyberinfrastructure for Astronomy II}, - publisher={SPIE}, - author={Mohr, Joseph J. and Armstrong, Robert and Bertin, Emmanuel and Daues, Greg and Desai, Shantanu and Gower, Michelle and Gruendl, Robert and Hanlon, William and Kuropatkin, Nikolay and Lin, Huan and et al.}, - editor={Radziwill, Nicole M. and Chiozzi, GianlucaEditors}, - year={2012}, - month={Sep} -} - @InProceedings{pandas:10, author = { {W}es {M}c{K}inney }, title = { {D}ata {S}tructures for {S}tatistical {C}omputing in {P}ython }, diff --git a/example/cfis_image_sims/config_tile_Mc_psfex.ini b/example/cfis_image_sims/config_tile_Mc_psfex.ini index 3b186484e..728ae89e7 100644 --- a/example/cfis_image_sims/config_tile_Mc_psfex.ini +++ b/example/cfis_image_sims/config_tile_Mc_psfex.ini @@ -70,10 +70,4 @@ FILE_EXT = .fits, .sqlite, .fits # sections below NUMBERING_SCHEME = -000-000 -# Star/galaxy spread-model classification. The image-sims pipeline has no -# spread_model_runner stage (make_cat inputs are sextractor + fake_psf + -# ngmix only), so there is nothing to classify — disable it. The newer -# make_cat_runner reads this key unconditionally (NoOptionError if absent). -SM_DO_CLASSIFICATION = False - SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/unions_800/cat_matched.param b/example/unions_800/cat_matched.param index e5621d472..3be3f6b16 100644 --- a/example/unions_800/cat_matched.param +++ b/example/unions_800/cat_matched.param @@ -15,13 +15,6 @@ NGMIX_MCAL_FLAGS NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# spread class -SPREAD_CLASS - -# spread model flag and error -SPREAD_MODEL -SPREADERR_MODEL - # Number of epochs (exposures) N_EPOCH NGMIX_N_EPOCH diff --git a/src/shapepipe/modules/make_cat_package/__init__.py b/src/shapepipe/modules/make_cat_package/__init__.py index 838a91c4d..3525abdc8 100644 --- a/src/shapepipe/modules/make_cat_package/__init__.py +++ b/src/shapepipe/modules/make_cat_package/__init__.py @@ -7,7 +7,6 @@ :Parent modules: - ``sextractor_runner`` -- ``spread_model_runner`` - ``psfex_interp_runner`` or ``mccd_interp_runner`` - ``ngmix_runner`` @@ -21,8 +20,9 @@ This module creates a *final* catalogue combining the output of various previous module runs. This gathers all relevant information on the measured galaxies for weak-lensing post-processing. This includes galaxy detection and -basic measurement parameters, the PSF model at galaxy positions, the -spread-model classification, and the shape measurement. Each object is +basic measurement parameters, the PSF model at galaxy positions, and the +shape measurement. Every detected object is kept: the catalogue carries no +star/galaxy classification, which is done downstream. Each object is tagged with its source tile via the ``TILE_ID`` column; objects duplicated across overlapping tiles are not deduplicated here, and are left to downstream selection. @@ -30,21 +30,6 @@ Module-specific config file entries =================================== -SM_DO_CLASSIFICATION : bool, optional - Adds spread-model star/galaxy classification flag as the column - ``SPREAD_CLASS`` to the output if ``True`` -SM_STAR_THRESH : float, optional - Threshold :math:`s_{\rm star, thresh}` for star selection; object is - classified as a star if - :math:`|x s + 2 \sigma_s | < s_{\textrm{star, thresh}}` - where :math:`s` is the spread model and :math:`\sigma_s` is the spread - model error; default value is ``0.003`` -SM_GAL_THRESH : float, optional - Threshold :math:`s_{\rm gal, thresh}` for galaxy selection; object is - classified as a galaxy if - :math:`s + 2 \sigma_s > s_{\textrm{gal, thresh}}` where :math:`s` is the - spread model and :math:`\sigma_s` is the spread model error; default value - is ``0.01`` SHAPE_MEASUREMENT_TYPE : list Shape measurement method; the only valid option is ``ngmix`` (the knob is retained as the extension point for a future estimator family) diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index c1f5a301c..659f70b18 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -140,78 +140,6 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): return cat_size -def save_sm_data( - final_cat_file, - sexcat_sm_path, - do_classif=True, - star_thresh=0.003, - gal_thresh=0.01, - n_obj=-1, -): - r"""Save Spread-Model Data. - - Save the spread-model data into the final catalogue. - - Parameters - ---------- - final_cat_file : file_io.FITSCatalogue - Final catalogue - sexcat_sm_path : str - Path to spread-model catalogue to save. If ``None``, spread_model is - set to 99 - do_classif : bool - If ``True`` objects will be classified into stars, galaxies, and other, - using the classifier - :math:`{\rm class} = {\rm sm} + 2 * {\rm sm}_{\rm err}` - star_thresh : float - Threshold for star selection; object is classified as star if - :math:`|{\rm class}| <` ``star_thresh`` - gal_thresh : float - Threshold for galaxy selection; object is classified as galaxy if - :math:`{\rm class} >` ``gal_thresh`` - nobj : int, optional - Number of objects, only used if sexcat_sm_path is ``-1`` - - Returns - ------- - int - Number of objects saved - @sc [decision:catalogue_assembly.star_galaxy_classification] - """ - final_cat_file.open() - - if sexcat_sm_path is not None: - sexcat_sm_file = file_io.FITSCatalogue( - sexcat_sm_path, - SEx_catalogue=True, - ) - sexcat_sm_file.open() - - sm = np.copy(sexcat_sm_file.get_data()["SPREAD_MODEL"]) - sm_err = np.copy(sexcat_sm_file.get_data()["SPREADERR_MODEL"]) - - sexcat_sm_file.close() - - else: - sm = np.ones(n_obj) * 99 - sm_err = np.ones(n_obj) * 99 - - final_cat_file.add_col("SPREAD_MODEL", sm) - final_cat_file.add_col("SPREADERR_MODEL", sm_err) - - if do_classif: - obj_flag = np.ones_like(sm, dtype="int16") * 2 - classif = sm + 2.0 * sm_err - obj_flag[np.where(np.abs(classif) < star_thresh)] = 0 - obj_flag[np.where(classif > gal_thresh)] = 1 - - final_cat_file.add_col("SPREAD_CLASS", obj_flag) - - final_cat_file.close() - - return len(sm) - - def parse_mask_ext_paths(paths_str): """Parse Mask Ext Paths. diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index d7a53038d..d27b71ef8 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -16,17 +16,15 @@ version="1.1", input_module=[ "sextractor_runner", - "spread_model_runner", "psfex_interp_runner", "ngmix_runner", ], file_pattern=[ "tile_sexcat", - "sexcat_sm", "galaxy_psf", "ngmix", ], - file_ext=[".fits", ".fits", ".sqlite", ".fits"], + file_ext=[".fits", ".sqlite", ".fits"], depends=["numpy", "sqlitedict"], ) def make_cat_runner( @@ -43,47 +41,7 @@ def make_cat_runner( @sc [decision:masking.sky_mask_application] """ - # Set input file paths - if len(input_file_list) == 3: - # No spread model input - ( - tile_sexcat_path, - galaxy_psf_path, - shape1_cat_path, - ) = input_file_list - sexcat_sm_path = None - else: - # With spread model input - ( - tile_sexcat_path, - sexcat_sm_path, - galaxy_psf_path, - shape1_cat_path, - ) = input_file_list[0:4] - - # Fetch classification options - do_classif = config.getboolean( - module_config_sec, - "SM_DO_CLASSIFICATION", - ) - if do_classif: - if sexcat_sm_path is None: - raise ValueError( - "SM_DO_CLASSIFICATION is True but no spread-model " - "catalogue input was provided; star/galaxy classification " - "requires a spread-model input to classify on" - ) - for key in ("SM_STAR_THRESH", "SM_GAL_THRESH"): - if not config.has_option(module_config_sec, key): - raise ValueError( - f"SM_DO_CLASSIFICATION is True but {key} is not set " - "in the config" - ) - star_thresh = config.getfloat(module_config_sec, "SM_STAR_THRESH") - gal_thresh = config.getfloat(module_config_sec, "SM_GAL_THRESH") - else: - star_thresh = None - gal_thresh = None + tile_sexcat_path, galaxy_psf_path, shape1_cat_path = input_file_list # Fetch shape measurement type shape_type_list = config.getlist( @@ -112,32 +70,9 @@ def make_cat_runner( # Save SExtractor data w_log.info("Save SExtractor data") - n_obj = make_cat.save_sextractor_data(final_cat_file, tile_sexcat_path) - cat_size_sextractor = n_obj - - # Save spread-model data - if sexcat_sm_path is None: - w_log.info( - "No sm cat input, spread model will not be written to the " - "final catalogue" - ) - else: - w_log.info("Save spread-model data") - cat_size_sm = make_cat.save_sm_data( - final_cat_file, - sexcat_sm_path, - do_classif, - star_thresh, - gal_thresh, - n_obj=n_obj - ) - - if cat_size_sextractor != cat_size_sm: - w_log.warning( - f"SExtractor catalogue {tile_sexcat_path} has different" - f" size ({cat_size_sextractor}) than spread_model" - f" catalogue {sexcat_sm_path} ({cat_size_sm})" - ) + cat_size_sextractor = make_cat.save_sextractor_data( + final_cat_file, tile_sexcat_path + ) # Save shape data sc_inst = make_cat.SaveCatalogue(final_cat_file, cat_size_sextractor, w_log) diff --git a/src/shapepipe/modules/spread_model_package/__init__.py b/src/shapepipe/modules/spread_model_package/__init__.py deleted file mode 100644 index 247f60308..000000000 --- a/src/shapepipe/modules/spread_model_package/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -"""SPREAD MODEL PACKAGE. - -This package contains the module for ``spread_model``. - -:Author: Axel Guinot - -:Parent modules: - -- ``sextractor_runner`` -- ``psfex_interp_runner`` or ``mccd_interp_runner`` - and ``vignetmaker_runner`` - -:Input: SExtractor output catalogue, PSFEx output catalogue and vignet files - -:Output: Updated SExtractor or new FITS file - -Description -=========== - -This module refines the galaxy sample that will be used for shape measurement -using the spread model method of :cite:`desai:12` and :cite:`mohr:12`. - -Module-specific config file entries -=================================== - -PREFIX : str, optional - Ouput file prefix -OUTPUT_MODE : str - Output mode, options are ``add`` to add the outputs to the input - SExtractor catalogue or ``new`` to generte a new output catalogue - -""" - -__all__ = ["spread_model"] diff --git a/src/shapepipe/modules/spread_model_package/spread_model.py b/src/shapepipe/modules/spread_model_package/spread_model.py deleted file mode 100644 index 57d88dc43..000000000 --- a/src/shapepipe/modules/spread_model_package/spread_model.py +++ /dev/null @@ -1,312 +0,0 @@ -"""SPREAD MODEL. - -Class to compute the spread model, criterion to select galaxies - -:Author: Axel Guinot - -""" - -import galsim -import numpy as np -from cs_util import size as cs_size -from sqlitedict import SqliteDict - -from shapepipe.pipeline import file_io - - -def get_sm(obj_vign, psf_vign, model_vign, weight_vign): - """Get Spread Model. - - This method compute the spread moel for an object. - - Parameters - ---------- - obj_vign : numpy.ndarray - Vignet of the object - psf_vign : numpy.ndarray - Vignet of the gaussian model of the PSF - model_vign : numpy.ndarray - Vignet of the galaxy model - weight_vign : numpy.ndarray - Vignet of the weight at the object position - - Returns - ------- - tuple - Spread model and corresponding error values - - """ - # Mask invalid pixels - m = (obj_vign > -1e29) & (weight_vign > 0) - w = m.astype(float) - - # Set noise as inverse weight - noise_v = (1 / weight_vign).ravel() - - # Remove infinite noise pixels - noise_v[np.isinf(noise_v)] = 0 - - # Transform 2D vignets to 1D vectors - t_v = model_vign.ravel() - g_v = obj_vign.ravel() - psf_v = psf_vign.ravel() - w_v = w.ravel() - - # Compute scalar products used in spread model - tg = np.sum(t_v * w_v * g_v) - pg = np.sum(psf_v * w_v * g_v) - tp = np.sum(t_v * w_v * psf_v) - pp = np.sum(psf_v * w_v * psf_v) - - tnt = np.sum(t_v * noise_v * t_v * w_v) - pnp = np.sum(psf_v * noise_v * psf_v * w_v) - tnp = np.sum(t_v * noise_v * psf_v * w_v) - err = tnt * pg**2 + pnp * tg**2 - 2 * tnp * pg * tg - - # Compute spread model - if pg > 0: - sm = (tg / pg) - (tp / pp) - else: - sm = 1 - - if (pg > 0) & (err > 0): - sm_err = np.sqrt(err) / pg**2 - else: - sm_err = 1 - - return sm, sm_err - - -def get_model(sigma, flux, img_shape): - """Get Model. - - This method computes - - an exponential galaxy model with scale radius = 1/16 FWHM - - a Gaussian model for the PSF - - The spread-model statistic compares the object, PSF, and galaxy-model - vignets entirely in pixel space, so the models are rendered in pixel - units. A physical pixel scale would cancel out exactly (every length is - ``pixels x pixel_scale``, drawn at ``scale=pixel_scale``), so none is - needed here. - - Parameters - ---------- - sigma : float - Sigma of the PSF (in pixel units) - flux : float - Flux of the galaxy for the model - img_shape : list - Size of the output vignet ``[xsize, ysize]`` - - Returns - ------- - tuple - Vignet of the galaxy model and of the PSF model - - """ - # Get scale radius (pixels) - scale_radius = 1 / 16 * cs_size.sigma_to_fwhm(sigma) - - # Get galaxy model - gal_obj = galsim.Exponential(scale_radius=scale_radius, flux=flux) - - # Get PSF (sigma in pixels) - psf_obj = galsim.Gaussian(sigma=sigma) - - # Convolve both - gal_obj = galsim.Convolve(gal_obj, psf_obj) - - # Draw galaxy and PSF on vignets, in pixel units (scale = 1 pixel) - gal_vign = gal_obj.drawImage( - nx=img_shape[0], ny=img_shape[1], scale=1.0 - ).array - - psf_vign = psf_obj.drawImage( - nx=img_shape[0], ny=img_shape[1], scale=1.0 - ).array - - return gal_vign, psf_vign - - -class SpreadModel(object): - """The Spread Model Class. - - Parameters - ---------- - sex_cat_path : str - Path to SExtractor catalogue - psf_cat_path : str - Path to PSF catalogue - weight_cat_path : str - Path to weight catalogue - output_path : str - Output file path of pasted catalog - output_mode : str - Options are ``new`` or ``add`` - - Notes - ----- - For the ``output_mode``: - - - ``new`` will create a new catalogue with - ``[number, mag, sm, sm_err]`` - - ``add`` will output a copy of the input SExtractor with the columns - ``sm`` and ``sm_err`` - - """ - - def __init__( - self, - sex_cat_path, - psf_cat_path, - weight_cat_path, - output_path, - output_mode, - ): - - self._sex_cat_path = sex_cat_path - self._psf_cat_path = psf_cat_path - self._weight_cat_path = weight_cat_path - self._output_path = output_path - self._output_mode = output_mode - - def process(self): - """Process. - - Process the spread model computation - - """ - # Get data - sex_cat = file_io.FITSCatalogue(self._sex_cat_path, SEx_catalogue=True) - sex_cat.open() - obj_id = np.copy(sex_cat.get_data()["NUMBER"]) - obj_vign = np.copy(sex_cat.get_data()["VIGNET"]) - obj_mag = None - if self._output_mode == "new": - obj_mag = np.copy(sex_cat.get_data()["MAG_AUTO"]) - sex_cat.close() - - psf_cat = SqliteDict(self._psf_cat_path) - - weight_cat = file_io.FITSCatalogue( - self._weight_cat_path, - SEx_catalogue=True, - ) - weight_cat.open() - weigh_vign = weight_cat.get_data()["VIGNET"] - weight_cat.close() - - # Get spread model - skip_obj = False - spread_model_final = [] - spread_model_err_final = [] - for idx, id_tmp in enumerate(obj_id): - sigma_list = [] - - if psf_cat[str(id_tmp)] == "empty": - spread_model_final.append(-1) - spread_model_err_final.append(1) - continue - - psf_expccd_name = list(psf_cat[str(id_tmp)].keys()) - - for expccd_name_tmp in psf_expccd_name: - psf_cat_id_ccd = psf_cat[str(id_tmp)][expccd_name_tmp] - # The HSM grammar stores PSF size as T = 2 sigma^2 under - # HSM_T_PSF; spread model needs the Gaussian sigma (pixels). - sigma_list.append( - cs_size.T_to_sigma(psf_cat_id_ccd["SHAPES"]["HSM_T_PSF"]) - ) - - obj_sigma_tmp = np.mean(sigma_list) - if obj_sigma_tmp > 0: - obj_vign_tmp = obj_vign[idx] - obj_flux_tmp = 1.0 - obj_weight_tmp = weigh_vign[idx] - obj_model_tmp, obj_psf_tmp = get_model( - obj_sigma_tmp, - obj_flux_tmp, - obj_vign_tmp.shape, - ) - - obj_sm, obj_sm_err = get_sm( - obj_vign_tmp, obj_psf_tmp, obj_model_tmp, obj_weight_tmp - ) - else: - # size < 0, something is not right with this object - obj_sm, obj_sm_err = -1.0, -1.0 - - spread_model_final.append(obj_sm) - spread_model_err_final.append(obj_sm_err) - - spread_model_final = np.array(spread_model_final, dtype="float64") - spread_model_err_final = np.array( - spread_model_err_final, - dtype="float64", - ) - - psf_cat.close() - - self.save_results( - spread_model_final, spread_model_err_final, obj_mag, obj_id - ) - - def save_results(self, sm, sm_err, mag, number): - """Save Results. - - Save output catalogue with spread model and errors. - - Parameters - ---------- - sm : numpy.ndarray - Value of the spread model for all objects - sm_err : numpy.ndarray - Value of the spread model error for all objects - mag : numpy.ndarray - Magnitude of all objects (only for a new catalogue) - number : numpy.ndarray - ID of all objects (only for a new catalogue) - - Raises - ------ - ValueError - For incorrect output mode - - """ - if self._output_mode == "new": - new_cat = file_io.FITSCatalogue( - self._output_path, - SEx_catalogue=True, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - ) - dict_data = { - "NUMBER": number, - "MAG": mag, - "SPREAD_MODEL": sm, - "SPREADERR_MODEL": sm_err, - } - new_cat.save_as_fits( - data=dict_data, sex_cat_path=self._sex_cat_path - ) - elif self._output_mode == "add": - ori_cat = file_io.FITSCatalogue( - self._sex_cat_path, - SEx_catalogue=True, - ) - ori_cat.open() - new_cat = file_io.FITSCatalogue( - self._output_path, - SEx_catalogue=True, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - ) - ori_cat.add_col( - "SPREAD_MODEL", sm, new_cat=True, new_cat_inst=new_cat - ) - ori_cat.add_col("SPREAD_MODEL", sm, new_cat=True, new_cat_inst=new_cat) - ori_cat.close() - new_cat.open() - new_cat.add_col("SPREADERR_MODEL", sm_err) - new_cat.close() - else: - raise ValueError("Mode must be in [new, add].") diff --git a/src/shapepipe/modules/spread_model_runner.py b/src/shapepipe/modules/spread_model_runner.py deleted file mode 100644 index 93b70b753..000000000 --- a/src/shapepipe/modules/spread_model_runner.py +++ /dev/null @@ -1,68 +0,0 @@ -"""SPREAD MODEL RUNNER. - -Module runner for ``spread_model``. - -:Author: Axel Guinot - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.spread_model_package.spread_model import SpreadModel - - -@module_runner( - version="1.1", - input_module=[ - "sextractor_runner", - "psfex_interp_runner", - "vignetmaker_runner", - ], - file_pattern=["sexcat", "galaxy_psf", "weight_vign"], - file_ext=[".fits", ".sqlite", ".fits"], - depends=["numpy", "galsim"], - run_method="parallel", -) -def spread_model_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Spread Model Runner.""" - # Get input files - sex_cat_path, psf_cat_path, weight_cat_path = input_file_list - - # Get file prefix (optional) - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - if (prefix.lower() != "none") & (prefix != ""): - prefix = prefix + "_" - else: - prefix = "" - else: - prefix = "" - - # Get output mode. The spread model is computed entirely in pixel space, - # so no pixel scale is needed (see spread_model.get_model). - output_mode = config.get(module_config_sec, "OUTPUT_MODE") - - # Set output file path - file_name = f"{prefix}sexcat_sm{file_number_string}.fits" - output_path = f'{run_dirs["output"]}/{file_name}' - - # Create spread model class instance - sm_inst = SpreadModel( - sex_cat_path, - psf_cat_path, - weight_cat_path, - output_path, - output_mode, - ) - - # Process spread model computation - sm_inst.process() - - # No return objects - return None, None diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index 4f7750baa..2b939ef7b 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -460,27 +460,7 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): assert out[col][1] == -1, col - - -# --- make_cat_runner / save_sm_data: spread-model path --- -# -# New tests live in TestSpreadModelPathDefects below, appended as a single -# class so a concurrent append to this file's end merges as one block -# rather than interleaving with these functions line by line. - - -class _RecordingLogger: - """Minimal stand-in for the pipeline's ``w_log``, recording calls.""" - - def __init__(self): - self.infos = [] - self.warnings = [] - - def info(self, msg, *_args, **_kwargs): - self.infos.append(msg) - - def warning(self, msg, *_args, **_kwargs): - self.warnings.append(msg) +# --- make_cat_runner: end-to-end catalogue assembly --- def _write_sex_like_cat(path, data): @@ -510,272 +490,42 @@ def _numbered_data(obj_ids): ) -def _write_matching_ngmix_cat(path, obj_ids): - """Reuse the module's ngmix-catalogue writer for full-runner tests.""" - _write_ngmix_cat(path, obj_ids) - +def test_make_cat_runner_ships_every_detection_unclassified(tmp_path): + """The runner assembles every detection, with no star/galaxy column. -def _run_make_cat_runner(tmp_path, obj_ids, sm_obj_ids, do_classif=False): - """Drive ``make_cat_runner`` end to end with synthetic 4-input data.""" + Star/galaxy separation happens downstream, so the final catalogue keeps + each SExtractor object and carries no classification or spread-model + column. + """ + obj_ids = [1, 2, 3] tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) - - sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" - _write_sex_like_cat( - sexcat_sm_path, - np.array( - [(0.001, 0.0001) for _ in sm_obj_ids], - dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], - ), - ) - + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" ngmix_path = tmp_path / "ngmix-350-100.fits" - _write_matching_ngmix_cat(ngmix_path, obj_ids) - - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" # unused: no SAVE_PSF_DATA + _write_ngmix_cat(ngmix_path, obj_ids) config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": str(do_classif), - "SHAPE_MEASUREMENT_TYPE": "ngmix", - **( - {"SM_STAR_THRESH": "0.003", "SM_GAL_THRESH": "0.01"} - if do_classif - else {} - ), - } - } - ) + config.read_dict({"MAKE_CAT_RUNNER": {"SHAPE_MEASUREMENT_TYPE": "ngmix"}}) - w_log = _RecordingLogger() result = make_cat_runner( - [ - str(tile_sexcat_path), - str(sexcat_sm_path), - str(galaxy_psf_path), - str(ngmix_path), - ], + [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], {"output": str(tmp_path)}, "-350-100", config, "MAKE_CAT_RUNNER", - w_log, + _NullLogger(), ) - return result, w_log - - -class TestSpreadModelPathDefects: - """Runner/``save_sm_data`` behaviour on the spread-model input path. - - Each test guards one defect in that path: a broken size check, a - crashing log call, a log message that misdescribes what is written, - and configuration errors that must surface clearly. - """ - - def test_save_sm_data_returns_actual_saved_count(self, tmp_path): - """``save_sm_data`` reports how many rows it wrote, not its ``n_obj``. - - The runner compares this count against the SExtractor catalogue's - size to catch a mismatch; if ``save_sm_data`` echoed back the - ``n_obj`` it was handed instead of its own row count, the two sides - of that comparison would always be equal and the check could never - fire. - """ - final_cat_path = tmp_path / "final_cat-0.fits" - final_cat_file = file_io.FITSCatalogue( - str(final_cat_path), - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - ) - final_cat_file.save_as_fits(_numbered_data([1, 2, 3]), ext_name="RESULTS") - - sm_path = tmp_path / "sexcat_sm.fits" - _write_sex_like_cat( - sm_path, - np.array( - [(0.001, 0.0001), (0.002, 0.0002)], - dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], - ), - ) - n_saved = make_cat.save_sm_data( - final_cat_file, - str(sm_path), - do_classif=False, - n_obj=3, - ) - - # Two rows were actually written to the spread-model catalogue, not - # the three the (mismatched) SExtractor catalogue carried. - assert n_saved == 2 - - def test_make_cat_runner_warns_on_size_mismatch_without_crashing( - self, tmp_path - ): - """A SExtractor/spread-model size mismatch is logged, not fatal. - - With three SExtractor objects and only two spread-model rows, the - run completes and the mismatch shows up as a warning, naming both - sizes. - """ - obj_ids = [1, 2, 3] - result, w_log = _run_make_cat_runner( - tmp_path, obj_ids=obj_ids, sm_obj_ids=[1, 2] - ) - - assert result == (None, None) - assert any( - "3" in msg and "2" in msg and "different" in msg - for msg in w_log.warnings - ), w_log.warnings - - final_cat = file_io.FITSCatalogue( - str(make_cat.get_output_name(str(tmp_path), "-350-100")) - ) - final_cat.open() - npt.assert_array_equal(final_cat.get_data()["NUMBER"], obj_ids) - final_cat.close() - - def test_make_cat_runner_no_sm_input_logs_honestly_and_writes_no_column( - self, tmp_path - ): - """With no spread-model input, the log matches reality: no column. - - This is the committed, currently-run configuration - (``SM_DO_CLASSIFICATION = False``, three inputs): the log message - must describe what actually happens, and no ``SPREAD_MODEL`` column - is written. - """ - obj_ids = [1, 2, 3] - tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" - ngmix_path = tmp_path / "ngmix-350-100.fits" - _write_matching_ngmix_cat(ngmix_path, obj_ids) - - config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": "False", - "SHAPE_MEASUREMENT_TYPE": "ngmix", - } - } - ) - w_log = _RecordingLogger() - - result = make_cat_runner( - [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], - {"output": str(tmp_path)}, - "-350-100", - config, - "MAKE_CAT_RUNNER", - w_log, - ) - - assert result == (None, None) - assert not any("99" in msg for msg in w_log.infos) - - final_cat = file_io.FITSCatalogue( - str(make_cat.get_output_name(str(tmp_path), "-350-100")) - ) - final_cat.open() - data = final_cat.get_data() - assert "SPREAD_MODEL" not in data.dtype.names - npt.assert_array_equal(data["NUMBER"], obj_ids) - final_cat.close() - - def test_make_cat_runner_refuses_classification_without_sm_input( - self, tmp_path - ): - """``SM_DO_CLASSIFICATION = True`` with no spread-model input is a config error. - - Classifying stars/galaxies needs real spread-model values; with no - spread-model catalogue there is nothing to classify on, so this - must raise clearly rather than silently skip classification or - crash deep inside the classifier. - """ - obj_ids = [1, 2, 3] - tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" - ngmix_path = tmp_path / "ngmix-350-100.fits" - _write_matching_ngmix_cat(ngmix_path, obj_ids) - - config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": "True", - "SM_STAR_THRESH": "0.003", - "SM_GAL_THRESH": "0.01", - "SHAPE_MEASUREMENT_TYPE": "ngmix", - } - } - ) - w_log = _RecordingLogger() - - with pytest.raises(ValueError, match="SM_DO_CLASSIFICATION"): - make_cat_runner( - [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], - {"output": str(tmp_path)}, - "-350-100", - config, - "MAKE_CAT_RUNNER", - w_log, - ) + assert result == (None, None) + final_cat = file_io.FITSCatalogue( + str(make_cat.get_output_name(str(tmp_path), "-350-100")) + ) + final_cat.open() + data = final_cat.get_data() + final_cat.close() + npt.assert_array_equal(data["NUMBER"], obj_ids) + assert not [name for name in data.dtype.names if "SPREAD" in name] - def test_make_cat_runner_raises_clear_error_for_missing_threshold_config( - self, tmp_path - ): - """A missing ``SM_STAR_THRESH``/``SM_GAL_THRESH`` fails clearly. - - With classification on but a threshold key missing, the error - names the missing key rather than surfacing as a bare - ``configparser.NoOptionError`` from inside ``config.getfloat``. - """ - obj_ids = [1, 2, 3] - tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" - _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) - sexcat_sm_path = tmp_path / "sexcat_sm-350-100.fits" - _write_sex_like_cat( - sexcat_sm_path, - np.array( - [(0.001, 0.0001) for _ in obj_ids], - dtype=[("SPREAD_MODEL", "f8"), ("SPREADERR_MODEL", "f8")], - ), - ) - galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" - ngmix_path = tmp_path / "ngmix-350-100.fits" - - config = CustomParser() - config.read_dict( - { - "MAKE_CAT_RUNNER": { - "SM_DO_CLASSIFICATION": "True", - "SM_STAR_THRESH": "0.003", - # SM_GAL_THRESH deliberately omitted - "SHAPE_MEASUREMENT_TYPE": "ngmix", - } - } - ) - w_log = _RecordingLogger() - - with pytest.raises(ValueError, match="SM_GAL_THRESH"): - make_cat_runner( - [ - str(tile_sexcat_path), - str(sexcat_sm_path), - str(galaxy_psf_path), - str(ngmix_path), - ], - {"output": str(tmp_path)}, - "-350-100", - config, - "MAKE_CAT_RUNNER", - w_log, - ) # --- _save_psf_data: fixed per-epoch slot count (N_EPOCH_SLOTS) --- diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 9475a819c..636803f48 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -1,5 +1,4 @@ -# ShapePipe post-run configuration file: create final catalogs, with -# no spread model on input +# ShapePipe post-run configuration file: create final catalogs ## Default ShapePipe options @@ -79,9 +78,6 @@ FILE_EXT = .fits, .sqlite, .fits # sections below NUMBERING_SCHEME = -000-000 -# @sc [decision:catalogue_assembly.star_galaxy_classification] -SM_DO_CLASSIFICATION = False - SHAPE_MEASUREMENT_TYPE = ngmix # Save per-epoch PSF shapes and epoch identity (HSM_*_PSF_n, EXP_ID_n, diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 6ebd1aae9..becb72501 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -19,13 +19,6 @@ NGMIX_MCAL_FLAGS NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# spread class -#SPREAD_CLASS - -# spread model flag and error -#SPREAD_MODEL -#SPREADERR_MODEL - # Number of epochs (exposures) N_EPOCH NGMIX_N_EPOCH