diff --git a/astra.yaml b/astra.yaml index b823f803b..45386dd1a 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1907,16 +1907,11 @@ analyses: star_galaxy_classification: label: Star/galaxy separation deferred out of the pipeline rationale: >- - Classification is off and no spread-model input is wired, so the - catalogue ships every object and separation happens downstream. The - dormant make_cat classifier computes class = sm + 2 sm_err, with - stars at |class| < SM_STAR_THRESH and galaxies at class > - SM_GAL_THRESH, both read from config (the function defaults, 0.003 - and 0.01, are bypassed); the committed config sets neither key, so - enabling classification alone raises. Unlike Guinot+22 - (guinot22_spread_model_cut), it applies no s > 0 or magnitude cut. - Values: - MAKE_CAT_RUNNER.SM_DO_CLASSIFICATION = False. + [HARDCODED] ShapePipe does no star/galaxy classification: make_cat + keeps every detection, and the catalogue carries no classification + column. Separation happens downstream, in sp_validation's size-based + selection. The spread-model classifier Guinot+22 used + (guinot22_spread_model_cut) is removed from the pipeline. default: deferred_downstream options: deferred_downstream: @@ -1924,10 +1919,10 @@ analyses: spread_model_inline: label: spread_model classification in make_cat insights: [guinot22_spread_model_cut] - description: >- - Not wired; needs spread_model_runner after the PSF - interpolation, its output added to make_cat's inputs, and both - thresholds set. + excluded: true + excluded_reason: >- + It did not work reliably when last run, and it separates stars + from galaxies worse than the downstream classification. tile_overlap_handling: label: Tile-overlap duplicates neither removed nor flagged rationale: >- diff --git a/docs/source/pipeline_tutorial.md b/docs/source/pipeline_tutorial.md index b7ae7985c..ff1059e2d 100644 --- a/docs/source/pipeline_tutorial.md +++ b/docs/source/pipeline_tutorial.md @@ -304,8 +304,7 @@ job_sp TILE_ID -j 16 is the selection of galaxies as extended objects compared to the PSF. First, the PSF model is interpolated to galaxy positions, according to the PSF model with `psfex_interp` or `mccd_interp`. Next, postage stamps around galaxies -of the weights maps are created via `vignetmaker`. Then, the spread model -is computed by the `spread_model` module. Finally, postage stamps +of the weights maps are created via `vignetmaker`. Finally, postage stamps around galaxies of single-exposure data is extracted with another call to `vignetmaker`. @@ -313,8 +312,8 @@ The output directory is - `run_sp_MiViSmVi` if the PSF model is `mccd`; - `run_sp_tile_PsViSmVi` for the `PSFEx` PSF model. -This corresponds to the MCCD/PSFex interpolation (`Mi`/`Pi`), `vignetmaker` (`Vi`), `spread_model` (`Sm`), and the -second call to `vignetmaker` (`Vi`). +This corresponds to the MCCD/PSFex interpolation (`Mi`/`Pi`), `vignetmaker` (`Vi`), +and the second call to `vignetmaker` (`Vi`). ## Shape measurement @@ -340,7 +339,7 @@ job_sp TILE_ID -j 64 This task first merges the `NJOB` parallel `ngmix` output files from the previous step into one output file. Then, previously obtained information are pasted into a _final_ shape catalogue via `make_cat`. Included are galaxy detection and basic measurement parameters, the PSF model at -galaxy positions, the spread-model classification, and the shape measurement. +galaxy positions, and the shape measurement. Two output directories are created. The first one is `run_sp_tile_Ms` for the `merge_sep` run. diff --git a/docs/source/refs.bib b/docs/source/refs.bib index c489ac915..6a9375420 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -193,21 +193,6 @@ @article{dalcin:11 keywords = "Python, MPI, PETSc", } -@article{desai:12, - title={THE BLANCO COSMOLOGY SURVEY: DATA ACQUISITION, PROCESSING, CALIBRATION, QUALITY DIAGNOSTICS, AND DATA RELEASE}, - volume={757}, - ISSN={1538-4357}, - url={http://dx.doi.org/10.1088/0004-637X/757/1/83}, - DOI={10.1088/0004-637x/757/1/83}, - number={1}, - journal={The Astrophysical Journal}, - publisher={IOP Publishing}, - author={Desai, S. and Armstrong, R. and Mohr, J. J. and Semler, D. R. and Liu, J. and Bertin, E. and Allam, S. S. and Barkhouse, W. A. and Bazin, G. and Buckley-Geer, E. J. and et al.}, - year={2012}, - month={Sep}, - pages={83} -} - @ARTICLE{ellison:19, author = {{Ellison}, Sara L. and {Viswanathan}, Akshara and {Patton}, David R. and {Bottrell}, Connor and {McConnachie}, Alan W. and {Gwyn}, Stephen and {Cuillandre}, Jean-Charles}, title = "{A definitive merger-AGN connection at z {\ensuremath{\sim}} 0 with CFIS: mergers have an excess of AGN and AGN hosts are more frequently disturbed}", @@ -574,18 +559,6 @@ @INPROCEEDINGS{marmo:08 adsnote = {Provided by the SAO/NASA Astrophysics Data System} } -@article{mohr:12, - title={The Dark Energy Survey data processing and calibration system}, - url={http://dx.doi.org/10.1117/12.926785}, - DOI={10.1117/12.926785}, - journal={Software and Cyberinfrastructure for Astronomy II}, - publisher={SPIE}, - author={Mohr, Joseph J. and Armstrong, Robert and Bertin, Emmanuel and Daues, Greg and Desai, Shantanu and Gower, Michelle and Gruendl, Robert and Hanlon, William and Kuropatkin, Nikolay and Lin, Huan and et al.}, - editor={Radziwill, Nicole M. and Chiozzi, GianlucaEditors}, - year={2012}, - month={Sep} -} - @InProceedings{pandas:10, author = { {W}es {M}c{K}inney }, title = { {D}ata {S}tructures for {S}tatistical {C}omputing in {P}ython }, diff --git a/example/cfis_image_sims/config_tile_Mc_psfex.ini b/example/cfis_image_sims/config_tile_Mc_psfex.ini index 3b186484e..728ae89e7 100644 --- a/example/cfis_image_sims/config_tile_Mc_psfex.ini +++ b/example/cfis_image_sims/config_tile_Mc_psfex.ini @@ -70,10 +70,4 @@ FILE_EXT = .fits, .sqlite, .fits # sections below NUMBERING_SCHEME = -000-000 -# Star/galaxy spread-model classification. The image-sims pipeline has no -# spread_model_runner stage (make_cat inputs are sextractor + fake_psf + -# ngmix only), so there is nothing to classify — disable it. The newer -# make_cat_runner reads this key unconditionally (NoOptionError if absent). -SM_DO_CLASSIFICATION = False - SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/example/unions_800/cat_matched.param b/example/unions_800/cat_matched.param index e5621d472..3be3f6b16 100644 --- a/example/unions_800/cat_matched.param +++ b/example/unions_800/cat_matched.param @@ -15,13 +15,6 @@ NGMIX_MCAL_FLAGS NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# spread class -SPREAD_CLASS - -# spread model flag and error -SPREAD_MODEL -SPREADERR_MODEL - # Number of epochs (exposures) N_EPOCH NGMIX_N_EPOCH diff --git a/src/shapepipe/modules/make_cat_package/__init__.py b/src/shapepipe/modules/make_cat_package/__init__.py index 838a91c4d..3525abdc8 100644 --- a/src/shapepipe/modules/make_cat_package/__init__.py +++ b/src/shapepipe/modules/make_cat_package/__init__.py @@ -7,7 +7,6 @@ :Parent modules: - ``sextractor_runner`` -- ``spread_model_runner`` - ``psfex_interp_runner`` or ``mccd_interp_runner`` - ``ngmix_runner`` @@ -21,8 +20,9 @@ This module creates a *final* catalogue combining the output of various previous module runs. This gathers all relevant information on the measured galaxies for weak-lensing post-processing. This includes galaxy detection and -basic measurement parameters, the PSF model at galaxy positions, the -spread-model classification, and the shape measurement. Each object is +basic measurement parameters, the PSF model at galaxy positions, and the +shape measurement. Every detected object is kept: the catalogue carries no +star/galaxy classification, which is done downstream. Each object is tagged with its source tile via the ``TILE_ID`` column; objects duplicated across overlapping tiles are not deduplicated here, and are left to downstream selection. @@ -30,21 +30,6 @@ Module-specific config file entries =================================== -SM_DO_CLASSIFICATION : bool, optional - Adds spread-model star/galaxy classification flag as the column - ``SPREAD_CLASS`` to the output if ``True`` -SM_STAR_THRESH : float, optional - Threshold :math:`s_{\rm star, thresh}` for star selection; object is - classified as a star if - :math:`|x s + 2 \sigma_s | < s_{\textrm{star, thresh}}` - where :math:`s` is the spread model and :math:`\sigma_s` is the spread - model error; default value is ``0.003`` -SM_GAL_THRESH : float, optional - Threshold :math:`s_{\rm gal, thresh}` for galaxy selection; object is - classified as a galaxy if - :math:`s + 2 \sigma_s > s_{\textrm{gal, thresh}}` where :math:`s` is the - spread model and :math:`\sigma_s` is the spread model error; default value - is ``0.01`` SHAPE_MEASUREMENT_TYPE : list Shape measurement method; the only valid option is ``ngmix`` (the knob is retained as the extension point for a future estimator family) diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 18de1474e..5c8cca748 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -145,78 +145,6 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): return cat_size -def save_sm_data( - final_cat_file, - sexcat_sm_path, - do_classif=True, - star_thresh=0.003, - gal_thresh=0.01, - n_obj=-1, -): - r"""Save Spread-Model Data. - - Save the spread-model data into the final catalogue. - - Parameters - ---------- - final_cat_file : file_io.FITSCatalogue - Final catalogue - sexcat_sm_path : str - Path to spread-model catalogue to save. If ``None``, spread_model is - set to 99 - do_classif : bool - If ``True`` objects will be classified into stars, galaxies, and other, - using the classifier - :math:`{\rm class} = {\rm sm} + 2 * {\rm sm}_{\rm err}` - star_thresh : float - Threshold for star selection; object is classified as star if - :math:`|{\rm class}| <` ``star_thresh`` - gal_thresh : float - Threshold for galaxy selection; object is classified as galaxy if - :math:`{\rm class} >` ``gal_thresh`` - nobj : int, optional - Number of objects, only used if sexcat_sm_path is ``-1`` - - Returns - ------- - int - Number of objects saved - @sc [decision:catalogue_assembly.star_galaxy_classification] - """ - final_cat_file.open() - - if sexcat_sm_path is not None: - sexcat_sm_file = file_io.FITSCatalogue( - sexcat_sm_path, - SEx_catalogue=True, - ) - sexcat_sm_file.open() - - sm = np.copy(sexcat_sm_file.get_data()["SPREAD_MODEL"]) - sm_err = np.copy(sexcat_sm_file.get_data()["SPREADERR_MODEL"]) - - sexcat_sm_file.close() - - else: - sm = np.ones(n_obj) * 99 - sm_err = np.ones(n_obj) * 99 - - final_cat_file.add_col("SPREAD_MODEL", sm) - final_cat_file.add_col("SPREADERR_MODEL", sm_err) - - if do_classif: - obj_flag = np.ones_like(sm, dtype="int16") * 2 - classif = sm + 2.0 * sm_err - obj_flag[np.where(np.abs(classif) < star_thresh)] = 0 - obj_flag[np.where(classif > gal_thresh)] = 1 - - final_cat_file.add_col("SPREAD_CLASS", obj_flag) - - final_cat_file.close() - - return n_obj - - def parse_mask_ext_paths(paths_str): """Parse Mask Ext Paths. diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index a7da5fd25..d27b71ef8 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -16,17 +16,15 @@ version="1.1", input_module=[ "sextractor_runner", - "spread_model_runner", "psfex_interp_runner", "ngmix_runner", ], file_pattern=[ "tile_sexcat", - "sexcat_sm", "galaxy_psf", "ngmix", ], - file_ext=[".fits", ".fits", ".sqlite", ".fits"], + file_ext=[".fits", ".sqlite", ".fits"], depends=["numpy", "sqlitedict"], ) def make_cat_runner( @@ -43,35 +41,7 @@ def make_cat_runner( @sc [decision:masking.sky_mask_application] """ - # Set input file paths - if len(input_file_list) == 3: - # No spread model input - ( - tile_sexcat_path, - galaxy_psf_path, - shape1_cat_path, - ) = input_file_list - sexcat_sm_path = None - else: - # With spread model input - ( - tile_sexcat_path, - sexcat_sm_path, - galaxy_psf_path, - shape1_cat_path, - ) = input_file_list[0:4] - - # Fetch classification options - do_classif = config.getboolean( - module_config_sec, - "SM_DO_CLASSIFICATION", - ) - if do_classif: - star_thresh = config.getfloat(module_config_sec, "SM_STAR_THRESH") - gal_thresh = config.getfloat(module_config_sec, "SM_GAL_THRESH") - else: - star_thresh = None - gal_thresh = None + tile_sexcat_path, galaxy_psf_path, shape1_cat_path = input_file_list # Fetch shape measurement type shape_type_list = config.getlist( @@ -100,29 +70,9 @@ def make_cat_runner( # Save SExtractor data w_log.info("Save SExtractor data") - n_obj = make_cat.save_sextractor_data(final_cat_file, tile_sexcat_path) - cat_size_sextractor = n_obj - - # Save spread-model data - if sexcat_sm_path is None: - w_log.info("No sm cat input, setting spread model to 99") - else: - w_log.info("Save spread-model data") - cat_size_sm = make_cat.save_sm_data( - final_cat_file, - sexcat_sm_path, - do_classif, - star_thresh, - gal_thresh, - n_obj=n_obj - ) - - if cat_size_sextractor != cat_size_sm: - w_log( - f"Warnign: SExtractor catalogue {tile_sexcat_path} has different size" - + f" ({cat_size_sextractor} than spread_model catalogue" - + f" {sexcat_sm_path} ({cat_size_sm})" - ) + cat_size_sextractor = make_cat.save_sextractor_data( + final_cat_file, tile_sexcat_path + ) # Save shape data sc_inst = make_cat.SaveCatalogue(final_cat_file, cat_size_sextractor, w_log) diff --git a/src/shapepipe/modules/spread_model_package/__init__.py b/src/shapepipe/modules/spread_model_package/__init__.py deleted file mode 100644 index 247f60308..000000000 --- a/src/shapepipe/modules/spread_model_package/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -"""SPREAD MODEL PACKAGE. - -This package contains the module for ``spread_model``. - -:Author: Axel Guinot - -:Parent modules: - -- ``sextractor_runner`` -- ``psfex_interp_runner`` or ``mccd_interp_runner`` - and ``vignetmaker_runner`` - -:Input: SExtractor output catalogue, PSFEx output catalogue and vignet files - -:Output: Updated SExtractor or new FITS file - -Description -=========== - -This module refines the galaxy sample that will be used for shape measurement -using the spread model method of :cite:`desai:12` and :cite:`mohr:12`. - -Module-specific config file entries -=================================== - -PREFIX : str, optional - Ouput file prefix -OUTPUT_MODE : str - Output mode, options are ``add`` to add the outputs to the input - SExtractor catalogue or ``new`` to generte a new output catalogue - -""" - -__all__ = ["spread_model"] diff --git a/src/shapepipe/modules/spread_model_package/spread_model.py b/src/shapepipe/modules/spread_model_package/spread_model.py deleted file mode 100644 index 57d88dc43..000000000 --- a/src/shapepipe/modules/spread_model_package/spread_model.py +++ /dev/null @@ -1,312 +0,0 @@ -"""SPREAD MODEL. - -Class to compute the spread model, criterion to select galaxies - -:Author: Axel Guinot - -""" - -import galsim -import numpy as np -from cs_util import size as cs_size -from sqlitedict import SqliteDict - -from shapepipe.pipeline import file_io - - -def get_sm(obj_vign, psf_vign, model_vign, weight_vign): - """Get Spread Model. - - This method compute the spread moel for an object. - - Parameters - ---------- - obj_vign : numpy.ndarray - Vignet of the object - psf_vign : numpy.ndarray - Vignet of the gaussian model of the PSF - model_vign : numpy.ndarray - Vignet of the galaxy model - weight_vign : numpy.ndarray - Vignet of the weight at the object position - - Returns - ------- - tuple - Spread model and corresponding error values - - """ - # Mask invalid pixels - m = (obj_vign > -1e29) & (weight_vign > 0) - w = m.astype(float) - - # Set noise as inverse weight - noise_v = (1 / weight_vign).ravel() - - # Remove infinite noise pixels - noise_v[np.isinf(noise_v)] = 0 - - # Transform 2D vignets to 1D vectors - t_v = model_vign.ravel() - g_v = obj_vign.ravel() - psf_v = psf_vign.ravel() - w_v = w.ravel() - - # Compute scalar products used in spread model - tg = np.sum(t_v * w_v * g_v) - pg = np.sum(psf_v * w_v * g_v) - tp = np.sum(t_v * w_v * psf_v) - pp = np.sum(psf_v * w_v * psf_v) - - tnt = np.sum(t_v * noise_v * t_v * w_v) - pnp = np.sum(psf_v * noise_v * psf_v * w_v) - tnp = np.sum(t_v * noise_v * psf_v * w_v) - err = tnt * pg**2 + pnp * tg**2 - 2 * tnp * pg * tg - - # Compute spread model - if pg > 0: - sm = (tg / pg) - (tp / pp) - else: - sm = 1 - - if (pg > 0) & (err > 0): - sm_err = np.sqrt(err) / pg**2 - else: - sm_err = 1 - - return sm, sm_err - - -def get_model(sigma, flux, img_shape): - """Get Model. - - This method computes - - an exponential galaxy model with scale radius = 1/16 FWHM - - a Gaussian model for the PSF - - The spread-model statistic compares the object, PSF, and galaxy-model - vignets entirely in pixel space, so the models are rendered in pixel - units. A physical pixel scale would cancel out exactly (every length is - ``pixels x pixel_scale``, drawn at ``scale=pixel_scale``), so none is - needed here. - - Parameters - ---------- - sigma : float - Sigma of the PSF (in pixel units) - flux : float - Flux of the galaxy for the model - img_shape : list - Size of the output vignet ``[xsize, ysize]`` - - Returns - ------- - tuple - Vignet of the galaxy model and of the PSF model - - """ - # Get scale radius (pixels) - scale_radius = 1 / 16 * cs_size.sigma_to_fwhm(sigma) - - # Get galaxy model - gal_obj = galsim.Exponential(scale_radius=scale_radius, flux=flux) - - # Get PSF (sigma in pixels) - psf_obj = galsim.Gaussian(sigma=sigma) - - # Convolve both - gal_obj = galsim.Convolve(gal_obj, psf_obj) - - # Draw galaxy and PSF on vignets, in pixel units (scale = 1 pixel) - gal_vign = gal_obj.drawImage( - nx=img_shape[0], ny=img_shape[1], scale=1.0 - ).array - - psf_vign = psf_obj.drawImage( - nx=img_shape[0], ny=img_shape[1], scale=1.0 - ).array - - return gal_vign, psf_vign - - -class SpreadModel(object): - """The Spread Model Class. - - Parameters - ---------- - sex_cat_path : str - Path to SExtractor catalogue - psf_cat_path : str - Path to PSF catalogue - weight_cat_path : str - Path to weight catalogue - output_path : str - Output file path of pasted catalog - output_mode : str - Options are ``new`` or ``add`` - - Notes - ----- - For the ``output_mode``: - - - ``new`` will create a new catalogue with - ``[number, mag, sm, sm_err]`` - - ``add`` will output a copy of the input SExtractor with the columns - ``sm`` and ``sm_err`` - - """ - - def __init__( - self, - sex_cat_path, - psf_cat_path, - weight_cat_path, - output_path, - output_mode, - ): - - self._sex_cat_path = sex_cat_path - self._psf_cat_path = psf_cat_path - self._weight_cat_path = weight_cat_path - self._output_path = output_path - self._output_mode = output_mode - - def process(self): - """Process. - - Process the spread model computation - - """ - # Get data - sex_cat = file_io.FITSCatalogue(self._sex_cat_path, SEx_catalogue=True) - sex_cat.open() - obj_id = np.copy(sex_cat.get_data()["NUMBER"]) - obj_vign = np.copy(sex_cat.get_data()["VIGNET"]) - obj_mag = None - if self._output_mode == "new": - obj_mag = np.copy(sex_cat.get_data()["MAG_AUTO"]) - sex_cat.close() - - psf_cat = SqliteDict(self._psf_cat_path) - - weight_cat = file_io.FITSCatalogue( - self._weight_cat_path, - SEx_catalogue=True, - ) - weight_cat.open() - weigh_vign = weight_cat.get_data()["VIGNET"] - weight_cat.close() - - # Get spread model - skip_obj = False - spread_model_final = [] - spread_model_err_final = [] - for idx, id_tmp in enumerate(obj_id): - sigma_list = [] - - if psf_cat[str(id_tmp)] == "empty": - spread_model_final.append(-1) - spread_model_err_final.append(1) - continue - - psf_expccd_name = list(psf_cat[str(id_tmp)].keys()) - - for expccd_name_tmp in psf_expccd_name: - psf_cat_id_ccd = psf_cat[str(id_tmp)][expccd_name_tmp] - # The HSM grammar stores PSF size as T = 2 sigma^2 under - # HSM_T_PSF; spread model needs the Gaussian sigma (pixels). - sigma_list.append( - cs_size.T_to_sigma(psf_cat_id_ccd["SHAPES"]["HSM_T_PSF"]) - ) - - obj_sigma_tmp = np.mean(sigma_list) - if obj_sigma_tmp > 0: - obj_vign_tmp = obj_vign[idx] - obj_flux_tmp = 1.0 - obj_weight_tmp = weigh_vign[idx] - obj_model_tmp, obj_psf_tmp = get_model( - obj_sigma_tmp, - obj_flux_tmp, - obj_vign_tmp.shape, - ) - - obj_sm, obj_sm_err = get_sm( - obj_vign_tmp, obj_psf_tmp, obj_model_tmp, obj_weight_tmp - ) - else: - # size < 0, something is not right with this object - obj_sm, obj_sm_err = -1.0, -1.0 - - spread_model_final.append(obj_sm) - spread_model_err_final.append(obj_sm_err) - - spread_model_final = np.array(spread_model_final, dtype="float64") - spread_model_err_final = np.array( - spread_model_err_final, - dtype="float64", - ) - - psf_cat.close() - - self.save_results( - spread_model_final, spread_model_err_final, obj_mag, obj_id - ) - - def save_results(self, sm, sm_err, mag, number): - """Save Results. - - Save output catalogue with spread model and errors. - - Parameters - ---------- - sm : numpy.ndarray - Value of the spread model for all objects - sm_err : numpy.ndarray - Value of the spread model error for all objects - mag : numpy.ndarray - Magnitude of all objects (only for a new catalogue) - number : numpy.ndarray - ID of all objects (only for a new catalogue) - - Raises - ------ - ValueError - For incorrect output mode - - """ - if self._output_mode == "new": - new_cat = file_io.FITSCatalogue( - self._output_path, - SEx_catalogue=True, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - ) - dict_data = { - "NUMBER": number, - "MAG": mag, - "SPREAD_MODEL": sm, - "SPREADERR_MODEL": sm_err, - } - new_cat.save_as_fits( - data=dict_data, sex_cat_path=self._sex_cat_path - ) - elif self._output_mode == "add": - ori_cat = file_io.FITSCatalogue( - self._sex_cat_path, - SEx_catalogue=True, - ) - ori_cat.open() - new_cat = file_io.FITSCatalogue( - self._output_path, - SEx_catalogue=True, - open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, - ) - ori_cat.add_col( - "SPREAD_MODEL", sm, new_cat=True, new_cat_inst=new_cat - ) - ori_cat.add_col("SPREAD_MODEL", sm, new_cat=True, new_cat_inst=new_cat) - ori_cat.close() - new_cat.open() - new_cat.add_col("SPREADERR_MODEL", sm_err) - new_cat.close() - else: - raise ValueError("Mode must be in [new, add].") diff --git a/src/shapepipe/modules/spread_model_runner.py b/src/shapepipe/modules/spread_model_runner.py deleted file mode 100644 index 93b70b753..000000000 --- a/src/shapepipe/modules/spread_model_runner.py +++ /dev/null @@ -1,68 +0,0 @@ -"""SPREAD MODEL RUNNER. - -Module runner for ``spread_model``. - -:Author: Axel Guinot - -""" - -from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.spread_model_package.spread_model import SpreadModel - - -@module_runner( - version="1.1", - input_module=[ - "sextractor_runner", - "psfex_interp_runner", - "vignetmaker_runner", - ], - file_pattern=["sexcat", "galaxy_psf", "weight_vign"], - file_ext=[".fits", ".sqlite", ".fits"], - depends=["numpy", "galsim"], - run_method="parallel", -) -def spread_model_runner( - input_file_list, - run_dirs, - file_number_string, - config, - module_config_sec, - w_log, -): - """Define The Spread Model Runner.""" - # Get input files - sex_cat_path, psf_cat_path, weight_cat_path = input_file_list - - # Get file prefix (optional) - if config.has_option(module_config_sec, "PREFIX"): - prefix = config.get(module_config_sec, "PREFIX") - if (prefix.lower() != "none") & (prefix != ""): - prefix = prefix + "_" - else: - prefix = "" - else: - prefix = "" - - # Get output mode. The spread model is computed entirely in pixel space, - # so no pixel scale is needed (see spread_model.get_model). - output_mode = config.get(module_config_sec, "OUTPUT_MODE") - - # Set output file path - file_name = f"{prefix}sexcat_sm{file_number_string}.fits" - output_path = f'{run_dirs["output"]}/{file_name}' - - # Create spread model class instance - sm_inst = SpreadModel( - sex_cat_path, - psf_cat_path, - weight_cat_path, - output_path, - output_mode, - ) - - # Process spread model computation - sm_inst.process() - - # No return objects - return None, None diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index 520a82fa5..bdcd13f84 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -24,8 +24,12 @@ from ngmix.flags import LM_FUNC_NOTFINITE, NAME_MAP from sqlitedict import SqliteDict +from shapepipe.modules.make_cat_package import make_cat from shapepipe.modules.make_cat_package.make_cat import SaveCatalogue +from shapepipe.modules.make_cat_runner import make_cat_runner from shapepipe.modules.ngmix_package.ngmix import Ngmix +from shapepipe.pipeline import file_io +from shapepipe.pipeline.config import CustomParser class _NullLogger: @@ -472,6 +476,73 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): assert out[col][1] == -1, col +# --- make_cat_runner: end-to-end catalogue assembly --- + + +def _write_sex_like_cat(path, data): + """Write a minimal SExtractor-format FITS (data lives at HDU index 2).""" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.BinTableHDU(name="LDAC_IMHEAD"), + fits.BinTableHDU(data, name="LDAC_OBJECTS"), + ] + ).writeto(str(path), overwrite=True) + + +def _numbered_data(obj_ids): + """A ``NUMBER`` + dummy second field structured array, one row per id. + + A structured array with a single field gets collapsed by + ``file_io.FITSCatalogue._save_to_fits`` into one row holding a + vector-valued column (``len(names) == 1`` triggers a + ``data = np.array([data])`` wrap), so every SExtractor-like fixture + that ``save_sextractor_data`` re-saves through ``save_as_fits`` needs a + second field to keep one row per object. + """ + return np.array( + [(oid, 0.0) for oid in obj_ids], + dtype=[("NUMBER", "i8"), ("X_IMAGE", "f8")], + ) + + +def test_make_cat_runner_ships_every_detection_unclassified(tmp_path): + """The runner assembles every detection, with no star/galaxy column. + + Star/galaxy separation happens downstream, so the final catalogue keeps + each SExtractor object and carries no classification or spread-model + column. + """ + obj_ids = [1, 2, 3] + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + ngmix_path = tmp_path / "ngmix-350-100.fits" + _write_ngmix_cat(ngmix_path, obj_ids) + + config = CustomParser() + config.read_dict({"MAKE_CAT_RUNNER": {"SHAPE_MEASUREMENT_TYPE": "ngmix"}}) + + result = make_cat_runner( + [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + _NullLogger(), + ) + + assert result == (None, None) + final_cat = file_io.FITSCatalogue( + str(make_cat.get_output_name(str(tmp_path), "-350-100")) + ) + final_cat.open() + data = final_cat.get_data() + final_cat.close() + npt.assert_array_equal(data["NUMBER"], obj_ids) + assert not [name for name in data.dtype.names if "SPREAD" in name] + + @pytest.mark.parametrize("shear", SHEAR_EXTS) @pytest.mark.parametrize("component", [0, 1], ids=["g1", "g2"]) @pytest.mark.parametrize("nonfinite", [np.nan, np.inf, -np.inf]) diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 9475a819c..636803f48 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -1,5 +1,4 @@ -# ShapePipe post-run configuration file: create final catalogs, with -# no spread model on input +# ShapePipe post-run configuration file: create final catalogs ## Default ShapePipe options @@ -79,9 +78,6 @@ FILE_EXT = .fits, .sqlite, .fits # sections below NUMBERING_SCHEME = -000-000 -# @sc [decision:catalogue_assembly.star_galaxy_classification] -SM_DO_CLASSIFICATION = False - SHAPE_MEASUREMENT_TYPE = ngmix # Save per-epoch PSF shapes and epoch identity (HSM_*_PSF_n, EXP_ID_n, diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 6ebd1aae9..becb72501 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -19,13 +19,6 @@ NGMIX_MCAL_FLAGS NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# spread class -#SPREAD_CLASS - -# spread model flag and error -#SPREAD_MODEL -#SPREADERR_MODEL - # Number of epochs (exposures) N_EPOCH NGMIX_N_EPOCH