diff --git a/astra.yaml b/astra.yaml index 24dd350d3..cd45fa59e 100644 --- a/astra.yaml +++ b/astra.yaml @@ -129,8 +129,7 @@ decisions: Values: default_noimaflags.param#VIGNET = 51; default.param#VIGNET = 51; - VIGNETMAKER_RUNNER_RUN_1.STAMP_SIZE = 51; - VIGNETMAKER_RUNNER_RUN_2.STAMP_SIZE = 51; + VIGNETMAKER_RUNNER.STAMP_SIZE = 51; PSF_SIZE = 51. default: px_51 options: @@ -241,7 +240,7 @@ analyses: unmodified pixels as one outside it. Values: config_exp_psfex.ini#SEXTRACTOR_RUNNER.FLAG_IMAGE = True; - VIGNETMAKER_RUNNER_RUN_2.ME_IMAGE_PATTERN = flag, image, weight, + VIGNETMAKER_RUNNER.ME_IMAGE_PATTERN = flag, image, weight, background, background_rms. default: instrument_flags_only options: @@ -788,12 +787,10 @@ analyses: blends and asymmetric galaxies, and the centroid feeds the position seed and the centroid prior. Values: - VIGNETMAKER_RUNNER_RUN_1.POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE; - VIGNETMAKER_RUNNER_RUN_1.COORD = PIX; config_tile_PiViVi_psfex.ini#PSFEX_INTERP_RUNNER.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD; - VIGNETMAKER_RUNNER_RUN_2.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD; - VIGNETMAKER_RUNNER_RUN_2.COORD = SPHE; + VIGNETMAKER_RUNNER.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD; + VIGNETMAKER_RUNNER.COORD = SPHE; config_exp_psfex.ini#PSFEX_INTERP_RUNNER.POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE. default: xwin_windowed @@ -1339,7 +1336,7 @@ analyses: (BKG_SUB, on unless set False). Values: NGMIX_RUNNER.BKG_RMS_VIGNET_PATH = - $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite. + $NGMIX_VIGNET_DIR/vignetmaker_runner/output/background_rms_vignet{file_number_string}.sqlite. default: rms_vignet_weights options: rms_vignet_weights: @@ -1398,8 +1395,7 @@ analyses: the galaxy light, is bounded by central_defect_veto. Noise stays the default until a survey A/B against interpolation. Values: - VIGNETMAKER_RUNNER_RUN_1.MASKING = False; - VIGNETMAKER_RUNNER_RUN_2.MASKING = False. + VIGNETMAKER_RUNNER.MASKING = False. default: noise options: noise: diff --git a/tests/unit/test_config_parse.py b/tests/unit/test_config_parse.py index 4aff1efbf..7c40f239b 100644 --- a/tests/unit/test_config_parse.py +++ b/tests/unit/test_config_parse.py @@ -160,8 +160,8 @@ def _module_config_sections(raw_module_list): Mirrors FileHandler.set_up_module / get_module_config_sec: a module named once uses its own upper-cased name as its section; a module repeated in MODULE gets ``_run_`` (1-indexed) upper-cased instead — every - invocation is returned, so a module used twice (e.g. ``vignetmaker_runner`` - for ``_RUN_1`` and ``_RUN_2``) yields two pairs, not one. A module whose + invocation is returned, so a module used twice yields two pairs + (``_RUN_1`` and ``_RUN_2``), not one. A module whose name is not resolvable statically (e.g. ``${SP_PSF}_interp_runner``) is skipped. """ @@ -230,24 +230,43 @@ def test_workflow_config_module_sections_have_required_keys(config_path): assert not problems, f"{config_path}: " + "; ".join(problems) +def test_module_sections_check_catches_a_missing_singleton_section(): + """A module called once must have its configuration section checked.""" + config_path = next( + p for p in WORKFLOW_CONFIG_FILES if p.name == "config_tile_PiViVi_mccd.ini" + ) + + parser = configparser.ConfigParser() + parser.read(config_path) + assert parser.has_section("VIGNETMAKER_RUNNER") + + parser.remove_section("VIGNETMAKER_RUNNER") + + problems = _missing_required_keys(parser) + + assert any("VIGNETMAKER_RUNNER" in problem for problem in problems) + + def test_module_sections_check_catches_a_missing_repeated_section(): """A module used twice must have both of its sections checked. Regression guard for _module_config_sections: it must return one - (module, section) pair per invocation of a repeated module, not - collapse them, or deleting one of the two sections a module needs - (here vignetmaker_runner's _RUN_1) would go unnoticed. + (module, section) pair per invocation of a repeated module, not collapse + them, or a missing ``_RUN_1`` section would go unnoticed. """ config_path = next( - p for p in WORKFLOW_CONFIG_FILES if p.name == "config_tile_PiViVi_mccd.ini" + p for p in WORKFLOW_CONFIG_FILES if p.name == "config_tile_PiViVi_psfex.ini" ) parser = configparser.ConfigParser() parser.read(config_path) - assert parser.has_section("VIGNETMAKER_RUNNER_RUN_1") - - parser.remove_section("VIGNETMAKER_RUNNER_RUN_1") + parser.set("EXECUTION", "MODULE", "vignetmaker_runner, vignetmaker_runner") + parser.add_section("VIGNETMAKER_RUNNER_RUN_2") + for key, value in parser.items("VIGNETMAKER_RUNNER", raw=True): + parser.set("VIGNETMAKER_RUNNER_RUN_2", key, value) problems = _missing_required_keys(parser) - assert any("VIGNETMAKER_RUNNER_RUN_1" in problem for problem in problems) + assert problems == [ + "vignetmaker_runner: missing section [VIGNETMAKER_RUNNER_RUN_1]" + ] diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index eeb61d950..6a043a9d5 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -16,15 +16,15 @@ "tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184", "tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c", "tile_get_images": "331a67e747f211ebf4c14b946a7af7f9fc9f55243d69d2791d74aecc3ca228c3", - "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23", + "tile_make_cat": "edbde3a9cfec5a1e9123708123a9ac822c17ab2a1b6e471015721a3208f69e2c", "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb", "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111", - "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415", + "tile_ngmix": "c1a80bee8c930ae81a799bc7b2a4a9f5d7506fe1ecc9afef9931e582b0e12341", "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3", "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e", + "sha256": "44e5634faefe6fa8a0c084827f734c6c9963ea215b72ba07555317e588e9e3bf", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index 61addb4f2..c31d94dc9 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -66,7 +66,7 @@ TIMEOUT = 96:00:00 # staging possible: [FILE] OUTPUT_DIR stays $SP_RUN/output (scratch, durable # for the life of the job's tile), while the two directories ngmix actually # READS AT RANDOM -- the psfex_interp galaxy_psf store and the vignetmaker -# run_2 image/background/weight/flag vignet stores, together the ~5.6 GB +# image/background/weight/flag stores, together the ~5.6 GB # per-tile vignette store that all 8 chunks re-read -- come from # $NGMIX_VIGNET_DIR. # @@ -88,7 +88,7 @@ TIMEOUT = 96:00:00 # with everything else already node-local. tile_local() copies it beside the # store and points this variable at the copy. Set it to # $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output to read in place. -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner/output, $SP_WCS_DIR FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers @@ -102,7 +102,7 @@ NUMBERING_SCHEME = -000-000 # every tile (missing file -> error, no per-tile fallback); omit the option # entirely to fall back to the scalar sigma_mad noise estimate. # @sc [decision:shape_measurement.galaxy_pixel_weights] -BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite +BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner/output/background_rms_vignet{file_number_string}.sqlite # Number of objects to batch save during processing, optional. Omit or set # to -1 for no batch saving. diff --git a/workflow/config/cfis/config_tile_PiViVi_mccd.ini b/workflow/config/cfis/config_tile_PiViVi_mccd.ini index 864f790f9..2c06d992a 100644 --- a/workflow/config/cfis/config_tile_PiViVi_mccd.ini +++ b/workflow/config/cfis/config_tile_PiViVi_mccd.ini @@ -22,7 +22,7 @@ RUN_DATETIME = False #MODULE = mccd_interp_runner, # @sc [decision:star_selection_psf.psf_modelling_software] -MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -106,42 +106,9 @@ ME_DOT_PSF_PATTERN = fitted_model # Multi-epoch mode parameters -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[VIGNETMAKER_RUNNER_RUN_2] - # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures +[VIGNETMAKER_RUNNER] INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output diff --git a/workflow/config/cfis/config_tile_PiViVi_psfex.ini b/workflow/config/cfis/config_tile_PiViVi_psfex.ini index 62073d482..afae52a97 100644 --- a/workflow/config/cfis/config_tile_PiViVi_psfex.ini +++ b/workflow/config/cfis/config_tile_PiViVi_psfex.ini @@ -22,7 +22,7 @@ RUN_DATETIME = False #MODULE = psfex_interp_runner, # @sc [decision:star_selection_psf.psf_modelling_software] -MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -120,46 +120,9 @@ ME_DOT_PSF_EXP_DIR = $SP_EXP ME_DOT_PSF_PATTERN = star_split_ratio_80 -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -# @sc [decision:shape_measurement.defect_fill] -MASKING = False - -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -# @sc [decision:preparation.object_position_columns] -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -# @sc [decision:postage_stamp_size] -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[VIGNETMAKER_RUNNER_RUN_2] - # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures +[VIGNETMAKER_RUNNER] INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini index 4651c4e7c..3ccfb1577 100644 --- a/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini @@ -22,7 +22,7 @@ RUN_DATETIME = False # Module name, single string or comma-separated list of valid module runner names #MODULE = psfex_interp_runner, -MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner # Parallel processing mode, SMP or MPI MODE = SMP @@ -94,42 +94,9 @@ NUMBERING_SCHEME = -000-000 PSF_DICT_PATH = $PSF_DICT -# Create vignets for tiles weights -[VIGNETMAKER_RUNNER_RUN_1] - -INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output - -FILE_PATTERN = sexcat, CFIS_weight - -FILE_EXT = .fits, .fits - -# NUMBERING_SCHEME (optional) string with numbering pattern for input files -NUMBERING_SCHEME = -000-000 - -MASKING = False -MASK_VALUE = 0 - -# Run mode for psfex interpolation: -# CLASSIC: 'classical' run, interpolate to object positions -# MULTI-EPOCH: interpolate for multi-epoch images -# VALIDATION: validation for single-epoch images -MODE = CLASSIC - -# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) -COORD = PIX -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Vignet size in pixels -STAMP_SIZE = 51 - -# Output file name prefix, file name is _vignet.fits -PREFIX = weight - - -[VIGNETMAKER_RUNNER_RUN_2] - # Create multi-epoch vignets for tiles corresponding to # positions on single-exposures +[VIGNETMAKER_RUNNER] INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index f105d23b4..1416cc449 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -331,7 +331,7 @@ TILE_NGMIX_RANGES = ( # vignets manifest would break clean_exposure, which keys reclamation # eligibility on exactly that file. TILE_VIGNET_REQUIRED = r""" -if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output" ]; then +if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner/output" ]; then echo "tile_shape: the node-local vignette store is missing." >&2 echo " expected: $NGMIX_VIGNET_DIR" >&2 echo " This means tile_vignets did NOT run in this group job — its manifest" >&2 diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 52720b724..7891949f4 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -151,25 +151,22 @@ }, "tile_vignets": { "psfex": { - "psfex_interp_runner": dict(expect=1), - "vignetmaker_runner_run_1": dict(expect=1), - # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); - # v2.0's 4 was the canfar flavor. every vignette feeds ngmix, so the expected count is all-or-nothing. - "vignetmaker_runner_run_2": dict(expect=5), + "psfex_interp_runner": dict(expect=1), + # 5 sqlites/tile (image/weight/flag/background/background_rms); + # every vignette feeds ngmix, so the expected count is all-or-nothing. + "vignetmaker_runner": dict(expect=5), }, # As psfex: mccd_interp writes the tile's galaxy_psf store from the # exposures' focal-plane models (SKiLLS star sim 1z2z_1, 233.293). "mccd": { - "mccd_interp_runner": dict(expect=1), - "vignetmaker_runner_run_1": dict(expect=1), - "vignetmaker_runner_run_2": dict(expect=5), + "mccd_interp_runner": dict(expect=1), + "vignetmaker_runner": dict(expect=5), }, # Image simulations: fake_interp_runner writes the same galaxy_psf # sqlite psfex_interp_runner writes, from the simulation's PSF dictionary. "fake": { - "fake_interp_runner": dict(expect=1), - "vignetmaker_runner_run_1": dict(expect=1), - "vignetmaker_runner_run_2": dict(expect=5), + "fake_interp_runner": dict(expect=1), + "vignetmaker_runner": dict(expect=5), }, }, # One check runs inside run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u per chunk,