Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions astra.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,7 @@ decisions:
Values:
default_noimaflags.param#VIGNET = 51;
default.param#VIGNET = 51;
VIGNETMAKER_RUNNER_RUN_1.STAMP_SIZE = 51;
VIGNETMAKER_RUNNER_RUN_2.STAMP_SIZE = 51;
VIGNETMAKER_RUNNER.STAMP_SIZE = 51;
PSF_SIZE = 51.
default: px_51
options:
Expand Down Expand Up @@ -241,7 +240,7 @@ analyses:
unmodified pixels as one outside it.
Values:
config_exp_psfex.ini#SEXTRACTOR_RUNNER.FLAG_IMAGE = True;
VIGNETMAKER_RUNNER_RUN_2.ME_IMAGE_PATTERN = flag, image, weight,
VIGNETMAKER_RUNNER.ME_IMAGE_PATTERN = flag, image, weight,
background, background_rms.
default: instrument_flags_only
options:
Expand Down Expand Up @@ -788,12 +787,10 @@ analyses:
blends and asymmetric galaxies, and the centroid feeds the position
seed and the centroid prior.
Values:
VIGNETMAKER_RUNNER_RUN_1.POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE;
VIGNETMAKER_RUNNER_RUN_1.COORD = PIX;
config_tile_PiViVi_psfex.ini#PSFEX_INTERP_RUNNER.POSITION_PARAMS =
XWIN_WORLD,YWIN_WORLD;
VIGNETMAKER_RUNNER_RUN_2.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD;
VIGNETMAKER_RUNNER_RUN_2.COORD = SPHE;
VIGNETMAKER_RUNNER.POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD;
VIGNETMAKER_RUNNER.COORD = SPHE;
config_exp_psfex.ini#PSFEX_INTERP_RUNNER.POSITION_PARAMS =
XWIN_IMAGE,YWIN_IMAGE.
default: xwin_windowed
Expand Down Expand Up @@ -1339,7 +1336,7 @@ analyses:
(BKG_SUB, on unless set False).
Values:
NGMIX_RUNNER.BKG_RMS_VIGNET_PATH =
$NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite.
$NGMIX_VIGNET_DIR/vignetmaker_runner/output/background_rms_vignet{file_number_string}.sqlite.
default: rms_vignet_weights
options:
rms_vignet_weights:
Expand Down Expand Up @@ -1398,8 +1395,7 @@ analyses:
the galaxy light, is bounded by central_defect_veto. Noise stays the
default until a survey A/B against interpolation.
Values:
VIGNETMAKER_RUNNER_RUN_1.MASKING = False;
VIGNETMAKER_RUNNER_RUN_2.MASKING = False.
VIGNETMAKER_RUNNER.MASKING = False.
default: noise
options:
noise:
Expand Down
39 changes: 29 additions & 10 deletions tests/unit/test_config_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,8 @@ def _module_config_sections(raw_module_list):
Mirrors FileHandler.set_up_module / get_module_config_sec: a module named
once uses its own upper-cased name as its section; a module repeated in
MODULE gets ``<module>_run_<n>`` (1-indexed) upper-cased instead — every
invocation is returned, so a module used twice (e.g. ``vignetmaker_runner``
for ``_RUN_1`` and ``_RUN_2``) yields two pairs, not one. A module whose
invocation is returned, so a module used twice yields two pairs
(``_RUN_1`` and ``_RUN_2``), not one. A module whose
name is not resolvable statically (e.g. ``${SP_PSF}_interp_runner``) is
skipped.
"""
Expand Down Expand Up @@ -230,24 +230,43 @@ def test_workflow_config_module_sections_have_required_keys(config_path):
assert not problems, f"{config_path}: " + "; ".join(problems)


def test_module_sections_check_catches_a_missing_singleton_section():
"""A module called once must have its configuration section checked."""
config_path = next(
p for p in WORKFLOW_CONFIG_FILES if p.name == "config_tile_PiViVi_mccd.ini"
)

parser = configparser.ConfigParser()
parser.read(config_path)
assert parser.has_section("VIGNETMAKER_RUNNER")

parser.remove_section("VIGNETMAKER_RUNNER")

problems = _missing_required_keys(parser)

assert any("VIGNETMAKER_RUNNER" in problem for problem in problems)


def test_module_sections_check_catches_a_missing_repeated_section():
"""A module used twice must have both of its sections checked.

Regression guard for _module_config_sections: it must return one
(module, section) pair per invocation of a repeated module, not
collapse them, or deleting one of the two sections a module needs
(here vignetmaker_runner's _RUN_1) would go unnoticed.
(module, section) pair per invocation of a repeated module, not collapse
them, or a missing ``_RUN_1`` section would go unnoticed.
"""
config_path = next(
p for p in WORKFLOW_CONFIG_FILES if p.name == "config_tile_PiViVi_mccd.ini"
p for p in WORKFLOW_CONFIG_FILES if p.name == "config_tile_PiViVi_psfex.ini"
)

parser = configparser.ConfigParser()
parser.read(config_path)
assert parser.has_section("VIGNETMAKER_RUNNER_RUN_1")

parser.remove_section("VIGNETMAKER_RUNNER_RUN_1")
parser.set("EXECUTION", "MODULE", "vignetmaker_runner, vignetmaker_runner")
parser.add_section("VIGNETMAKER_RUNNER_RUN_2")
for key, value in parser.items("VIGNETMAKER_RUNNER", raw=True):
parser.set("VIGNETMAKER_RUNNER_RUN_2", key, value)

problems = _missing_required_keys(parser)

assert any("VIGNETMAKER_RUNNER_RUN_1" in problem for problem in problems)
assert problems == [
"vignetmaker_runner: missing section [VIGNETMAKER_RUNNER_RUN_1]"
]
6 changes: 3 additions & 3 deletions tests/workflow/params_pin.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@
"tile_find_exposures": "8704317871744996c44351c2836fcb222d7986a602e9e046a90d684ba7b3c184",
"tile_get_catalogue": "7e3f889a955a14b0b917a015c2a8bc90e433a89b1cc5e00b8ca94695c4c93d3c",
"tile_get_images": "331a67e747f211ebf4c14b946a7af7f9fc9f55243d69d2791d74aecc3ca228c3",
"tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23",
"tile_make_cat": "edbde3a9cfec5a1e9123708123a9ac822c17ab2a1b6e471015721a3208f69e2c",
"tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb",
"tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111",
"tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415",
"tile_ngmix": "c1a80bee8c930ae81a799bc7b2a4a9f5d7506fe1ecc9afef9931e582b0e12341",
"tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3",
"tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91"
},
"schema": 1,
"sha256": "f9772987e517e9827c8d501d310ec323be251a62b7b89aad0725a03f8f4f120e",
"sha256": "44e5634faefe6fa8a0c084827f734c6c9963ea215b72ba07555317e588e9e3bf",
"unit_pre": {
"exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c",
"exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec",
Expand Down
6 changes: 3 additions & 3 deletions workflow/config/cfis/config_tile_Ng_template.ini
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ TIMEOUT = 96:00:00
# staging possible: [FILE] OUTPUT_DIR stays $SP_RUN/output (scratch, durable
# for the life of the job's tile), while the two directories ngmix actually
# READS AT RANDOM -- the psfex_interp galaxy_psf store and the vignetmaker
# run_2 image/background/weight/flag vignet stores, together the ~5.6 GB
# image/background/weight/flag stores, together the ~5.6 GB
# per-tile vignette store that all 8 chunks re-read -- come from
# $NGMIX_VIGNET_DIR.
#
Expand All @@ -88,7 +88,7 @@ TIMEOUT = 96:00:00
# with everything else already node-local. tile_local() copies it beside the
# store and points this variable at the copy. Set it to
# $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output to read in place.
INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output, $SP_WCS_DIR
INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $NGMIX_VIGNET_DIR/${SP_PSF}_interp_runner/output, $NGMIX_VIGNET_DIR/vignetmaker_runner/output, $SP_WCS_DIR

FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers

Expand All @@ -102,7 +102,7 @@ NUMBERING_SCHEME = -000-000
# every tile (missing file -> error, no per-tile fallback); omit the option
# entirely to fall back to the scalar sigma_mad noise estimate.
# @sc [decision:shape_measurement.galaxy_pixel_weights]
BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite
BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner/output/background_rms_vignet{file_number_string}.sqlite

# Number of objects to batch save during processing, optional. Omit or set
# to -1 for no batch saving.
Expand Down
37 changes: 2 additions & 35 deletions workflow/config/cfis/config_tile_PiViVi_mccd.ini
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN_DATETIME = False
#MODULE = mccd_interp_runner,

# @sc [decision:star_selection_psf.psf_modelling_software]
MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner
MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner

# Parallel processing mode, SMP or MPI
MODE = SMP
Expand Down Expand Up @@ -106,42 +106,9 @@ ME_DOT_PSF_PATTERN = fitted_model

# Multi-epoch mode parameters

# Create vignets for tiles weights
[VIGNETMAKER_RUNNER_RUN_1]

INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output

FILE_PATTERN = sexcat, CFIS_weight

FILE_EXT = .fits, .fits

# NUMBERING_SCHEME (optional) string with numbering pattern for input files
NUMBERING_SCHEME = -000-000

MASKING = False
MASK_VALUE = 0

# Run mode for psfex interpolation:
# CLASSIC: 'classical' run, interpolate to object positions
# MULTI-EPOCH: interpolate for multi-epoch images
# VALIDATION: validation for single-epoch images
MODE = CLASSIC

# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates)
COORD = PIX
POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE

# Vignet size in pixels
STAMP_SIZE = 51

# Output file name prefix, file name is <PREFIX>_vignet.fits
PREFIX = weight


[VIGNETMAKER_RUNNER_RUN_2]

# Create multi-epoch vignets for tiles corresponding to
# positions on single-exposures
[VIGNETMAKER_RUNNER]

INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output

Expand Down
41 changes: 2 additions & 39 deletions workflow/config/cfis/config_tile_PiViVi_psfex.ini
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN_DATETIME = False
#MODULE = psfex_interp_runner,

# @sc [decision:star_selection_psf.psf_modelling_software]
MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner
MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner

# Parallel processing mode, SMP or MPI
MODE = SMP
Expand Down Expand Up @@ -120,46 +120,9 @@ ME_DOT_PSF_EXP_DIR = $SP_EXP
ME_DOT_PSF_PATTERN = star_split_ratio_80


# Create vignets for tiles weights
[VIGNETMAKER_RUNNER_RUN_1]

INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output

FILE_PATTERN = sexcat, CFIS_weight

FILE_EXT = .fits, .fits

# NUMBERING_SCHEME (optional) string with numbering pattern for input files
NUMBERING_SCHEME = -000-000

# @sc [decision:shape_measurement.defect_fill]
MASKING = False

MASK_VALUE = 0

# Run mode for psfex interpolation:
# CLASSIC: 'classical' run, interpolate to object positions
# MULTI-EPOCH: interpolate for multi-epoch images
# VALIDATION: validation for single-epoch images
MODE = CLASSIC

# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates)
# @sc [decision:preparation.object_position_columns]
COORD = PIX
POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE

# Vignet size in pixels
# @sc [decision:postage_stamp_size]
STAMP_SIZE = 51

# Output file name prefix, file name is <PREFIX>_vignet.fits
PREFIX = weight


[VIGNETMAKER_RUNNER_RUN_2]

# Create multi-epoch vignets for tiles corresponding to
# positions on single-exposures
[VIGNETMAKER_RUNNER]

INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output

Expand Down
37 changes: 2 additions & 35 deletions workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN_DATETIME = False
# Module name, single string or comma-separated list of valid module runner names
#MODULE = psfex_interp_runner,

MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner
MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner

# Parallel processing mode, SMP or MPI
MODE = SMP
Expand Down Expand Up @@ -94,42 +94,9 @@ NUMBERING_SCHEME = -000-000
PSF_DICT_PATH = $PSF_DICT


# Create vignets for tiles weights
[VIGNETMAKER_RUNNER_RUN_1]

INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output

FILE_PATTERN = sexcat, CFIS_weight

FILE_EXT = .fits, .fits

# NUMBERING_SCHEME (optional) string with numbering pattern for input files
NUMBERING_SCHEME = -000-000

MASKING = False
MASK_VALUE = 0

# Run mode for psfex interpolation:
# CLASSIC: 'classical' run, interpolate to object positions
# MULTI-EPOCH: interpolate for multi-epoch images
# VALIDATION: validation for single-epoch images
MODE = CLASSIC

# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates)
COORD = PIX
POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE

# Vignet size in pixels
STAMP_SIZE = 51

# Output file name prefix, file name is <PREFIX>_vignet.fits
PREFIX = weight


[VIGNETMAKER_RUNNER_RUN_2]

# Create multi-epoch vignets for tiles corresponding to
# positions on single-exposures
[VIGNETMAKER_RUNNER]

INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output

Expand Down
2 changes: 1 addition & 1 deletion workflow/rules/tile.smk
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ TILE_NGMIX_RANGES = (
# vignets manifest would break clean_exposure, which keys reclamation
# eligibility on exactly that file.
TILE_VIGNET_REQUIRED = r"""
if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output" ]; then
if [ ! -d "$NGMIX_VIGNET_DIR/vignetmaker_runner/output" ]; then
echo "tile_shape: the node-local vignette store is missing." >&2
echo " expected: $NGMIX_VIGNET_DIR" >&2
echo " This means tile_vignets did NOT run in this group job — its manifest" >&2
Expand Down
19 changes: 8 additions & 11 deletions workflow/scripts/completeness.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,25 +151,22 @@
},
"tile_vignets": {
"psfex": {
"psfex_interp_runner": dict(expect=1),
"vignetmaker_runner_run_1": dict(expect=1),
# 5 sqlites/tile on nibi (image/weight/flag/background/background_rms);
# v2.0's 4 was the canfar flavor. every vignette feeds ngmix, so the expected count is all-or-nothing.
"vignetmaker_runner_run_2": dict(expect=5),
"psfex_interp_runner": dict(expect=1),
# 5 sqlites/tile (image/weight/flag/background/background_rms);
# every vignette feeds ngmix, so the expected count is all-or-nothing.
"vignetmaker_runner": dict(expect=5),
},
# As psfex: mccd_interp writes the tile's galaxy_psf store from the
# exposures' focal-plane models (SKiLLS star sim 1z2z_1, 233.293).
"mccd": {
"mccd_interp_runner": dict(expect=1),
"vignetmaker_runner_run_1": dict(expect=1),
"vignetmaker_runner_run_2": dict(expect=5),
"mccd_interp_runner": dict(expect=1),
"vignetmaker_runner": dict(expect=5),
},
# Image simulations: fake_interp_runner writes the same galaxy_psf
# sqlite psfex_interp_runner writes, from the simulation's PSF dictionary.
"fake": {
"fake_interp_runner": dict(expect=1),
"vignetmaker_runner_run_1": dict(expect=1),
"vignetmaker_runner_run_2": dict(expect=5),
"fake_interp_runner": dict(expect=1),
"vignetmaker_runner": dict(expect=5),
},
},
# One check runs inside run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u per chunk,
Expand Down
Loading