├── .codecov.yml ├── .flake8 ├── .github ├── CODEOWNERS ├── dependabot.yml ├── labeler.yml ├── pull_request_template.md └── workflows │ ├── build.yml │ ├── changelog.yml │ ├── contexts.yml │ ├── data.yml │ ├── download_data.yml │ ├── label_pull_request.yml │ ├── retrieve_cache.yml │ ├── roman_ci.yml │ ├── roman_ci_cron.yaml │ └── tests_devdeps.yml ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGES.rst ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── changes ├── .gitkeep ├── 1438.flatfield.rst ├── 1760.general.rst ├── 1764.assign_wcs.rst └── 1776.mosaic_pipeline.rst ├── docs ├── Makefile ├── _static │ ├── favicon.ico │ ├── roman_logo_black_w200px.png │ └── roman_logo_white_w100px.png ├── conf.py ├── conftest.py ├── exts │ └── numfig.py ├── images │ ├── r0000501001001001001_0001_wfi01_cal.png │ ├── r0000501001001001001_0001_wfi01_cal_thumb.png │ └── wfi_array.png ├── index.rst ├── make.bat ├── roman │ ├── assign_wcs │ │ ├── index.rst │ │ └── main.rst │ ├── associations │ │ ├── asn_from_list.rst │ │ ├── commands.rst │ │ ├── design.rst │ │ ├── graphics │ │ │ ├── generator_flow_concept.png │ │ │ ├── generator_list_processing.png │ │ │ ├── level3_rule_inheritance.png │ │ │ ├── overview.png │ │ │ ├── overview_classes.png │ │ │ ├── rule_sets.png │ │ │ ├── rule_to_association.png │ │ │ ├── sources │ │ │ │ ├── associations.monopic │ │ │ │ ├── workflow-generic.odg │ │ │ │ └── workflow-wfss.odg │ │ │ ├── workflow-generic.pdf │ │ │ ├── workflow-generic.png │ │ │ ├── workflow-wfss.pdf │ │ │ └── workflow-wfss.png │ │ ├── index.rst │ │ ├── mk_skycell_asn_from_skycell_list.rst │ │ ├── mk_skycell_list.rst │ │ ├── overview.rst │ │ ├── roman_conventions.rst │ │ ├── skycell_asn.rst │ │ └── technote_sdp_workflow.rst │ ├── changes.rst │ ├── dark_current │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ └── reference_files.rst │ ├── data_products │ │ ├── file_naming.rst │ │ ├── index.rst │ │ ├── product_types.rst │ │ ├── science_products.rst │ │ └── stages.rst │ ├── datamodels │ │ ├── datamodels_asdf.rst │ │ ├── index.rst │ │ ├── library.rst │ │ ├── metadata.rst │ │ └── models.rst │ ├── dq_init │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ └── reference_files.rst │ ├── error_propagation │ │ ├── index.rst │ │ └── main.rst │ ├── flatfield │ │ ├── index.rst │ │ ├── main.rst │ │ └── reference_files.rst │ ├── flux │ │ ├── arguments.rst │ │ ├── flux_step.rst │ │ ├── index.rst │ │ └── main.rst │ ├── includes │ │ └── standard_keywords.inc │ ├── introduction.rst │ ├── linearity │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ └── reference_files.rst │ ├── outlier_detection │ │ ├── arguments.rst │ │ ├── index.rst │ │ ├── main.rst │ │ ├── outlier_detection.rst │ │ ├── outlier_detection_step.rst │ │ └── outlier_examples.rst │ ├── package_index.rst │ ├── photom │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ └── reference_files.rst │ ├── pipeline │ │ ├── exposure_pipeline.rst │ │ ├── graphics │ │ │ ├── mosaic4_regular_grid.png │ │ │ ├── mosaic_regular_grid.png │ │ │ └── wfi_4sca_skycell.png │ │ ├── index.rst │ │ ├── main.rst │ │ └── mosaic_pipeline.rst │ ├── pipeline_installation.rst │ ├── pipeline_levels.rst │ ├── pipeline_naming_conventions.rst │ ├── pipeline_parameters.rst │ ├── pipeline_ref_files.rst │ ├── pipeline_run.rst │ ├── pipeline_static_preview.rst │ ├── pipeline_steps.rst │ ├── ramp_fitting │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ └── reference_files.rst │ ├── references_general │ │ ├── dark_reffile.inc │ │ ├── dark_selection.inc │ │ ├── dark_specific.inc │ │ ├── distortion_reffile.rst │ │ ├── distortion_selection.inc │ │ ├── dq_flags.inc │ │ ├── flat_reffile.inc │ │ ├── flat_selection.inc │ │ ├── gain_reffile.inc │ │ ├── gain_reffile.rst │ │ ├── gain_selection.inc │ │ ├── gain_selection.rst │ │ ├── index.rst │ │ ├── linearity_reffile.inc │ │ ├── linearity_reffile.rst │ │ ├── linearity_selection.inc │ │ ├── linearity_selection.rst │ │ ├── mask_reffile.inc │ │ ├── mask_selection.inc │ │ ├── photom_reffile.inc │ │ ├── photom_selection.inc │ │ ├── readnoise_reffile.inc │ │ ├── readnoise_reffile.rst │ │ ├── readnoise_selection.inc │ │ ├── readnoise_selection.rst │ │ ├── references_general.rst │ │ ├── saturation_reffile.inc │ │ └── saturation_selection.inc │ ├── refpix │ │ ├── description.rst │ │ └── index.rst │ ├── resample │ │ ├── arguments.rst │ │ ├── index.rst │ │ ├── main.rst │ │ ├── resample_step.rst │ │ └── resample_utils.rst │ ├── saturation │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ └── reference_files.rst │ ├── skymatch │ │ ├── arguments.rst │ │ ├── description.rst │ │ ├── index.rst │ │ ├── reference_files.rst │ │ └── skymatch_step.rst │ ├── source_catalog │ │ ├── arguments.rst │ │ ├── index.rst │ │ └── main.rst │ ├── stpipe │ │ ├── call_via_call.rst │ │ ├── call_via_run.rst │ │ ├── config_asdf.rst │ │ ├── devel_logging.rst │ │ ├── devel_pipeline.rst │ │ ├── devel_step.rst │ │ ├── index.rst │ │ ├── parameter_files.rst │ │ ├── user_logging.rst │ │ ├── user_pipeline.rst │ │ └── user_step.rst │ └── tweakreg │ │ ├── README.rst │ │ ├── astrometric_utils.rst │ │ ├── index.rst │ │ ├── tweakreg_examples.rst │ │ └── tweakreg_step.rst └── rtd_environment.yaml ├── pyproject.toml ├── pytest_romancal ├── __init__.py └── stpsf_plugin.py ├── requirements-dev-st.txt ├── requirements-dev-thirdparty.txt ├── romancal ├── __init__.py ├── assign_wcs │ ├── __init__.py │ ├── assign_wcs_step.py │ ├── pointing.py │ ├── tests │ │ ├── __init__.py │ │ └── test_wcs.py │ └── utils.py ├── associations │ ├── __init__.py │ ├── asn_from_list.py │ ├── association.py │ ├── association_io.py │ ├── config.py │ ├── exceptions.py │ ├── generate.py │ ├── lib │ │ ├── __init__.py │ │ ├── acid.py │ │ ├── asn_schema_jw_level3.json │ │ ├── association_rules.py │ │ ├── callback_registry.py │ │ ├── constraint.py │ │ ├── counter.py │ │ ├── decorators.py │ │ ├── diff.py │ │ ├── dms_base.py │ │ ├── ioregistry.py │ │ ├── keyvalue_registry.py │ │ ├── log_config.py │ │ ├── member.py │ │ ├── process_list.py │ │ ├── product_utils.py │ │ ├── rules_elpp_base.py │ │ ├── rules_level2.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ └── test_asn_from_list_lib.py │ │ ├── update_path.py │ │ └── utilities.py │ ├── load_as_asn.py │ ├── load_asn.py │ ├── main.py │ ├── mk_skycell_asn_from_skycell_list.py │ ├── mk_skycell_list.py │ ├── pool.py │ ├── registry.py │ ├── skycell_asn.py │ └── tests │ │ ├── __init__.py │ │ ├── data │ │ ├── __init__.py │ │ ├── asn_level2.json │ │ ├── asn_mosaic.json │ │ ├── jw93060_20150312T160130_pool.csv │ │ ├── pool_001_candidates.csv │ │ ├── pool_002_wfi_image.csv │ │ └── rules_basic.py │ │ ├── helpers.py │ │ ├── test_asn_from_list.py │ │ ├── test_constraints.py │ │ ├── test_generate.py │ │ ├── test_level2_basics.py │ │ ├── test_level2_candidates.py │ │ ├── test_mk_skycell_asn_from_skycell_list.py │ │ ├── test_mk_skycell_list.py │ │ ├── test_pool.py │ │ ├── test_registry.py │ │ ├── test_skycell_asn.py │ │ ├── test_update_path.py │ │ └── test_version.py ├── conftest.py ├── dark_current │ ├── __init__.py │ ├── dark_current_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_dark.py ├── datamodels │ ├── __init__.py │ ├── filetype.py │ ├── library.py │ └── tests │ │ ├── __init__.py │ │ ├── data │ │ ├── detectorFOV_asn.json │ │ ├── detector_asn.json │ │ ├── empty.asdf │ │ ├── empty.json │ │ ├── example_schema.json │ │ ├── fake.asdf │ │ ├── fake.json │ │ └── pluto.asdf │ │ ├── test_filetype.py │ │ └── test_library.py ├── dq_init │ ├── __init__.py │ ├── dq_init_step.py │ ├── dq_initialization.py │ └── tests │ │ ├── __init__.py │ │ └── test_dq_init.py ├── flatfield │ ├── __init__.py │ ├── flat_field.py │ ├── flat_field_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_flatfield.py ├── flux │ ├── __init__.py │ ├── flux_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_flux_step.py ├── lib │ ├── __init__.py │ ├── basic_utils.py │ ├── dqflags.py │ ├── progress.py │ ├── save_wcs.py │ ├── signal_slot.py │ ├── suffix.py │ ├── tests │ │ ├── __init__.py │ │ ├── helpers.py │ │ ├── test_basic_utils.py │ │ └── test_suffix.py │ └── wcsinfo_to_wcs.py ├── linearity │ ├── __init__.py │ ├── linearity_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_linearity.py ├── multiband_catalog │ ├── __init__.py │ ├── background.py │ ├── detection_image.py │ ├── multiband_catalog_step.py │ ├── tests │ │ ├── __init__.py │ │ └── test_multiband_catalog.py │ └── utils.py ├── outlier_detection │ ├── __init__.py │ ├── _fileio.py │ ├── outlier_detection_step.py │ ├── tests │ │ ├── __init__.py │ │ └── test_outlier_detection.py │ └── utils.py ├── patch_match │ └── tests │ │ └── __init__.py ├── photom │ ├── __init__.py │ ├── photom.py │ ├── photom_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_photom.py ├── pipeline │ ├── __init__.py │ ├── exposure_pipeline.py │ ├── mosaic_pipeline.py │ └── tests │ │ ├── __init__.py │ │ └── test_exposure_pipeline.py ├── ramp_fitting │ ├── __init__.py │ ├── ramp_fit_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_ramp_fit_cas22.py ├── refpix │ ├── __init__.py │ ├── data.py │ ├── refpix.py │ ├── refpix_step.py │ └── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── reference_utils.py │ │ ├── test_data.py │ │ ├── test_refpix.py │ │ └── test_step.py ├── regtest │ ├── __init__.py │ ├── conftest.py │ ├── regtestdata.py │ ├── resource_tracker.py │ ├── test_dark_current.py │ ├── test_elp_tvac.py │ ├── test_linearity.py │ ├── test_match_skycell.py │ ├── test_mos_pipeline.py │ ├── test_mos_skycell_pipeline.py │ ├── test_multiband_catalog.py │ ├── test_ramp_fitting.py │ ├── test_refpix.py │ ├── test_regtestdata.py │ ├── test_resample.py │ ├── test_resource_tracker.py │ ├── test_skycell.py │ ├── test_skycell_generation.py │ ├── test_source_catalog.py │ ├── test_tweakreg.py │ ├── test_wfi_dq_init.py │ ├── test_wfi_flat_field.py │ ├── test_wfi_grism_16resultants.py │ ├── test_wfi_grism_pipeline.py │ ├── test_wfi_image_16resultants.py │ ├── test_wfi_image_pipeline.py │ ├── test_wfi_photom.py │ ├── test_wfi_saturation.py │ └── util.py ├── resample │ ├── __init__.py │ ├── exptime_resampler.py │ ├── l3_wcs.py │ ├── meta_blender.py │ ├── resample.py │ ├── resample_step.py │ ├── resample_utils.py │ └── tests │ │ ├── __init__.py │ │ ├── test_resample.py │ │ └── test_resample_step.py ├── saturation │ ├── __init__.py │ ├── saturation.py │ ├── saturation_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_saturation.py ├── scripts │ ├── __init__.py │ ├── make_regtestdata.sh │ ├── patch_name_to_skycell_name.py │ ├── static_preview.py │ └── tests │ │ ├── __init__.py │ │ └── test_scripts.py ├── skycell │ ├── __init__.py │ ├── match.py │ ├── plot.py │ ├── skymap.py │ └── tests │ │ ├── data │ │ ├── L3_mosaic_asn.json │ │ ├── L3_mosaic_wcs_info_asn.json │ │ ├── L3_regtest_asn.json │ │ ├── L3_skycell_mbcat_asn.json │ │ └── skymap_subset.asdf │ │ ├── skymap_subset.py │ │ ├── test_skycell.py │ │ └── test_skycell_match.py ├── skymatch │ ├── __init__.py │ ├── skymatch_step.py │ └── tests │ │ ├── __init__.py │ │ └── test_skymatch.py ├── source_catalog │ ├── __init__.py │ ├── _wcs_helpers.py │ ├── aperture.py │ ├── background.py │ ├── daofind.py │ ├── detection.py │ ├── neighbors.py │ ├── psf.py │ ├── save_utils.py │ ├── segment.py │ ├── source_catalog.py │ ├── source_catalog_step.py │ └── tests │ │ ├── __init__.py │ │ ├── test_psf.py │ │ └── test_source_catalog.py ├── step.py ├── stpipe │ ├── __init__.py │ ├── core.py │ ├── integration.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_core.py │ │ └── test_integration.py │ └── utilities.py ├── tests │ ├── __init__.py │ ├── base_classes.py │ ├── dms_requirement_tests.json │ ├── test_dms_requirements.py │ └── test_import.py └── tweakreg │ ├── __init__.py │ ├── astrometric_utils.py │ ├── tests │ ├── __init__.py │ ├── test_astrometric_utils.py │ └── test_tweakreg.py │ └── tweakreg_step.py └── tox.ini /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | target: auto 6 | threshold: 0.1% 7 | unit: 8 | target: auto 9 | flags: 10 | - unit 11 | nightly: 12 | target: auto 13 | flags: 14 | - nightly 15 | 16 | flags: 17 | unit: 18 | carryforward: false 19 | nightly: 20 | carryforward: true 21 | 22 | comment: 23 | show_carryforward_flags: true 24 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # flake8 does not support pyproject.toml (https://github.com/PyCQA/flake8/issues/234) 2 | 3 | [flake8] 4 | select = F, W, E27, E70, E71, E101, E111, E112, E113, E201, E202, E221, E222, E241, E401, E402, E501, E704, E722 5 | max-line-length = 88 6 | exclude = 7 | docs, 8 | .tox, 9 | .eggs, 10 | build 11 | ignore = E203, W503, W504 12 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # automatically requests pull request reviews for files matching the given pattern; the last match takes precendence 2 | 3 | # maintainers own everything 4 | * @spacetelescope/romancal-maintainers 5 | 6 | # developers can own some specific paths 7 | /.github/ @spacetelescope/romancal-developers # CI files 8 | /* @spacetelescope/romancal-developers # top level files only 9 | /changes/** @spacetelescope/romancal-developers # changelogs 10 | /docs/** @spacetelescope/romancal-developers # docs 11 | /romancal/regtest/** @spacetelescope/romancal-developers # regtest code 12 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | target-branch: "main" 6 | schedule: 7 | interval: monthly 8 | open-pull-requests-limit: 10 9 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | release: 5 | types: [ released ] 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | jobs: 10 | build: 11 | uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 12 | with: 13 | upload_to_pypi: ${{ (github.event_name == 'release') && (github.event.action == 'released') }} 14 | secrets: 15 | pypi_token: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }} 16 | -------------------------------------------------------------------------------- /.github/workflows/changelog.yml: -------------------------------------------------------------------------------- 1 | name: changelog 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - labeled 7 | - unlabeled 8 | - opened 9 | - synchronize 10 | - reopened 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | check: 18 | if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog-entry-needed') }} 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/setup-python@v5 22 | with: 23 | python-version: 3 24 | - uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | - run: pip install . 28 | - run: pip install towncrier 29 | - run: towncrier check 30 | - run: towncrier build --draft | grep -P '#${{ github.event.number }}' 31 | prevent_manually_editing_changlog: 32 | if: ${{ !contains(github.event.pull_request.labels.*.name, 'allow-manual-changelog-edit') }} 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/checkout@v4 36 | with: 37 | fetch-depth: 0 38 | - name: prevent direct changes to `CHANGES.rst` (write a towncrier fragment in `changes/` instead; you can override this with the `allow-manual-changelog-edit` label) 39 | run: git diff HEAD ${{ github.event.pull_request.base.sha }} --no-patch --exit-code CHANGES.rst 40 | -------------------------------------------------------------------------------- /.github/workflows/contexts.yml: -------------------------------------------------------------------------------- 1 | name: contexts 2 | 3 | on: 4 | workflow_call: 5 | outputs: 6 | roman: 7 | value: ${{ jobs.contexts.outputs.roman }} 8 | workflow_dispatch: 9 | 10 | jobs: 11 | contexts: 12 | name: retrieve latest CRDS contexts 13 | runs-on: ubuntu-latest 14 | outputs: 15 | roman: ${{ steps.roman_crds_context.outputs.pmap }} 16 | steps: 17 | - id: roman_crds_context 18 | env: 19 | OBSERVATORY: roman 20 | CRDS_SERVER_URL: https://roman-crds.stsci.edu 21 | run: > 22 | echo "pmap=$( 23 | curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}", null], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ --retry 8 | 24 | python -c "import sys, json; print(json.load(sys.stdin)['result'])" 25 | )" >> $GITHUB_OUTPUT 26 | - run: if [[ ! -z "${{ steps.roman_crds_context.outputs.pmap }}" ]]; then echo ${{ steps.roman_crds_context.outputs.pmap }}; else exit 1; fi 27 | -------------------------------------------------------------------------------- /.github/workflows/data.yml: -------------------------------------------------------------------------------- 1 | name: download and cache data 2 | 3 | on: 4 | schedule: 5 | - cron: "42 4 * * 3" 6 | workflow_dispatch: 7 | inputs: 8 | stpsf_minimal: 9 | description: minimal STPSF dataset 10 | type: boolean 11 | required: false 12 | default: true 13 | 14 | jobs: 15 | download_stpsf_data: 16 | uses: ./.github/workflows/download_data.yml 17 | with: 18 | minimal: ${{ github.event_name != 'workflow_dispatch' && true || inputs.stpsf_minimal }} 19 | move_data_cache_path: 20 | needs: [ download_stpsf_data ] 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: retrieve cached STPSF data 24 | uses: actions/cache/restore@v4 25 | with: 26 | path: ${{ needs.download_stpsf_data.outputs.cache_path }} 27 | key: ${{ needs.download_stpsf_data.outputs.cache_key }} 28 | - run: mkdir -p /tmp/data/ 29 | - run: mv ${{ needs.download_stpsf_data.outputs.cache_path }}/stpsf-data/ /tmp/data/ 30 | - run: echo STPSF_PATH=/tmp/data/stpsf-data/ >> $GITHUB_ENV 31 | # save a new cache to the generalized data directory 32 | - name: save a single combined data cache 33 | uses: actions/cache/save@v4 34 | with: 35 | path: /tmp/data/ 36 | key: ${{ needs.download_stpsf_data.outputs.cache_key }} 37 | -------------------------------------------------------------------------------- /.github/workflows/label_pull_request.yml: -------------------------------------------------------------------------------- 1 | name: label pull request 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - synchronize 8 | 9 | jobs: 10 | triage: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/labeler@v5 14 | if: github.event_name == 'pull_request_target' || github.event_name == 'pull_request' 15 | with: 16 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled files 2 | *.py[cod] 3 | *.a 4 | *.o 5 | *.so 6 | __pycache__ 7 | 8 | # Ignore .c files by default to avoid including generated code. If you want to 9 | # add a non-generated .c extension, use `git add -f filename.c`. 10 | *.c 11 | 12 | # Other generated files 13 | */version.py 14 | */cython_version.py 15 | htmlcov 16 | .coverage 17 | .cover* 18 | MANIFEST 19 | .ipynb_checkpoints 20 | 21 | # Sphinx 22 | docs/api 23 | docs/_build 24 | 25 | # Eclipse editor project files 26 | .project 27 | .pydevproject 28 | .settings 29 | 30 | # Pycharm editor project files 31 | .idea 32 | 33 | # Floobits project files 34 | .floo 35 | .flooignore 36 | 37 | # Visual Studio Code project files 38 | .vscode 39 | 40 | # Packages/installer info 41 | *.egg 42 | *.egg-info 43 | dist 44 | build 45 | eggs 46 | .eggs 47 | parts 48 | bin 49 | var 50 | sdist 51 | develop-eggs 52 | .installed.cfg 53 | distribute-*.tar.gz 54 | 55 | # emacs save files 56 | \#* 57 | .#* 58 | 59 | # Other 60 | .cache 61 | .tox 62 | .*.sw[op] 63 | *~ 64 | .project 65 | .pydevproject 66 | .settings 67 | pip-wheel-metadata/ 68 | 69 | # Mac OSX 70 | .DS_Store 71 | 72 | .python-version 73 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/.gitmodules -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ".*\\.asdf$" 2 | 3 | repos: 4 | 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: v5.0.0 7 | hooks: 8 | - id: check-added-large-files 9 | - id: check-ast 10 | - id: check-case-conflict 11 | - id: check-yaml 12 | args: ["--unsafe"] 13 | - id: check-toml 14 | - id: check-merge-conflict 15 | - id: check-symlinks 16 | - id: debug-statements 17 | - id: detect-private-key 18 | - id: end-of-file-fixer 19 | - id: trailing-whitespace 20 | 21 | - repo: https://github.com/pre-commit/pygrep-hooks 22 | rev: v1.10.0 23 | hooks: 24 | - id: rst-directive-colons 25 | - id: rst-inline-touching-normal 26 | - id: text-unicode-replacement-char 27 | 28 | - repo: https://github.com/astral-sh/ruff-pre-commit 29 | rev: 'v0.11.12' 30 | hooks: 31 | - id: ruff 32 | args: ["--fix", "--show-fixes"] 33 | - id: ruff-format 34 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | builder: html 11 | configuration: docs/conf.py 12 | fail_on_warning: true 13 | 14 | # Optionally build your docs in additional formats such as PDF and ePub 15 | formats: 16 | - htmlzip 17 | - pdf 18 | 19 | build: 20 | os: ubuntu-22.04 21 | tools: 22 | python: mambaforge-4.10 23 | jobs: 24 | post_checkout: 25 | # Download and uncompress the binary 26 | # https://git-lfs.github.com/ 27 | - wget https://github.com/git-lfs/git-lfs/releases/download/v3.1.4/git-lfs-linux-amd64-v3.1.4.tar.gz 28 | - tar xvfz git-lfs-linux-amd64-v3.1.4.tar.gz 29 | # Modify LFS config paths to point where git-lfs binary was downloaded 30 | - git config filter.lfs.process "`pwd`/git-lfs filter-process" 31 | - git config filter.lfs.smudge "`pwd`/git-lfs smudge -- %f" 32 | - git config filter.lfs.clean "`pwd`/git-lfs clean -- %f" 33 | # Make LFS available in current repository 34 | - ./git-lfs install 35 | # Download content from remote 36 | - ./git-lfs fetch 37 | # Make local files to have the real content on them 38 | - ./git-lfs checkout 39 | post_install: 40 | - towncrier build --keep 41 | 42 | conda: 43 | environment: docs/rtd_environment.yaml 44 | 45 | # Optionally set the version of Python and requirements required to build your docs 46 | python: 47 | install: 48 | - method: pip 49 | path: . 50 | extra_requirements: 51 | - docs 52 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Please open a new issue or new pull request for bugs, feedback, or new features you would like to see. If there is an issue you would like to work on, please leave a comment and we will be happy to assist. New contributions and contributors are very welcome! 2 | 3 | The main development work is done on the "main" branch. The "stable" branch is protected and used for official releases. The rest of the branches are for release maintenance and should not be used normally. Unless otherwise told by a maintainer, pull request should be made and submitted to the "main" branch. 4 | 5 | New to github or open source projects? If you are unsure about where to start or haven't used github before, please feel free to contact the package maintainers. 6 | 7 | Feedback and feature requests? Is there something missing you would like to see? Please open an issue or send an email to the maintainers. This package follows the Spacetelescope `Code of Conduct`_ strives to provide a welcoming community to all of our users and contributors. 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2010 Association of Universities for Research in Astronomy (AURA) 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following 11 | disclaimer in the documentation and/or other materials provided 12 | with the distribution. 13 | 14 | 3. The name of AURA and its representatives may not be used to 15 | endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY AURA ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL AURA BE LIABLE FOR ANY DIRECT, INDIRECT, 22 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 24 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 26 | TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 27 | USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 28 | DAMAGE. 29 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include CHANGES.rst 3 | include LICENSE 4 | include pyproject.toml 5 | 6 | recursive-include docs * 7 | recursive-include scripts * 8 | 9 | recursive-include romancal *.fits 10 | recursive-include romancal *.inc 11 | recursive-include romancal *.json 12 | recursive-include romancal *.txt 13 | recursive-include romancal *.cfg 14 | recursive-include romancal *.csv 15 | recursive-include romancal *.yaml 16 | recursive-include romancal *.asdf 17 | 18 | prune build 19 | prune docs/_build 20 | prune docs/api 21 | 22 | global-exclude *.pyc *.o 23 | -------------------------------------------------------------------------------- /changes/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/changes/.gitkeep -------------------------------------------------------------------------------- /changes/1438.flatfield.rst: -------------------------------------------------------------------------------- 1 | Make var_flat optional. 2 | -------------------------------------------------------------------------------- /changes/1760.general.rst: -------------------------------------------------------------------------------- 1 | Replace uses of ``roman_datamodels.maker_utils`` with ``DataModel.create_fake_data``. 2 | -------------------------------------------------------------------------------- /changes/1764.assign_wcs.rst: -------------------------------------------------------------------------------- 1 | remove unused function ``wcs_from_footprints`` 2 | -------------------------------------------------------------------------------- /changes/1776.mosaic_pipeline.rst: -------------------------------------------------------------------------------- 1 | Updated source_catalog class alias in the mosaic pipeline. 2 | -------------------------------------------------------------------------------- /docs/_static/roman_logo_black_w200px.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1d3b1cad9d07c9dd96ceac84601cb7e4a15a62f02f1c7e39216a53f70c085679 3 | size 47381 4 | -------------------------------------------------------------------------------- /docs/_static/roman_logo_white_w100px.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8c0c9f637a271f1f358551a37232cfe5e83964b63b85da8a4ccca1afa7325ade 3 | size 18255 4 | -------------------------------------------------------------------------------- /docs/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | from contextlib import contextmanager 3 | 4 | import pytest 5 | 6 | # chdir contextmanager was added with Python 3.11 7 | try: 8 | from contextlib import chdir 9 | except ImportError: 10 | 11 | @contextmanager 12 | def chdir(path): 13 | old_path = os.getcwd() 14 | 15 | try: 16 | os.chdir(path) 17 | yield 18 | finally: 19 | os.chdir(old_path) 20 | 21 | 22 | @pytest.fixture(autouse=True) 23 | def _docdir(request): 24 | # Trigger ONLY for doctestplus. 25 | doctest_plugin = request.config.pluginmanager.getplugin("doctestplus") 26 | if isinstance(request.node.parent, doctest_plugin._doctest_textfile_item_cls): 27 | tmp_path = request.getfixturevalue("tmp_path") 28 | with chdir(tmp_path): 29 | yield 30 | else: 31 | yield 32 | -------------------------------------------------------------------------------- /docs/images/r0000501001001001001_0001_wfi01_cal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/images/r0000501001001001001_0001_wfi01_cal.png -------------------------------------------------------------------------------- /docs/images/r0000501001001001001_0001_wfi01_cal_thumb.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:316405217b1e91886b58a0da60a1512fd37dafb91cd4d651cb2ce4fb41b46a64 3 | size 70887 4 | -------------------------------------------------------------------------------- /docs/images/wfi_array.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/images/wfi_array.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. romancal documentation master file 2 | You can adapt this file completely to your liking, but it should at least 3 | contain the root `toctree` directive. 4 | 5 | :ref:`genindex` | :ref:`modindex` 6 | 7 | .. _roman-pipeline-doc-index: 8 | 9 | ============================================== 10 | The Roman Space Telescope Calibration Pipeline 11 | ============================================== 12 | 13 | .. image:: _static/roman_logo_black_w200px.png 14 | :align: center 15 | :alt: Nancy Roman Space Telescope 16 | 17 | **Version**: |release| 18 | 19 | Welcome to the documentation for the Roman calibration software, 20 | `romancal `__. 21 | This package contains the Python software suite for the 22 | Roman Space Telescope (RST) calibration pipeline, which processes data 23 | from the Roman Wide-Field Instrument (WFI) by applying various corrections 24 | to produce science-ready, calibrated output products including fully calibrated 25 | individual exposures as well as high-level data products (mosaics, 26 | catalogs, etc.). The tools in this package allow users to run and 27 | configure the pipeline to custom process their Roman data. 28 | Additionally, the romancal package contains the interface to 29 | Roman datamodels, the recommended method of reading and writing 30 | Roman data files in Python. 31 | 32 | 33 | If you have questions or concerns regarding the software, please contact the Roman Help 34 | desk at `Roman Help Desk `_. 35 | 36 | -------------------------------- 37 | 38 | .. include:: roman/introduction.rst 39 | 40 | .. toctree:: 41 | :caption: RomanCal Pipeline 42 | :maxdepth: 2 43 | 44 | roman/pipeline_installation.rst 45 | roman/pipeline_levels.rst 46 | roman/pipeline_run.rst 47 | roman/pipeline_steps.rst 48 | roman/pipeline_ref_files.rst 49 | roman/pipeline_parameters.rst 50 | roman/pipeline_naming_conventions.rst 51 | roman/error_propagation/index.rst 52 | 53 | .. toctree:: 54 | :maxdepth: 2 55 | :caption: Data Products Documentation 56 | 57 | roman/data_products/index.rst 58 | 59 | .. toctree:: 60 | :caption: RomanCal Package Index 61 | :maxdepth: 3 62 | 63 | roman/package_index.rst 64 | 65 | .. toctree:: 66 | :caption: Datamodels 67 | :maxdepth: 3 68 | 69 | roman/datamodels/index.rst 70 | 71 | .. toctree:: 72 | :caption: Additional Information 73 | :maxdepth: 1 74 | 75 | roman/pipeline_static_preview.rst 76 | roman/changes.rst 77 | -------------------------------------------------------------------------------- /docs/roman/assign_wcs/index.rst: -------------------------------------------------------------------------------- 1 | .. _assign_wcs_step: 2 | 3 | ========== 4 | Assign WCS 5 | ========== 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | 10 | main.rst 11 | 12 | .. automodapi:: romancal.assign_wcs 13 | -------------------------------------------------------------------------------- /docs/roman/associations/asn_from_list.rst: -------------------------------------------------------------------------------- 1 | .. _asn-from-list: 2 | 3 | asn_from_list 4 | ============= 5 | 6 | Create an association using either the command line tool 7 | ``asn_from_list`` or through the Python API using 8 | :func:`romancal.associations.asn_from_list.asn_from_list` 9 | 10 | 11 | Associations 12 | ^^^^^^^^^^^^ 13 | 14 | Refer to TBD for a full description of associations. 15 | 16 | To create an association, use the following command: 17 | 18 | .. code-block:: python 19 | 20 | import romancal.associations.asn_from_list as asn_from_list 21 | product_name = 'test_product' 22 | items = {'r0000101001001001001_0001_wfi01_uncal.asdf': 'science', 'r0000101001001001001_3_0001_wfi01_uncal.asdf': 'guide_window', 'c': 'somethingelse'} 23 | asn = asn_from_list.asn_from_list([(item, type_) for item, type_ in items.items()], product_name=product_name, with_exptype=True) 24 | asn['asn_rule'] 25 | 'DMS_ELPP_Base' 26 | 27 | 28 | an example product that has both a science and guide window exposures 29 | would look like the following:: 30 | 31 | asn['products'] 32 | [ { 'members': [ { 'expname': 'r0000101001001001001_0001_wfi01_uncal.asdf', 33 | 'exptype': 'science'}, 34 | { 'expname': 'r0000101001001001001_3_0001_wfi01_uncal.asdf', 35 | 'exptype': 'guide_window'}, 36 | {'expname': 'c', 'exptype': 'somethingelse'}], 37 | 'name': 'test_product'}] 38 | 39 | 40 | To create a association with all the detectors for a given exposure from the command line, 41 | 42 | .. code-block:: python 43 | 44 | asn_from_list -o detector_asn.json --product-name r0000101001001001001_0001_wfi data/*_cal.asdf 45 | 46 | where the individual calibrated detector files are in a data subdirectory. 47 | 48 | In addition you can also specify a target for the association file, 49 | 50 | .. code-block:: python 51 | 52 | asn_from_list -o level3_mosaic_asn.json --product-name level3_mosaic --target 270p65x48y69 data/*_cal.asdf 53 | -------------------------------------------------------------------------------- /docs/roman/associations/commands.rst: -------------------------------------------------------------------------------- 1 | .. _association-commands: 2 | 3 | Association Commands 4 | ==================== 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | asn_from_list.rst 10 | skycell_asn.rst 11 | mk_skycell_list.rst 12 | mk_skycell_asn_from_skycell_list.rst 13 | -------------------------------------------------------------------------------- /docs/roman/associations/design.rst: -------------------------------------------------------------------------------- 1 | .. _design: 2 | 3 | 4 | Association Design 5 | ================== 6 | 7 | .. _figure-association-generator-overview: 8 | 9 | .. figure:: graphics/overview.png 10 | :scale: 50% 11 | 12 | Association Generator Overview 13 | 14 | As introduced in the :ref:`asn-overview`, the figure above shows all the 15 | major players used in generating associations. Since this section will 16 | be describing the code design, the figure below is the overview but 17 | using the class names involved. 18 | 19 | .. _figure-class-overview: 20 | 21 | .. figure:: graphics/overview_classes.png 22 | :scale: 50% 23 | 24 | Association Class Relationship overview 25 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/generator_flow_concept.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:37cf00edf44b9beec4c0460a5f1b9136bcba9e06881c147fdea759aec3bac88e 3 | size 94692 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/generator_list_processing.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9e9c0bd68ecaf6af1574aaac12f1d80531e079756b1f04e3a9d5a2cebad21a5e 3 | size 87417 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/level3_rule_inheritance.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:90eb9a6f5aa9efface4fb2fc1928f11b917f63ed49933fcd1239ed4795c31c5f 3 | size 54190 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/overview.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5a7cbf892099d6c8f6a44701c05a4e94e078c1d3d6391927ebdba0796d91e84e 3 | size 44201 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/overview_classes.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:9666711553d4f0f1dadb8e8d257c9e809c6a5b4b84c3748f374d481dba840531 3 | size 47753 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/rule_sets.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:307618714b40af1d9c793f3898f7c880e79ee2fd558e9e42b175333484cf0edc 3 | size 36972 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/rule_to_association.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f52e1e5832693d31db247ab5325316d2e571c9c4f58014c21e8dff8465a0155f 3 | size 41433 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/sources/associations.monopic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/associations/graphics/sources/associations.monopic -------------------------------------------------------------------------------- /docs/roman/associations/graphics/sources/workflow-generic.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/associations/graphics/sources/workflow-generic.odg -------------------------------------------------------------------------------- /docs/roman/associations/graphics/sources/workflow-wfss.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/associations/graphics/sources/workflow-wfss.odg -------------------------------------------------------------------------------- /docs/roman/associations/graphics/workflow-generic.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/associations/graphics/workflow-generic.pdf -------------------------------------------------------------------------------- /docs/roman/associations/graphics/workflow-generic.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:128a567b20058305149e0fc02dd070d2c1597cfad5bfa8a28834475d6976482e 3 | size 227010 4 | -------------------------------------------------------------------------------- /docs/roman/associations/graphics/workflow-wfss.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/associations/graphics/workflow-wfss.pdf -------------------------------------------------------------------------------- /docs/roman/associations/graphics/workflow-wfss.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a93509d78654610912350250cac6f3c4b21d92d80b70e1840cb03e77d46da6aa 3 | size 228851 4 | -------------------------------------------------------------------------------- /docs/roman/associations/index.rst: -------------------------------------------------------------------------------- 1 | .. _associations: 2 | 3 | ============= 4 | Associations 5 | ============= 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | overview.rst 11 | 12 | Roman Associations 13 | ================== 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | 18 | roman_conventions.rst 19 | technote_sdp_workflow.rst 20 | 21 | Design 22 | ====== 23 | .. toctree:: 24 | :maxdepth: 2 25 | 26 | design.rst 27 | 28 | Reference 29 | ========= 30 | .. toctree:: 31 | :maxdepth: 2 32 | 33 | commands.rst 34 | -------------------------------------------------------------------------------- /docs/roman/associations/mk_skycell_asn_from_skycell_list.rst: -------------------------------------------------------------------------------- 1 | .. _mk_skycell_asn_from_skycell_list: 2 | 3 | mk_skycell_asn_from_skycell_list 4 | ================================= 5 | 6 | The function reads a list of ascii match files that were created using the ``mk_skycell_list`` 7 | program and generates a list of associations for all the skycells that touch the given exposure. 8 | using either the command line tool 9 | ``mk_skycell_asn_from_skycell_list`` or through the Python API using 10 | :func:`romancal.associations.mk_skycell_asn_from_skycell_list.mk_skycell_asn_from_skycell_list` 11 | 12 | Association Files 13 | ^^^^^^^^^^^^^^^^^^^ 14 | 15 | To create the list of association files, use the following command: 16 | 17 | .. code-block:: python 18 | 19 | mk_skycell_asn_from_skycell_list file_list 20 | 21 | where the individual calibrated detector files are in the current directory and the file_list is a 22 | list of the match files that will be used to generate the list of associations. 23 | To get a complete list of options you can run the command with the 24 | \-h option 25 | 26 | .. code-block:: python 27 | 28 | mk_skycell_asn_from_skycell_list -h 29 | usage: mk_skycell_asn_from_skycell_list *.match 30 | 31 | Create level 3 associations from a list of match files 32 | 33 | positional arguments: 34 | filelist A list of match files to generate level 3 asn's 35 | 36 | options: 37 | -h, --help show this help message and exit 38 | --release-product RELEASE_PRODUCT 39 | The release product when creating the association 40 | --optical_element OPTICAL_ELEMENT 41 | The optical element used for the visit 42 | 43 | The match file names are based on the input calibrated file names with the .asdf extention replaced 44 | with .match and the match file contains the name of the calibrated exposure file and a list of index 45 | values of the skycells that touch the input exposure. 46 | 47 | .. code-block:: text 48 | 49 | mk_skycell_asn_from_skycell_list r0099101001001003001_*_cal.match 50 | 51 | Where the wildcard selects all the exposures for visit 001 and generates associations based on the 52 | list of skycell matches for each of the exposures. There will be one association for each unique 53 | skycell index in the match files. 54 | -------------------------------------------------------------------------------- /docs/roman/associations/mk_skycell_list.rst: -------------------------------------------------------------------------------- 1 | .. _mk_skycell_list: 2 | 3 | mk_skycell_list 4 | =============== 5 | 6 | Creates a list of files that contain the calibrated file name with the 7 | skycell index for all the skycells that touch the given exposure in 8 | ascii format. These are created using either the command line tool 9 | ``mk_skycell_list`` or through the Python API using 10 | :func:`romancal.associations.mk_skycell_list.mk_skycell_list` 11 | 12 | 13 | Match Files 14 | ^^^^^^^^^^^ 15 | 16 | To create the list of match files, use the following command: 17 | 18 | .. code-block:: python 19 | 20 | mk_skycell_list file_list 21 | 22 | where the individual calibrated detector files are in a directory. 23 | To get a complete list of options you can run the command with the 24 | \-h option 25 | 26 | .. code-block:: python 27 | 28 | mk_skycell_list -h 29 | usage: mk_skycell_list *_cal.asdf 30 | 31 | Create list of skycells from a level 2 file or files 32 | 33 | positional arguments: 34 | filelist Input file list to generate a list of skycells 35 | 36 | options: 37 | -h, --help show this help message and exit 38 | --output_dir OUTPUT_DIR 39 | The optional directory to write the list of skycells 40 | 41 | The match file names are based on the input Level 2 calibrated file names with the .asdf extention replaced 42 | with .match and the match file contains the name of the calibrated exposure file and a list of index 43 | values of the skycells that touch the input exposure. The output directory for the match files is 44 | determined from the input directory of the calibrated exposures. You can override this by using 45 | the flag to set the output directory. 46 | 47 | As an example to create a list of match files for all the exposures in a visit the command is 48 | 49 | .. code-block:: text 50 | 51 | mk_skycell_list r0099101001001003001_*_cal.asdf 52 | 53 | Where the wildcard selects all the exposures for visit 001 and generates a list of skycell matches for each 54 | of the exposures. There will be one match file for each input exposure file. For the example above this 55 | will create match files named, with the `*` replaced by the exposure number of the file, 56 | 57 | .. code-block:: text 58 | 59 | r0099101001001003001_*_cal.match 60 | -------------------------------------------------------------------------------- /docs/roman/associations/technote_sdp_workflow.rst: -------------------------------------------------------------------------------- 1 | .. _sdp-workflow: 2 | 3 | ================================ 4 | Science Data Processing Workflow 5 | ================================ 6 | 7 | General Workflow for Generating Association Products 8 | ==================================================== 9 | 10 | See :ref:`asn-associations-and-roman` for an overview of how Roman uses 11 | associations. This document describes how associations are used by the 12 | ground processing system to execute the stage 2 and stage 3 pipelines. 13 | -------------------------------------------------------------------------------- /docs/roman/changes.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: roman 2 | 3 | *********** 4 | Change Log 5 | *********** 6 | 7 | .. include:: ../../CHANGES.rst 8 | -------------------------------------------------------------------------------- /docs/roman/dark_current/arguments.rst: -------------------------------------------------------------------------------- 1 | Step Arguments 2 | ============== 3 | 4 | The dark current step has one step-specific argument: 5 | 6 | * ``--dark_output`` 7 | 8 | If the ``dark_output`` argument is given with a filename for its value, 9 | the frame-averaged dark data that are created within the step will be 10 | saved to that file. 11 | -------------------------------------------------------------------------------- /docs/roman/dark_current/description.rst: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Assumptions 5 | ----------- 6 | 7 | It is assumed that the input science data have had the zero group (or 8 | bias) subtracted. Accordingly, the dark reference data 9 | should have their own group zero subtracted from all groups. 10 | 11 | Algorithm 12 | --------- 13 | 14 | The dark current step removes dark current from a Roman exposure by subtracting 15 | dark current data stored in a dark reference file. 16 | 17 | The current implementation uses dark reference files that are matched to the 18 | MA table entry in the exposure metadata. Note that the data reference file 19 | for a science group (SCI) is named `data`. The dark data are then subtracted, 20 | group-by-group, from the science exposure groups, in which 21 | each SCI group of the dark data is subtracted from the corresponding SCI 22 | group of the science data. 23 | 24 | The ERR arrays of the science data are not modified. 25 | 26 | The DQ flags from the dark reference file are propagated into science 27 | exposure PIXELDQ array using a bitwise OR operation. 28 | 29 | Upon successful completion of the dark subtraction the cal_step attribute is 30 | set to "COMPLETE". 31 | 32 | Special Handling 33 | ++++++++++++++++ 34 | 35 | Any pixel values in the dark reference data that are set to NaN will have their 36 | values reset to zero before being subtracted from the science data, which 37 | will effectively skip the dark subtraction operation for those pixels. 38 | -------------------------------------------------------------------------------- /docs/roman/dark_current/index.rst: -------------------------------------------------------------------------------- 1 | .. _dark_current_step: 2 | 3 | ======================== 4 | Dark Current Subtraction 5 | ======================== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | 15 | .. automodapi:: romancal.dark_current 16 | -------------------------------------------------------------------------------- /docs/roman/dark_current/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference File 2 | ============== 3 | The ``dark`` step uses a DARK reference file. 4 | 5 | .. include:: ../references_general/dark_reffile.inc 6 | -------------------------------------------------------------------------------- /docs/roman/data_products/file_naming.rst: -------------------------------------------------------------------------------- 1 | .. _file_naming_schemes: 2 | 3 | File Naming Schemes 4 | ------------------- 5 | 6 | .. _exp_file_names: 7 | 8 | Exposure file names 9 | ^^^^^^^^^^^^^^^^^^^ 10 | The names of the exposure level data are constructed with information from the 11 | science header of the exposure, allowing users to map it to the observation in their corresponding 12 | APT files. The ASDF file naming scheme for the Exposure products is:: 13 | 14 | r___.asdf 15 | 16 | where 17 | 18 | - ppppp: program ID number 19 | - cc: Execution Plan number 20 | - aaa: Pass Number (within execution plan) 21 | - sss: Segment Number (within pass) 22 | - ooo: observation number 23 | - vvv: visit number 24 | - gg: visit group 25 | - s: sequence ID (1=prime, >1 parallel) 26 | - aa: activity number (base 36) 27 | - eeee: exposure number 28 | - detector: detector name (e.g. wfi01, wfi02, ...) 29 | - prodType: product type identifier (e.g. 'uncal', 'cal') 30 | 31 | The standard for the pipeline are uncal and cal, for the input products and resulting 32 | calibrated product. There are optional suffixes for intermediate products that are not routinely 33 | produced by the pipeline and are based of the processing level and can include dqinit, saturation, 34 | linearity, jump, darkcurrent, rampfit, assignwcs, flat, and photom. 35 | 36 | An example Exposure Level 1 upcalibrated ASDF file name is:: 37 | 38 | r0000101001001001001_0001_wfi01_uncal.asdf 39 | 40 | An example Exposure Level 2 product ASDF file name is:: 41 | 42 | r0000101001001001001_0001_wfi01_cal.asdf 43 | -------------------------------------------------------------------------------- /docs/roman/data_products/index.rst: -------------------------------------------------------------------------------- 1 | .. _data_products: 2 | 3 | Data Products Information 4 | ========================= 5 | 6 | .. toctree:: 7 | :maxdepth: 3 8 | 9 | stages.rst 10 | file_naming.rst 11 | product_types.rst 12 | science_products.rst 13 | -------------------------------------------------------------------------------- /docs/roman/datamodels/index.rst: -------------------------------------------------------------------------------- 1 | .. _data-models: 2 | 3 | .. toctree:: 4 | :maxdepth: 1 5 | 6 | models.rst 7 | metadata.rst 8 | datamodels_asdf.rst 9 | library.rst 10 | -------------------------------------------------------------------------------- /docs/roman/datamodels/library.rst: -------------------------------------------------------------------------------- 1 | .. _library: 2 | 3 | ============== 4 | ModelLibrary 5 | ============== 6 | 7 | .. automodule:: romancal.datamodels.library 8 | :members: 9 | :undoc-members: 10 | -------------------------------------------------------------------------------- /docs/roman/dq_init/arguments.rst: -------------------------------------------------------------------------------- 1 | Step Arguments 2 | ============== 3 | The Data Quality Initialization step has no step-specific arguments. 4 | -------------------------------------------------------------------------------- /docs/roman/dq_init/index.rst: -------------------------------------------------------------------------------- 1 | .. _dq_init_step: 2 | 3 | ================================ 4 | Data Quality (DQ) Initialization 5 | ================================ 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | 15 | .. automodapi:: romancal.dq_init 16 | -------------------------------------------------------------------------------- /docs/roman/dq_init/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference Files 2 | =============== 3 | The ``dq_init`` step uses a MASK reference file. 4 | 5 | A list of the allowed DQ values are: 6 | 7 | .. include:: ../references_general/dq_flags.inc 8 | 9 | .. include:: ../references_general/mask_reffile.inc 10 | -------------------------------------------------------------------------------- /docs/roman/error_propagation/index.rst: -------------------------------------------------------------------------------- 1 | .. _error_propagation: 2 | 3 | ================= 4 | Error Propagation 5 | ================= 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | main.rst 11 | -------------------------------------------------------------------------------- /docs/roman/flatfield/index.rst: -------------------------------------------------------------------------------- 1 | .. _flatfield_step: 2 | 3 | ==================== 4 | Flatfield Correction 5 | ==================== 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | main.rst 10 | reference_files.rst 11 | 12 | .. automodapi:: romancal.flatfield 13 | -------------------------------------------------------------------------------- /docs/roman/flatfield/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference Files 2 | =============== 3 | The ``flat_field`` step uses a FLAT reference file. 4 | 5 | .. include:: ../references_general/flat_reffile.inc 6 | -------------------------------------------------------------------------------- /docs/roman/flux/arguments.rst: -------------------------------------------------------------------------------- 1 | .. _flux_step_args: 2 | 3 | Step Arguments 4 | ============== 5 | 6 | The ``flux`` step takes no step-specific arguments. 7 | -------------------------------------------------------------------------------- /docs/roman/flux/flux_step.rst: -------------------------------------------------------------------------------- 1 | .. flux_step_: 2 | 3 | Python Step Interface: FluxStep() 4 | ================================= 5 | 6 | .. automodapi:: romancal.flux.flux_step 7 | -------------------------------------------------------------------------------- /docs/roman/flux/index.rst: -------------------------------------------------------------------------------- 1 | .. _flux_step: 2 | 3 | ==== 4 | Flux 5 | ==== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | main.rst 11 | arguments.rst 12 | flux_step.rst 13 | 14 | .. automodapi:: romancal.flux 15 | -------------------------------------------------------------------------------- /docs/roman/flux/main.rst: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | :Classes: `romancal.flux.FluxStep` 5 | :Alias: flux 6 | 7 | This step will apply the flux scale factor, as defined in the meta of the input 8 | ImageModel. The step will check units and not re-apply the correction, simply 9 | returning the original data. 10 | 11 | The ``flux`` step can take: 12 | 13 | * a single 2D input image (in the format of either a string with the full 14 | path and filename of an ASDF file or a Roman 15 | Datamodel/:py:class:`~romancal.datamodels.library.ModelLibrary`); 16 | * an association table (in JSON format). 17 | 18 | 19 | This step takes no other parameters than the standard ``stpipe`` parameters. 20 | 21 | The output product are ImageModels, but with the flux scale applied to all the data and variance arrays. 22 | 23 | 24 | Flux Application 25 | ---------------- 26 | If the input data is in :math:`DN / second`, the flux scale factor, as 27 | found in the meta ``meta.photometry.conversion_megajanskys``, is simply 28 | multiplied to the ``data`` and ``err`` arrays. The square of the scale factor is 29 | multiplied to all the variance arrays. The resultant units is in :math:`MJy/sr`. 30 | 31 | .. math:: 32 | DATA &= DATA * SCALE 33 | 34 | ERR &= ERR * SCALE 35 | 36 | VAR_{rnoise} &= VAR_{rnoise} * SCALE ^ 2 37 | 38 | VAR_{poisson} &= VAR_{poisson} * SCALE ^2 39 | 40 | VAR_{flat} &= VAR_{flat} * SCALE ^2 41 | -------------------------------------------------------------------------------- /docs/roman/includes/standard_keywords.inc: -------------------------------------------------------------------------------- 1 | 2 | Standard ASDF metadata 3 | ++++++++++++++++++++++ 4 | The following table lists the attributes that are *required* to be present 5 | in all reference files. The first column shows the attribute in the ASDF 6 | reference file headers, which is the same as the name within the data 7 | model meta tree (second column). The second column gives the roman 8 | data model name for each attribute, which is useful when using data 9 | models in creating and populating a new reference file. 10 | 11 | =========== =============================== 12 | Attribute Fully Qualified Path 13 | =========== =============================== 14 | author model.meta.author 15 | model_type model.meta.model_type 16 | date model.meta.date 17 | description model.meta.description 18 | instrument model.meta.instrument.name 19 | reftype model.meta.reftype 20 | telescope model.meta.telescope 21 | useafter model.meta.useafter 22 | =========== =============================== 23 | 24 | 25 | **NOTE:** More information on standard required attributes can be found here: 26 | :ref:`Standard ASDF metadata` 27 | -------------------------------------------------------------------------------- /docs/roman/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | Welcome to the documentation for `romancal `__. 5 | This package contains the calibration software for the WFI instrument on the Nancy Grace Roman 6 | Space Telescope. 7 | 8 | The romancal package processes data by applying various corrections to produce 9 | science-ready output products, including fully calibrated individual 10 | exposures as well as high-level data products (e.g., mosaics and catalogs). 11 | 12 | The tools in this package allow users to run and configure the pipeline 13 | steps to custom process the Roman data. Additionally, the romancal package 14 | contains the interface to the Roman datamodels, the recommended method 15 | of reading and writing Roman data files in Python. 16 | -------------------------------------------------------------------------------- /docs/roman/linearity/arguments.rst: -------------------------------------------------------------------------------- 1 | Arguments 2 | ========= 3 | 4 | The linearity correction has no step-specific arguments. 5 | -------------------------------------------------------------------------------- /docs/roman/linearity/description.rst: -------------------------------------------------------------------------------- 1 | Description 2 | ============ 3 | 4 | Assumptions 5 | ----------- 6 | It is assumed that the saturation step has already been applied to 7 | the input data, so that saturation flags are set in the GROUPDQ array of 8 | the input science data. 9 | 10 | Algorithm 11 | --------- 12 | The linearity step applies the "classic" linearity correction adapted from 13 | the HST WFC3/IR linearity correction routine, correcting science data values 14 | for detector non-linearity. The correction is applied pixel-by-pixel, 15 | group-by-group, integration-by-integration within a science exposure. 16 | 17 | The correction is represented by an nth-order polynomial for 18 | each pixel in the detector, with n+1 arrays of coefficients read from the 19 | linearity reference file. 20 | 21 | The algorithm for correcting the observed pixel value in each group of an 22 | integration is currently of the form: 23 | 24 | .. math:: 25 | F_\text{c} = c_{0} + c_{1}F + c_{2}F^2 + c_{3}F^3 + ... + c_{n}F^n 26 | 27 | where :math:`F` is the observed counts (in DN), :math:`c_n` are the polynomial 28 | coefficients, and :math:`F_\text{c}` is the corrected counts. There is no 29 | limit to the order of the polynomial correction; all coefficients contained in 30 | the reference file will be applied. 31 | 32 | Upon successful completion of the linearity correction, "cal_step" in the 33 | metadata is set to "COMPLETE". 34 | 35 | Special Handling 36 | ++++++++++++++++ 37 | 38 | - Pixels having at least one correction coefficient equal to NaN will not have 39 | the linearity correction applied and the DQ flag "NO_LIN_CORR" is added to 40 | the science exposure PIXELDQ array. 41 | 42 | - Pixels that have the "NO_LIN_CORR" flag set in the DQ array of the linearity 43 | reference file will not have the correction applied and the "NO_LIN_CORR" flag 44 | is added to the science exposure PIXELDQ array. 45 | 46 | - Pixel values that have the "SATURATED" flag set in a particular group of the 47 | science exposure GROUPDQ array will not have the linearity correction 48 | applied to that group. Any groups for that pixel that are not flagged as 49 | saturated will be corrected. 50 | 51 | The ERR array of the input science exposure is not modified. 52 | 53 | The flags from the linearity reference file DQ array are propagated into the 54 | PIXELDQ array of the science exposure using a bitwise OR operation. 55 | -------------------------------------------------------------------------------- /docs/roman/linearity/index.rst: -------------------------------------------------------------------------------- 1 | .. _linearity_step: 2 | 3 | ==================== 4 | Linearity Correction 5 | ==================== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | .. automodapi:: romancal.linearity 15 | -------------------------------------------------------------------------------- /docs/roman/linearity/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference File Types 2 | ==================== 3 | 4 | The ``linearity`` step uses a LINEARITY reference file. 5 | 6 | .. include:: ../references_general/linearity_reffile.inc 7 | -------------------------------------------------------------------------------- /docs/roman/outlier_detection/index.rst: -------------------------------------------------------------------------------- 1 | .. _outlier_detection_step: 2 | 3 | ================= 4 | Outlier Detection 5 | ================= 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | main.rst 11 | arguments.rst 12 | outlier_detection_step.rst 13 | outlier_detection.rst 14 | outlier_examples.rst 15 | 16 | .. automodapi:: romancal.outlier_detection 17 | -------------------------------------------------------------------------------- /docs/roman/outlier_detection/main.rst: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | :Classes: `romancal.outlier_detection.OutlierDetectionStep` 5 | :Aliases: outlier_detection, outlier_detection_scaled, outlier_detection_stack 6 | 7 | Processing multiple datasets together allows for the identification of 8 | bad pixels or cosmic-rays that remain in each of the input images, 9 | many times at levels which were not detectable by jump detection in 10 | :ref:`ramp_fitting ` step. The 11 | ``outlier_detection`` step implements the following algorithm to 12 | identify and flag any remaining cosmic-rays or other artifacts left 13 | over from previous calibrations: 14 | 15 | - build a stack of input data 16 | 17 | - all inputs will need to have the same WCS, which is done by 18 | `romancal.tweakreg.TweakRegStep`), since outlier detection assumes 19 | the same flux for each point on the sky, and variations from one image to 20 | the next would indicate a spurious signal 21 | 22 | - if needed, each input will be resampled to a common output WCS 23 | 24 | - create a median image from the stack of input data 25 | 26 | - this median operation will ignore any input pixels which have a weight 27 | which is too low (<70% max weight) 28 | 29 | - create "blotted" data from the median image to exactly match each original 30 | input dataset 31 | 32 | - perform a statistical comparison (pixel-by-pixel) between the median blotted 33 | data with the original input data to look for pixels with values that are 34 | different from the mean value by more than some specified sigma 35 | based on the noise model 36 | 37 | - the noise model used relies on the error array computed by previous 38 | calibration steps based on the readnoise and calibration errors 39 | 40 | - flag the DQ array for the input data for any pixel (or affected neighboring 41 | pixels) identified as a statistical outlier. 42 | -------------------------------------------------------------------------------- /docs/roman/outlier_detection/outlier_detection_step.rst: -------------------------------------------------------------------------------- 1 | .. _outlier_design: 2 | 3 | OutlierDetectionStep 4 | -------------------- 5 | 6 | This module provides the sole interface to all methods of performing outlier detection 7 | on Roman observations. The outlier detection algorithm used for WFI data is 8 | described in :ref:`outlier-detection-imaging`. 9 | 10 | .. note:: 11 | Whether the data are being provided in an `association file`_ or as a list of ASDF filenames, 12 | they must always be wrapped with a 13 | :py:class:`~romancal.datamodels.library.ModelLibrary`, which will handle and 14 | read in the input properly. 15 | 16 | .. _association file: https://jwst-pipeline.readthedocs.io/en/latest/jwst/associations/asn_from_list.html 17 | 18 | On successful completion, this step will return the input models with DQ arrays updated 19 | with flags for identified outliers. 20 | 21 | 22 | .. automodapi:: romancal.outlier_detection.outlier_detection_step 23 | -------------------------------------------------------------------------------- /docs/roman/package_index.rst: -------------------------------------------------------------------------------- 1 | .. toctree:: 2 | :maxdepth: 2 3 | 4 | assign_wcs/index.rst 5 | associations/index.rst 6 | dark_current/index.rst 7 | dq_init/index.rst 8 | flatfield/index.rst 9 | flux/index.rst 10 | linearity/index.rst 11 | outlier_detection/index.rst 12 | pipeline/index.rst 13 | ramp_fitting/index.rst 14 | references_general/index.rst 15 | refpix/index.rst 16 | resample/index.rst 17 | saturation/index.rst 18 | skymatch/index.rst 19 | source_catalog/index.rst 20 | stpipe/index.rst 21 | tweakreg/index.rst 22 | -------------------------------------------------------------------------------- /docs/roman/photom/arguments.rst: -------------------------------------------------------------------------------- 1 | Step Arguments 2 | ============== 3 | 4 | The photometric calibration step has one step-specific argument: 5 | 6 | * ``--photom`` 7 | 8 | If the ``photom`` argument is given with a filename for its value, 9 | the photometric calibrated data that are created within the step will be 10 | saved to that file. 11 | -------------------------------------------------------------------------------- /docs/roman/photom/description.rst: -------------------------------------------------------------------------------- 1 | Description 2 | ============ 3 | 4 | Algorithm 5 | --------- 6 | The ``photom`` step adds flux (photometric) calibrations to the metadata of 7 | a data product. The calibration information is read from a photometric reference 8 | file, and the exact nature of the calibration information loaded from the reference file 9 | is described below. This step does not affect the pixel values of the data product. 10 | 11 | Upon successful completion of the photometric correction, the "photom" keyword in 12 | "cal_step" in the metadata is set to "COMPLETE". 13 | 14 | 15 | Photom and Pixel Area Data 16 | -------------------------------- 17 | 18 | The photom reference file contains a table of 19 | exposure parameters that define the flux 20 | conversion and pixel area data for each optical element. The table contains one row 21 | for each optical_element, and the photom step searches the 22 | table for the row that matches the parameters of the science exposure and 23 | then loads the calibration information from that row of the table. 24 | 25 | 26 | 27 | For these table-based PHOTOM reference files, the calibration information in each 28 | row includes a scalar flux conversion constant, the conversion uncertainty, and the nominal pixel area. 29 | 30 | The scalar conversion constant is copied to the header keyword "conversion_megajanskys", which 31 | gives the conversion from DN/s to megaJy/steradian, and converted to microJy/square arcseconds and saved to 32 | the header keyword "conversion_microjanskys". The same process is performed for the uncertainty, with 33 | the values saved in "conversion_megajanskys_uncertainty" and "conversion_microjanskys_uncertainty", 34 | respectively. 35 | 36 | The step also populates the metadata keywords "pixelarea_steradians" and "pixelarea_arcsecsq" in the 37 | science data product, which give the average pixel area in units of 38 | steradians and square arcseconds, respectively. 39 | -------------------------------------------------------------------------------- /docs/roman/photom/index.rst: -------------------------------------------------------------------------------- 1 | .. _photom_step: 2 | 3 | ======================= 4 | Photometric Calibration 5 | ======================= 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | 15 | .. automodapi:: romancal.photom 16 | -------------------------------------------------------------------------------- /docs/roman/photom/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference Files 2 | =============== 3 | The ``photom`` step uses :ref:`PHOTOM ` 4 | reference files. 5 | 6 | .. include:: ../references_general/photom_reffile.inc 7 | -------------------------------------------------------------------------------- /docs/roman/pipeline/graphics/mosaic4_regular_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/pipeline/graphics/mosaic4_regular_grid.png -------------------------------------------------------------------------------- /docs/roman/pipeline/graphics/mosaic_regular_grid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/pipeline/graphics/mosaic_regular_grid.png -------------------------------------------------------------------------------- /docs/roman/pipeline/graphics/wfi_4sca_skycell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/docs/roman/pipeline/graphics/wfi_4sca_skycell.png -------------------------------------------------------------------------------- /docs/roman/pipeline/index.rst: -------------------------------------------------------------------------------- 1 | .. _pipeline-modules: 2 | 3 | ================ 4 | Pipeline Modules 5 | ================ 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | main.rst 10 | exposure_pipeline.rst 11 | mosaic_pipeline.rst 12 | 13 | .. automodapi:: romancal.pipeline 14 | -------------------------------------------------------------------------------- /docs/roman/pipeline/main.rst: -------------------------------------------------------------------------------- 1 | .. _pipelines: 2 | 3 | Pipeline Stages 4 | =============== 5 | 6 | The data from different observing modes needs to be processed with 7 | the proper pipeline levels listed below. The pipeline, and step selection in a pipeline, 8 | are usually based solely on the exposure type (``exposure.type`` attribute). 9 | End-to-end calibration of Roman data is divided into levels of 10 | processing: 11 | 12 | - The Exposure Level Processing (ELP) consists of detector-level corrections applied to 13 | each resultant, followed by ramp fitting. The output of the exposure level 14 | processing is a count rate image per exposure, that is aligned to the Gaia reference system. 15 | The details differ for imaging and spectroscopic exposures and can be found at :ref:`exposure_pipeline`. 16 | 17 | 18 | - The Mosaic Level Processing (MLP) uses overlapping exposures to match the sky background, 19 | detect aberrant data values and resample the image to produce a single undistorted product. 20 | Details are at: :ref:`mosaic_pipeline` 21 | 22 | The table below represents the same information as described above, but 23 | alphabetically ordered by pipeline class. 24 | 25 | +--------------------------------------------+------------------+------------------+ 26 | | Pipeline Class | Alias | Used For | 27 | +============================================+==================+==================+ 28 | | `~romancal.pipeline.ExposurePipeline` | roman_elp | Exposure Level | 29 | +--------------------------------------------+------------------+------------------+ 30 | | `~romancal.pipeline.MosaicPipeline` | roman_mos | High Level | 31 | +--------------------------------------------+------------------+------------------+ 32 | -------------------------------------------------------------------------------- /docs/roman/pipeline_levels.rst: -------------------------------------------------------------------------------- 1 | Pipeline Levels 2 | =============== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | pipeline/exposure_pipeline.rst 7 | pipeline/mosaic_pipeline.rst 8 | -------------------------------------------------------------------------------- /docs/roman/pipeline_ref_files.rst: -------------------------------------------------------------------------------- 1 | Reference Files 2 | --------------- 3 | 4 | Many pipeline steps rely on the use of reference files that contain different types of 5 | calibration data or information necessary for processing the data. The reference files are 6 | instrument-specific and are periodically updated as the data processing evolves and the 7 | understanding of the instruments improves. They are created, tested, and validated by the 8 | Roman Instrument Team. They ensure all the files are in the correct format and have all 9 | required attributes. The files are then delivered to the Reference Data for Calibration 10 | and Tools (ReDCaT) Management Team. The result of this process is the files being ingested 11 | into the Roman Calibration Reference Data System (CRDS), and made available to the pipeline 12 | team and any other ground subsystem that needs access to them. 13 | 14 | Information about all the reference files used by the Calibration Pipeline can be found at 15 | :ref:`reference_file_information`, 16 | as well as in the documentation for each Calibration Step that uses a reference file. 17 | 18 | CRDS 19 | ^^^^ 20 | 21 | CRDS reference file mappings are usually set by default to always give access 22 | to the most recent reference file deliveries and selection rules. On 23 | occasion it might be necessary or desirable to use one of the non-default 24 | mappings in order to, for example, run different versions of the pipeline 25 | software or use older versions of the reference files. This can be 26 | accomplished by setting the environment variable ``CRDS_CONTEXT`` to the 27 | desired project mapping version, e.g. 28 | :: 29 | 30 | $ export CRDS_CONTEXT='roman_0017.pmap' 31 | 32 | Within STScI, the current storage location for all Roman CRDS reference files is: 33 | :: 34 | 35 | /grp/crds/roman 36 | -------------------------------------------------------------------------------- /docs/roman/pipeline_steps.rst: -------------------------------------------------------------------------------- 1 | Pipeline Steps 2 | ============== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | dq_init/index.rst 7 | saturation/index.rst 8 | refpix/index.rst 9 | linearity/index.rst 10 | dark_current/index.rst 11 | ramp_fitting/index.rst 12 | assign_wcs/index.rst 13 | flatfield/index.rst 14 | photom/index.rst 15 | flux/index.rst 16 | tweakreg/index.rst 17 | skymatch/index.rst 18 | outlier_detection/index.rst 19 | resample/index.rst 20 | source_catalog/index.rst 21 | -------------------------------------------------------------------------------- /docs/roman/ramp_fitting/arguments.rst: -------------------------------------------------------------------------------- 1 | .. _rampfit-arguments: 2 | 3 | Arguments 4 | ========= 5 | The ramp fitting step has the following optional argument that can be set by the user: 6 | 7 | * ``--algorithm``: Algorithm to use. At the moment the only supported option 8 | is `ols_cas22` (the default). 9 | 10 | * ``--use_ramp_jump_detection``: A True/False value that specifies whether to use 11 | the unevenly-spaced jump detection integrated into the ramp fitting algorithm. 12 | If True, then the jump detection step will be skipped and then revisited alongside 13 | the ramp fitting step. If False, then the jump detection step will be run. The 14 | default is True. 15 | -------------------------------------------------------------------------------- /docs/roman/ramp_fitting/index.rst: -------------------------------------------------------------------------------- 1 | .. _ramp_fitting_step: 2 | 3 | ============ 4 | Ramp Fitting 5 | ============ 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | .. automodapi:: romancal.ramp_fitting 15 | -------------------------------------------------------------------------------- /docs/roman/ramp_fitting/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference Files 2 | =============== 3 | The ``ramp_fit`` step uses three reference file types: 4 | 5 | - :ref:`GAIN ` 6 | - :ref:`READNOISE ` 7 | - :ref:`DARK ` 8 | 9 | During ramp fitting, the GAIN values are used to temporarily convert the pixel 10 | values from units of DN to electrons, and convert the results of ramp fitting 11 | back to DN. The READNOISE values are used as part of the noise estimate for 12 | each pixel. The DARK values are needed to estimate the Poisson noise. See 13 | :ref:`rampfit-algorithm-ols22` for more details. 14 | -------------------------------------------------------------------------------- /docs/roman/references_general/dark_reffile.inc: -------------------------------------------------------------------------------- 1 | .. _dark_reffile: 2 | 3 | DARK Reference File 4 | ------------------- 5 | 6 | :REFTYPE: DARK 7 | :Data models: `~roman_datamodels.datamodels.DarkRefModel` 8 | 9 | The DARK reference file contains pixel-by-pixel and frame-by-frame 10 | dark current values for a given detector readout mode. 11 | 12 | .. include:: ../references_general/dark_selection.inc 13 | 14 | .. include:: ../includes/standard_keywords.inc 15 | 16 | .. include:: ../references_general/dark_specific.inc 17 | 18 | Reference File Format 19 | +++++++++++++++++++++ 20 | DARK reference files are ASDF format, with 3 data arrays. 21 | The format and content of the file is as follows 22 | (see `~roman_datamodels.datamodels.DarkRefModel`): 23 | 24 | ======= ============ ========================== ============= 25 | Data Array Type Dimensions Data type 26 | ======= ============ ========================== ============= 27 | data NDArray 4096 x 4096 x ngroups float32 28 | err NDArray 4096 x 4096 x ngroups float32 29 | dq NDArray 4096 x 4096 uint32 30 | ======= ============ ========================== ============= 31 | 32 | The ASDF file contains a single set of data, err, and dq arrays. 33 | 34 | .. 35 | Commented until DQ Docs 36 | .. include:: ../includes/dq_def.inc 37 | -------------------------------------------------------------------------------- /docs/roman/references_general/dark_selection.inc: -------------------------------------------------------------------------------- 1 | .. _dark_selectors: 2 | 3 | Reference Selection Keyword Attributes for DARK 4 | +++++++++++++++++++++++++++++++++++++++++++++++ 5 | CRDS selects appropriate DARK references based on the following keyword attributes. 6 | DARK is not applicable for instruments not in the table. 7 | 8 | ========== =========================================== 9 | Instrument Keyword Attributes 10 | ========== =========================================== 11 | WFI instrument, detector, date, time 12 | ========== =========================================== 13 | -------------------------------------------------------------------------------- /docs/roman/references_general/dark_specific.inc: -------------------------------------------------------------------------------- 1 | Type Specific Keyword Attributes for DARK 2 | +++++++++++++++++++++++++++++++++++++++++ 3 | In addition to the standard reference file keyword attributes listed above, 4 | the following keyword attributes are *required* in DARK reference files, 5 | because they are used as CRDS selectors 6 | (see :ref:`dark_selectors`): 7 | 8 | =============== ====================================== ============== 9 | Attribute Fully qualified path Instruments 10 | =============== ====================================== ============== 11 | detector model.meta.instrument.detector WFI 12 | =============== ====================================== ============== 13 | -------------------------------------------------------------------------------- /docs/roman/references_general/distortion_reffile.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _distortion_reffile: 4 | 5 | DISTORTION Reference File 6 | ------------------------- 7 | 8 | :REFTYPE: DISTORTION 9 | :Data model: `~roman_datamodels.datamodels.DistortionRefModel` 10 | 11 | The distortion reference file contains a combination of astropy models, 12 | representing the transform from detector to the telescope V2, V3 system. 13 | The following convention was adopted: 14 | 15 | - the input x and y are 0-based coordinates in the DMS system; 16 | - the output is in the V2, V3 system; 17 | - the center of the first pixel is (0, 0), so the first pixel goes from -0.5 to 0.5; 18 | - the origin of the transform is taken to be (0, 0). 19 | Note that while a different origin can be used for some transforms the relevant 20 | offset should first be prepended to the distortion transform to account for the change 21 | in origin of the coordinate frame. 22 | 23 | Internally the WCS pipeline works with 0-based coordinates. 24 | 25 | .. include:: ../references_general/distortion_selection.inc 26 | 27 | .. include:: ../includes/standard_keywords.inc 28 | 29 | Reference File Format 30 | +++++++++++++++++++++ 31 | DISTORTION reference files are ASDF format, and contain an astropy model object. 32 | The format and content of the file is as follows 33 | (see `~roman_datamodels.datamodels.DistortionRefModel`): 34 | 35 | ================================== ======================== 36 | Data Data type 37 | ================================== ======================== 38 | coordinate_distortion_transform astropy.modeling.Model 39 | ================================== ======================== 40 | 41 | The ASDF file contains a single astropy model object. 42 | 43 | :model: Transform from detector to an intermediate frame (instrument dependent). 44 | -------------------------------------------------------------------------------- /docs/roman/references_general/distortion_selection.inc: -------------------------------------------------------------------------------- 1 | .. _distortion_selectors: 2 | 3 | Reference Selection Keyword Attributes for DISTORTION 4 | +++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 | CRDS automatically selects the appropriate DISTORTION reference 6 | files based on the following keyword attributes. 7 | 8 | N.B.: DISTORTION is not applicable for instruments not in the table. 9 | 10 | ========== =================================================== 11 | Instrument Keyword Attributes 12 | ========== =================================================== 13 | WFI instrument, detector, optical_element, date, time 14 | ========== =================================================== 15 | -------------------------------------------------------------------------------- /docs/roman/references_general/dq_flags.inc: -------------------------------------------------------------------------------- 1 | Flags for the DQ, PIXELDQ, and GROUPDQ Arrays. 2 | 3 | === ========== ================ =========================================== 4 | Bit Value Name Description 5 | === ========== ================ =========================================== 6 | 0 1 DO_NOT_USE Bad pixel. Do not use. 7 | 1 2 SATURATED Pixel saturated during exposure 8 | 2 4 JUMP_DET Jump detected during exposure 9 | 3 8 DROPOUT Data lost in transmission 10 | 4 16 GW_AFFECTED_DATA Data affected by the GW read window 11 | 5 32 PERSISTENCE High persistence (was RESERVED_2) 12 | 6 64 AD_FLOOR Below A/D floor (0 DN, was RESERVED_3) 13 | 7 128 OUTLIER Detected as outlier in coadded image 14 | 8 256 UNRELIABLE_ERROR Uncertainty exceeds quoted error 15 | 9 512 NON_SCIENCE Pixel not on science portion of detector 16 | 10 1024 DEAD Dead pixel 17 | 11 2048 HOT Hot pixel 18 | 12 4096 WARM Warm pixel 19 | 13 8192 LOW_QE Low quantum efficiency 20 | 15 32768 TELEGRAPH Telegraph pixel 21 | 16 65536 NONLINEAR Pixel highly nonlinear 22 | 17 131072 BAD_REF_PIXEL Reference pixel cannot be used 23 | 18 262144 NO_FLAT_FIELD Flat field cannot be measured 24 | 19 524288 NO_GAIN_VALUE Gain cannot be measured 25 | 20 1048576 NO_LIN_CORR Linearity correction not available 26 | 21 2097152 NO_SAT_CHECK Saturation check not available 27 | 22 4194304 UNRELIABLE_BIAS Bias variance large 28 | 23 8388608 UNRELIABLE_DARK Dark variance large 29 | 24 16777216 UNRELIABLE_SLOPE Slope variance large (i.e., noisy pixel) 30 | 25 33554432 UNRELIABLE_FLAT Flat variance large 31 | 26 67108864 RESERVED_5 32 | 27 134217728 RESERVED_6 33 | 28 268435456 UNRELIABLE_RESET Sensitive to reset anomaly 34 | 29 536870912 RESERVED_7 35 | 30 1073741824 OTHER_BAD_PIXEL A catch-all flag 36 | 31 2147483648 REFERENCE_PIXEL Pixel is a reference pixel 37 | === ========== ================ =========================================== 38 | -------------------------------------------------------------------------------- /docs/roman/references_general/flat_reffile.inc: -------------------------------------------------------------------------------- 1 | .. _flat_reffile: 2 | 3 | FLAT Reference File 4 | ------------------- 5 | 6 | :REFTYPE: FLAT 7 | :Data model: `roman_datamodels.datamodels.FlatRefModel` 8 | 9 | The FLAT reference file contains pixel-by-pixel detector response values. 10 | It is used. 11 | 12 | .. include:: ../references_general/flat_selection.inc 13 | 14 | .. include:: ../includes/standard_keywords.inc 15 | 16 | Type Specific Keywords for FLAT 17 | +++++++++++++++++++++++++++++++ 18 | In addition to the standard reference file attributes listed above, 19 | the following attributes are *required* in FLAT reference files, 20 | because they are used as CRDS selectors 21 | (see :ref:`flat_selectors`): 22 | 23 | =============== ====================================== ============== 24 | Attribute Fully qualified path Instruments 25 | =============== ====================================== ============== 26 | detector model.meta.instrument.detector WFI 27 | exptype model.meta.exposure.type 28 | optical_element model.meta.instrument.optical_element 29 | =============== ====================================== ============== 30 | 31 | Reference File Format 32 | +++++++++++++++++++++ 33 | 34 | FLAT reference files are ASDF format, with 3 data arrays. 35 | The format and content of the file is as follows: 36 | 37 | ======= ============ ============== ============= 38 | Data Array Type Dimensions Data type 39 | ======= ============ ============== ============= 40 | data NDArray 4096 x 4096 float32 41 | err NDArray 4096 x 4096 float32 42 | dq NDArray 4096 x 4096 uint32 43 | ======= ============ ============== ============= 44 | 45 | The ASDF file contains a single set of data, err, and dq arrays. 46 | -------------------------------------------------------------------------------- /docs/roman/references_general/flat_selection.inc: -------------------------------------------------------------------------------- 1 | .. _flat_selectors: 2 | 3 | Reference Selection Keywords for FLAT 4 | +++++++++++++++++++++++++++++++++++++ 5 | CRDS selects appropriate FLAT references based on the following attributes. 6 | FLAT is not applicable for instruments not in the table. 7 | Non-standard attributes used for file selection are *required*. 8 | 9 | ========== ======================================================================== 10 | Instrument Metadata 11 | ========== ======================================================================== 12 | WFI instrument, detector, optical_element, date, time 13 | ========== ======================================================================== 14 | -------------------------------------------------------------------------------- /docs/roman/references_general/gain_reffile.inc: -------------------------------------------------------------------------------- 1 | GAIN reference file 2 | ------------------- 3 | 4 | :REFTYPE: GAIN 5 | :Data model: `~roman_datamodels.datamodels.GainRefModel` 6 | 7 | The GAIN reference file contains a pixel-by-pixel gain map, which can be 8 | used to convert pixel values from units of DN to electrons. The gain 9 | values are assumed to be in units of e/DN. 10 | 11 | .. include:: ../references_general/gain_selection.inc 12 | 13 | .. include:: ../includes/standard_keywords.inc 14 | 15 | Type Specific Keyword Attributes for GAIN 16 | +++++++++++++++++++++++++++++++++++++++++ 17 | In addition to the standard reference file keyword attributes listed above, 18 | the following keyword attributes are *required* in GAIN reference files, 19 | because they are used as CRDS selectors 20 | (see :ref:`gain_selectors`): 21 | 22 | =============== ============================ =========== 23 | Keyword Attr. Fully qualified path Instruments 24 | =============== ============================ =========== 25 | detector model.meta.wfi_mode.detector WFI 26 | =============== ============================ =========== 27 | 28 | Reference File Format 29 | +++++++++++++++++++++ 30 | GAIN reference files are asdf files with a single data array. 31 | 32 | ======= ============ ============== ============= 33 | Data Array Type Dimensions Data type 34 | ======= ============ ============== ============= 35 | data NDArray 4096 x 4096 float32 36 | ======= ============ ============== ============= 37 | 38 | The ASDF file contains a single data arrays. 39 | -------------------------------------------------------------------------------- /docs/roman/references_general/gain_reffile.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _gain_reffile: 4 | 5 | .. include:: gain_reffile.inc 6 | -------------------------------------------------------------------------------- /docs/roman/references_general/gain_selection.inc: -------------------------------------------------------------------------------- 1 | Reference Selection Keywords for GAIN 2 | +++++++++++++++++++++++++++++++++++++ 3 | CRDS selects appropriate GAIN references based on the following keywords. 4 | GAIN is not applicable for instruments not in the table. 5 | All keywords used for file selection are *required*. 6 | 7 | ========== ================================================ 8 | Instrument Keywords 9 | ========== ================================================ 10 | WFI instrument, detector, date, time 11 | ========== ================================================ 12 | -------------------------------------------------------------------------------- /docs/roman/references_general/gain_selection.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _gain_selectors: 4 | 5 | .. include:: ../references_general/gain_selection.inc 6 | -------------------------------------------------------------------------------- /docs/roman/references_general/index.rst: -------------------------------------------------------------------------------- 1 | .. _reference_file_information: 2 | 3 | Reference File Information 4 | ========================== 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 5 9 | 10 | references_general.rst 11 | -------------------------------------------------------------------------------- /docs/roman/references_general/linearity_reffile.inc: -------------------------------------------------------------------------------- 1 | LINEARITY Reference File 2 | ------------------------- 3 | 4 | :REFTYPE: LINEARITY 5 | :Data model: `~roman_datamodels.datamodels.LinearityModel` 6 | 7 | The LINEARITY reference file contains pixel-by-pixel polynomial correction 8 | coefficients. 9 | 10 | .. include:: ../references_general/linearity_selection.inc 11 | 12 | .. include:: ../includes/standard_keywords.inc 13 | 14 | Type Specific Attributes for LINEARITY 15 | ++++++++++++++++++++++++++++++++++++++ 16 | In addition to the standard reference file attributes listed above, 17 | the following attributes are *required* in LINEARITY reference files, 18 | because they are used as CRDS selectors 19 | (see :ref:`linearity_selectors`): 20 | 21 | =============== ================================= ============= 22 | Attribute Fully qualified path Instruments 23 | =============== ================================= ============= 24 | detector model.meta.instrument.detector WFI 25 | =============== ================================= ============= 26 | 27 | Reference File Format 28 | +++++++++++++++++++++ 29 | LINEARITY reference files are ASDF format, with 2 data arrays. 30 | The format and content of the file is as follows: 31 | 32 | ======= ============ ========================= =========== 33 | Data Array Type Dimensions Data type 34 | ======= ============ ========================= =========== 35 | coeffs NDArray ncols x nrows x ncoeffs float32 36 | dq NDArray ncols x nrows uint32 37 | ======= ============ ========================= =========== 38 | 39 | Each plane of the COEFFS data cube contains the pixel-by-pixel coefficients for 40 | the associated order of the polynomial. There can be any number of planes to 41 | accommodate a polynomial of any order. 42 | -------------------------------------------------------------------------------- /docs/roman/references_general/linearity_reffile.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _linearity_reffile: 4 | 5 | .. include:: linearity_reffile.inc 6 | -------------------------------------------------------------------------------- /docs/roman/references_general/linearity_selection.inc: -------------------------------------------------------------------------------- 1 | Reference Selection Keywords for LINEARITY 2 | ++++++++++++++++++++++++++++++++++++++++++ 3 | CRDS selects appropriate LINEARITY references based on the following keyword 4 | attributes. 5 | All keyword attributes used for file selection are *required*. 6 | 7 | ========== ===================================== 8 | Instrument Keyword Attributes 9 | ========== ===================================== 10 | WFI instrument, detector, date, time 11 | ========== ===================================== 12 | -------------------------------------------------------------------------------- /docs/roman/references_general/linearity_selection.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _linearity_selectors: 4 | 5 | .. include:: ../references_general/linearity_selection.inc 6 | -------------------------------------------------------------------------------- /docs/roman/references_general/mask_reffile.inc: -------------------------------------------------------------------------------- 1 | .. _mask_reffile: 2 | 3 | MASK Reference File 4 | ------------------- 5 | 6 | :reftype: MASK 7 | :Data model: `~roman_datamodels.datamodels.MaskRefModel` 8 | 9 | The MASK reference file contains pixel-by-pixel DQ flag values that indicate 10 | problem conditions. 11 | 12 | .. include:: ../references_general/mask_selection.inc 13 | 14 | .. include:: ../includes/standard_keywords.inc 15 | 16 | Type Specific Keywords for MASK 17 | +++++++++++++++++++++++++++++++ 18 | In addition to the standard reference file keyword attributes listed above, 19 | the following keyword attributes are *required* in MASK reference files, 20 | because they are used as CRDS selectors 21 | (see :ref:`mask_selectors`): 22 | 23 | =============== ====================================== ============== 24 | Attribute Fully qualified path Instruments 25 | =============== ====================================== ============== 26 | detector model.meta.instrument.detector WFI 27 | =============== ====================================== ============== 28 | 29 | 30 | Reference File Format 31 | +++++++++++++++++++++ 32 | MASK reference files are ASDF format, with one data object. 33 | The format and content of the file is as follows: 34 | 35 | ======= ============ ============== ============= 36 | Data Object Type Dimensions Data type 37 | ======= ============ ============== ============= 38 | dq NDArray 4096 x 4096 uint32 39 | ======= ============ ============== ============= 40 | 41 | The values in the ``dq`` array give the per-pixel flag conditions that are 42 | to be propagated into the science exposure's ``pixeldq`` array. 43 | The dimensions of the ``dq`` array should be equal to the number of columns 44 | and rows in a full-frame readout of a given detector, including reference 45 | pixels. 46 | 47 | The ASDF file contains a single dq array. 48 | -------------------------------------------------------------------------------- /docs/roman/references_general/mask_selection.inc: -------------------------------------------------------------------------------- 1 | .. _mask_selectors: 2 | 3 | Reference Selection Keywords for MASK 4 | +++++++++++++++++++++++++++++++++++++ 5 | CRDS selects appropriate MASK references based on the following keywords. 6 | MASK is not applicable for instruments not in the table. 7 | 8 | ============ ========================================================== 9 | Instrument Metadata 10 | ============ ========================================================== 11 | WFI instrument, detector, date, time 12 | ============ ========================================================== 13 | -------------------------------------------------------------------------------- /docs/roman/references_general/photom_selection.inc: -------------------------------------------------------------------------------- 1 | .. _photom_selectors: 2 | 3 | Reference Selection Keywords for PHOTOM 4 | +++++++++++++++++++++++++++++++++++++++ 5 | CRDS selects appropriate PHOTOM reference based on the following keyword. 6 | PHOTOM is not applicable for instruments not in the table. 7 | All keywords used for file selection are *required*. 8 | 9 | ========== =========================================== 10 | Instrument Keyword Attributes 11 | ========== =========================================== 12 | WFI instrument, date, time 13 | ========== =========================================== 14 | -------------------------------------------------------------------------------- /docs/roman/references_general/readnoise_reffile.inc: -------------------------------------------------------------------------------- 1 | READNOISE reference file 2 | ------------------------ 3 | 4 | :REFTYPE: READNOISE 5 | :Data model: `~roman_datamodels.datamodels.ReadnoiseRefModel` 6 | 7 | The READNOISE reference file contains a pixel-by-pixel map of readnoise, which is 8 | used in estimating the expected noise in each pixel. 9 | 10 | .. include:: ../references_general/readnoise_selection.inc 11 | 12 | .. include:: ../includes/standard_keywords.inc 13 | 14 | Type Specific Keyword Attributes for READNOISE 15 | ++++++++++++++++++++++++++++++++++++++++++++++ 16 | In addition to the standard reference file keyword attributes listed above, 17 | the following keyword attributes are *required* in READNOISE reference files, 18 | because they are used as CRDS selectors 19 | (see :ref:`readnoise_selectors`): 20 | 21 | =============== ============================ =========== 22 | Keyword Attr. Fully qualified path Instruments 23 | =============== ============================ =========== 24 | detector model.meta.wfi_mode.detector WFI 25 | =============== ============================ =========== 26 | 27 | Reference File Format 28 | +++++++++++++++++++++ 29 | READNOISE reference files are asdf files with a single data array. 30 | 31 | ======= ============ ============== ============= 32 | Data Array Type Dimensions Data type 33 | ======= ============ ============== ============= 34 | data NDArray 4096 x 4096 float32 35 | ======= ============ ============== ============= 36 | 37 | The ASDF file contains a single data array. 38 | -------------------------------------------------------------------------------- /docs/roman/references_general/readnoise_reffile.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _readnoise_reffile: 4 | 5 | .. include:: readnoise_reffile.inc 6 | -------------------------------------------------------------------------------- /docs/roman/references_general/readnoise_selection.inc: -------------------------------------------------------------------------------- 1 | Reference Selection Keywords for READNOISE 2 | ++++++++++++++++++++++++++++++++++++++++++ 3 | CRDS selects appropriate READNOISE references based on the following keywords. 4 | READNOISE is not applicable for instruments not in the table. 5 | All keywords used for file selection are *required*. 6 | 7 | ========== ================================================ 8 | Instrument Keywords 9 | ========== ================================================ 10 | WFI instrument, detector, date, time 11 | ========== ================================================ 12 | -------------------------------------------------------------------------------- /docs/roman/references_general/readnoise_selection.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | .. _readnoise_selectors: 4 | 5 | .. include:: ../references_general/readnoise_selection.inc 6 | -------------------------------------------------------------------------------- /docs/roman/references_general/saturation_reffile.inc: -------------------------------------------------------------------------------- 1 | .. _saturation_reffile: 2 | 3 | SATURATION Reference File 4 | ------------------------- 5 | 6 | :REFTYPE: SATURATION 7 | :Data model: `~roman_datamodels.datamodels.SaturationRefModel` 8 | 9 | The SATURATION reference file contains pixel-by-pixel saturation threshold 10 | values. 11 | 12 | .. include:: ../references_general/saturation_selection.inc 13 | 14 | .. include:: ../includes/standard_keywords.inc 15 | 16 | Type Specific Keywords for SATURATION 17 | +++++++++++++++++++++++++++++++++++++ 18 | In addition to the standard reference file keywords listed above, 19 | the following keywords are *required* in SATURATION reference files, 20 | because they are used as CRDS selectors 21 | (see :ref:`saturation_selectors`): 22 | 23 | ========= ============================== ========== 24 | Keyword Data Model Name Instrument 25 | ========= ============================== ========== 26 | detector model.meta.instrument.detector WFI 27 | ========= ============================== ========== 28 | 29 | Reference File Format 30 | +++++++++++++++++++++ 31 | SATURATION reference files are ASDF format, with two data objects. 32 | The format and content of the file is as follows: 33 | 34 | ======= ============ ============== ============= 35 | Data Object Type Dimensions Data type 36 | ======= ============ ============== ============= 37 | data NDArray 4096 x 4096 float32 38 | dq NDArray 4096 x 4096 uint32 39 | ======= ============ ============== ============= 40 | 41 | The values in the ``data`` array give the saturation threshold in units of DN 42 | for each pixel. 43 | 44 | The ASDF file contains two data arrays. 45 | -------------------------------------------------------------------------------- /docs/roman/references_general/saturation_selection.inc: -------------------------------------------------------------------------------- 1 | .. _saturation_selectors: 2 | 3 | Reference Selection Keywords for SATURATION 4 | +++++++++++++++++++++++++++++++++++++++++++ 5 | CRDS selects appropriate SATURATION references based on the following metadata 6 | attributes. 7 | All attributes used for file selection are *required*. 8 | 9 | ========== ================================================ 10 | Instrument Keywords 11 | ========== ================================================ 12 | WFI instrument, detector, date, time 13 | ========== ================================================ 14 | -------------------------------------------------------------------------------- /docs/roman/refpix/index.rst: -------------------------------------------------------------------------------- 1 | .. _refpix: 2 | 3 | ========================== 4 | Reference Pixel Correction 5 | ========================== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | -------------------------------------------------------------------------------- /docs/roman/resample/index.rst: -------------------------------------------------------------------------------- 1 | .. _resample_step: 2 | 3 | ========== 4 | Resample 5 | ========== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | main.rst 11 | arguments.rst 12 | resample_step.rst 13 | resample_utils.rst 14 | 15 | .. automodapi:: romancal.resample 16 | -------------------------------------------------------------------------------- /docs/roman/resample/resample_step.rst: -------------------------------------------------------------------------------- 1 | .. resample_step_: 2 | 3 | Python Step Interface: ResampleStep() 4 | ===================================== 5 | 6 | .. automodapi:: romancal.resample.resample_step 7 | -------------------------------------------------------------------------------- /docs/roman/resample/resample_utils.rst: -------------------------------------------------------------------------------- 1 | Resample Utilities 2 | ================== 3 | 4 | .. currentmodule:: romancal.resample.resample_utils 5 | 6 | .. automodule:: romancal.resample.resample_utils 7 | :members: 8 | -------------------------------------------------------------------------------- /docs/roman/saturation/arguments.rst: -------------------------------------------------------------------------------- 1 | Step Arguments 2 | ============== 3 | The ``saturation`` step has no step-specific arguments. 4 | -------------------------------------------------------------------------------- /docs/roman/saturation/description.rst: -------------------------------------------------------------------------------- 1 | Description 2 | ============ 3 | 4 | The ``saturation`` step flags pixels at or below the A/D floor or above the 5 | saturation threshold. Pixels values are flagged as saturated if the pixel value 6 | is larger than the defined saturation threshold. Pixel values are flagged as 7 | below the A/D floor if they have a value of zero DN. 8 | 9 | This step examines the data group-by-group, comparing the pixel values in the data array with defined 10 | saturation thresholds for each pixel. When it finds a pixel value in a given 11 | group that is above the saturation threshold (high saturation) times a 12 | dilution factor, it sets the 13 | "SATURATED" flag in the corresponding location of the "groupdq" array in the 14 | science exposure. When it finds a pixel in a given group that has a zero or 15 | negative value (below the A/D floor), it sets the "AD_FLOOR" and "DO_NOT_USE" 16 | flags in the corresponding location of the "groupdq" array in the science 17 | exposure For the saturation case, it also flags all subsequent groups for that 18 | pixel as saturated. For example, if there are 10 groups and 19 | group 7 is the first one to cross the saturation threshold for a given pixel, 20 | then groups 7 through 10 will all be flagged for that pixel. 21 | 22 | Pixels with thresholds set to NaN or flagged as "NO_SAT_CHECK" in the saturation 23 | reference file have their thresholds set to the 16-bit A-to-D converter limit 24 | of 65535 and hence will only be flagged as saturated if the pixel reaches that 25 | hard limit in at least one group. The "NO_SAT_CHECK" flag is propagated to the 26 | PIXELDQ array in the output science data to indicate which pixels fall into 27 | this category. 28 | 29 | The "dilution factor" is intended to account for the fact that Roman 30 | downlinks resultants to the ground, which are usually averages over 31 | several reads. The saturation threshold corresponds to the number of 32 | counts at which a detector pixel saturates. Because a resultant is 33 | the average of a number of reads, later reads in a resultant can 34 | saturate, but if earlier reads are unsaturated, the value of the 35 | resultant can fall under the saturation threshold. The dilution 36 | factor varies resultant-by-resultant and is given by 37 | :math:`/max(t)` for all times :math:`t` entering a resultant. 38 | -------------------------------------------------------------------------------- /docs/roman/saturation/index.rst: -------------------------------------------------------------------------------- 1 | .. _saturation_step: 2 | 3 | ==================== 4 | Saturation Detection 5 | ==================== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | .. automodapi:: romancal.saturation 15 | -------------------------------------------------------------------------------- /docs/roman/saturation/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference Files 2 | =============== 3 | The ``saturation`` step uses a SATURATION reference file. 4 | 5 | .. include:: ../references_general/saturation_reffile.inc 6 | -------------------------------------------------------------------------------- /docs/roman/skymatch/index.rst: -------------------------------------------------------------------------------- 1 | .. _skymatch_step: 2 | 3 | ======== 4 | SkyMatch 5 | ======== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | description.rst 11 | arguments.rst 12 | reference_files.rst 13 | 14 | skymatch_step 15 | 16 | .. automodapi:: romancal.skymatch 17 | -------------------------------------------------------------------------------- /docs/roman/skymatch/reference_files.rst: -------------------------------------------------------------------------------- 1 | Reference File 2 | ============== 3 | 4 | The ``skymatch`` step does not use any reference files. 5 | -------------------------------------------------------------------------------- /docs/roman/skymatch/skymatch_step.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | skymatch_step 3 | ============= 4 | 5 | The ``skymatch_step`` function (class name ``SkyMatchStep``) 6 | is the top-level function used to call the skymatch operation 7 | from the Roman calibration pipeline. 8 | 9 | .. currentmodule:: romancal.skymatch.skymatch_step 10 | 11 | .. automodule:: romancal.skymatch.skymatch_step 12 | :members: 13 | :undoc-members: 14 | -------------------------------------------------------------------------------- /docs/roman/source_catalog/arguments.rst: -------------------------------------------------------------------------------- 1 | Step Arguments 2 | ============== 3 | 4 | The ``source_catalog`` step has the following arguments: 5 | 6 | * ``--bkg_boxsize``: An integer value giving the background mesh box 7 | size in pixels 8 | 9 | * ``--kernel_fwhm``: A floating-point value giving the Gaussian kernel 10 | FWHM in pixels 11 | 12 | * ``--snr_threshold``: A floating-point value that sets the 13 | signal-to-noise ratio (SNR) threshold above the background for source 14 | detection. 15 | 16 | * ``--npixels``: An integer value that sets the minimum number of 17 | pixels in a source 18 | 19 | * ``--deblend``: A boolean indicating whether to deblend sources (default 20 | is ``False``) 21 | 22 | * ``--suffix``: A string value giving the file name suffix to use for 23 | the output catalog file (default is ``'cat'``). 24 | 25 | * ``--fit_psf``: A boolean value indicating whether to perform PSF 26 | photometry (default is ``True``) 27 | 28 | * ``--forced_segmentation``: A string value indicating the filename of 29 | the segmentation map to use for forced segmentation 30 | -------------------------------------------------------------------------------- /docs/roman/source_catalog/index.rst: -------------------------------------------------------------------------------- 1 | .. _source_catalog_step: 2 | 3 | ============== 4 | Source Catalog 5 | ============== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | main.rst 11 | arguments.rst 12 | 13 | .. automodapi:: romancal.source_catalog 14 | -------------------------------------------------------------------------------- /docs/roman/stpipe/call_via_run.rst: -------------------------------------------------------------------------------- 1 | .. _run_examples: 2 | 3 | Executing a pipeline or pipeline step directly, or via run() 4 | ============================================================ 5 | 6 | When calling a pipeline or step instance directly, or using the ``run`` method, 7 | you can specify individual parameter values manually. In this case, parameter 8 | files are not used. If you use ``run`` after instantiating with a parameter 9 | file (as is done when using the :ref:`call` method), 10 | the parameter file will be ignored. 11 | 12 | :: 13 | 14 | # Instantiate the class. Do not provide a parameter file. 15 | pipe = ExposurePipeline() 16 | 17 | # Manually set any desired non-default parameter values 18 | pipe.assign_wcs.skip = True 19 | pipe.ramp_fit.override_gain = 'my_gain_file.asdf' 20 | pipe.save_result = True 21 | pipe.output_dir = '/my/data/pipeline_outputs' 22 | 23 | # Run the pipeline 24 | result = pipe.run('r0000101001001001001_0001_wfi01_uncal.asdf') 25 | 26 | To run a single step: 27 | 28 | :: 29 | 30 | from romancal.ramp_fitting import RampFitStep 31 | 32 | # Instantiate the step 33 | step = RampFitStep() 34 | 35 | # Set parameter values 36 | step.override_gain = 'my_gain_file.asdf' 37 | step.save_results = True 38 | step.output_dir = '/my/data/jump_data' 39 | 40 | # Execute using the run method 41 | result = step.run('r0000101001001001001_0001_wfi01_linearity.asdf') 42 | -------------------------------------------------------------------------------- /docs/roman/stpipe/devel_logging.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Logging 3 | ======= 4 | 5 | The logging in stpipe is built on the Python standard library’s 6 | `logging` module. For detailed information about logging, refer to 7 | the documentation there. This document basically outlines some simple 8 | conventions to follow so that the configuration mechanism described in 9 | :ref:`user-logging` works. 10 | 11 | Logging from a Step or Pipeline 12 | =============================== 13 | 14 | Each Step instance (and thus also each Pipeline instance) has a `log` 15 | member, which is a Python `logging.Logger` instance. All messages 16 | from the Step should use this object to log messages. For example, 17 | from a `process` method:: 18 | 19 | self.log.info("This Step wants to say something") 20 | 21 | Logging from library code 22 | ========================= 23 | 24 | Often, you may want to log something from code that is oblivious to 25 | the concept of stpipe Steps. In that case, stpipe has special code 26 | that allows library code to use any logger and have those messages 27 | appear as if they were coming from the step that used the library. 28 | All the library code has to do is use a Python `logging.Logger` as 29 | normal:: 30 | 31 | import logging 32 | 33 | # ... 34 | log = logging.getLogger(__name__) 35 | 36 | # If the log on its own won’t emit, neither will it in the 37 | # context of an stpipe step, so make sure the level is set to 38 | # allow everything through 39 | log.setLevel(logging.DEBUG) 40 | 41 | def my_library_call(): 42 | # ... 43 | log.info("I want to make note of something") 44 | # ... 45 | -------------------------------------------------------------------------------- /docs/roman/stpipe/index.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | STPIPE 3 | ====== 4 | 5 | .. _stpipe-user-steps: 6 | 7 | For Users 8 | ========= 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | user_step.rst 13 | user_pipeline.rst 14 | user_logging.rst 15 | config_asdf.rst 16 | call_via_call.rst 17 | call_via_run.rst 18 | parameter_files.rst 19 | 20 | 21 | .. _stpipe-devel-steps: 22 | 23 | For Developers 24 | ============== 25 | .. toctree:: 26 | :maxdepth: 2 27 | 28 | devel_step.rst 29 | devel_pipeline.rst 30 | devel_logging.rst 31 | 32 | .. automodapi:: romancal.stpipe 33 | -------------------------------------------------------------------------------- /docs/roman/stpipe/parameter_files.rst: -------------------------------------------------------------------------------- 1 | .. _parameter_files: 2 | 3 | Parameter Files 4 | =============== 5 | 6 | Parameter files can be used to specify parameter values when running a 7 | pipeline or individual steps. These values can be 8 | overridden either by the command line options and/or a 9 | local parameter file. See :ref:`Parameter Precedence` for a full description of 10 | how a parameter gets its final value. 11 | 12 | A parameter file should be used when there are parameters a user wishes to 13 | change from the default version for a custom run of the step. To create a 14 | parameter file add ``--save-parameters `` to the command: 15 | :: 16 | 17 | $ strun --save-parameters 18 | 19 | For example, to save the parameters used for a run of the ``ExposurePipeline`` 20 | pipeline, use: 21 | :: 22 | 23 | $ strun roman_elp r0000101001001001001_0001_wfi01_uncal.asdf --save-parameters my_exppars.asdf 24 | 25 | Once saved, the file can be edited, removing parameters that should be left 26 | at their default values, and setting the remaining parameters to the 27 | desired values. Once modified, the new parameter file can be used: 28 | :: 29 | 30 | $ strun my_exppars2.asdf r0000101001001001001_0001_wfi01_uncal.asdf 31 | 32 | Note that the parameter values will reflect whatever was set on the 33 | command-line, or through a specified local parameter file. In short, the 34 | values will be those actually used in the running of the step. 35 | 36 | For more information about and editing of parameter files, see 37 | :ref:`config_asdf_files`. 38 | 39 | 40 | More information on parameter files can be found in the ``stpipe`` User's 41 | Guide at :ref:`stpipe-user-steps`. 42 | -------------------------------------------------------------------------------- /docs/roman/tweakreg/astrometric_utils.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | astrometric_utils 3 | ================= 4 | 5 | The ``astrometric_utils`` module provides functions for generating 6 | astrometric catalogs of sources for the field-of-view covered by a 7 | set of images. 8 | 9 | .. currentmodule:: romancal.tweakreg.astrometric_utils 10 | 11 | .. automodule:: romancal.tweakreg.astrometric_utils 12 | :members: 13 | :undoc-members: 14 | -------------------------------------------------------------------------------- /docs/roman/tweakreg/index.rst: -------------------------------------------------------------------------------- 1 | .. _tweakreg_step: 2 | 3 | ======== 4 | TweakReg 5 | ======== 6 | 7 | .. moduleauthor:: Mihai Cara 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | README.rst 13 | tweakreg_examples.rst 14 | 15 | **Also See:** 16 | 17 | .. toctree:: 18 | :maxdepth: 2 19 | 20 | tweakreg_step 21 | astrometric_utils 22 | 23 | .. automodapi:: romancal.tweakreg 24 | -------------------------------------------------------------------------------- /docs/roman/tweakreg/tweakreg_step.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | tweakreg_step 3 | ============= 4 | 5 | The ``tweakreg_step`` function (class name ``TweakRegStep``) 6 | is the top-level function used to call the "tweakreg" operation 7 | from the Roman Calibration pipeline. 8 | 9 | .. currentmodule:: romancal.tweakreg.tweakreg_step 10 | 11 | .. automodule:: romancal.tweakreg.tweakreg_step 12 | :members: 13 | :undoc-members: 14 | -------------------------------------------------------------------------------- /docs/rtd_environment.yaml: -------------------------------------------------------------------------------- 1 | name: rtd311 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - python=3.11 7 | - pip 8 | - graphviz 9 | - sphinx_rtd_theme>1.2.0 10 | - towncrier 11 | -------------------------------------------------------------------------------- /pytest_romancal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/pytest_romancal/__init__.py -------------------------------------------------------------------------------- /pytest_romancal/stpsf_plugin.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def pytest_configure(config): 5 | config.addinivalue_line( 6 | "markers", "stpsf: mark test as requiring stpsf data to run" 7 | ) 8 | 9 | 10 | def pytest_addoption(parser): 11 | parser.addoption( 12 | "--stpsf", action="store_true", default=False, help="run stpsf tests" 13 | ) 14 | 15 | 16 | def pytest_collection_modifyitems(config, items): 17 | if config.getoption("--stpsf"): 18 | # --runslow given in cli: do not skip slow tests 19 | return 20 | skip_stpsf = pytest.mark.skip(reason="need --stpsf option to run") 21 | for item in items: 22 | if "stpsf" in item.keywords: 23 | item.add_marker(skip_stpsf) 24 | -------------------------------------------------------------------------------- /requirements-dev-st.txt: -------------------------------------------------------------------------------- 1 | # Roman upstream packages 2 | git+https://github.com/spacetelescope/roman_datamodels.git 3 | git+https://github.com/spacetelescope/rad.git 4 | 5 | # shared upstream packages 6 | git+https://github.com/spacetelescope/stcal.git 7 | git+https://github.com/spacetelescope/stpipe.git 8 | 9 | # Other important upstream packages 10 | git+https://github.com/spacetelescope/crds.git 11 | git+https://github.com/spacetelescope/gwcs.git 12 | git+https://github.com/spacetelescope/tweakwcs.git 13 | git+https://github.com/spacetelescope/stpsf.git 14 | # although not directly used install the dev version of poppy for stpsf 15 | git+https://github.com/spacetelescope/poppy.git 16 | -------------------------------------------------------------------------------- /requirements-dev-thirdparty.txt: -------------------------------------------------------------------------------- 1 | # ASDF upstream packages 2 | git+https://github.com/asdf-format/asdf-standard.git 3 | git+https://github.com/asdf-format/asdf.git 4 | git+https://github.com/asdf-format/asdf-transform-schemas.git 5 | git+https://github.com/asdf-format/asdf-coordinates-schemas.git 6 | git+https://github.com/asdf-format/asdf-wcs-schemas.git 7 | 8 | # Use current dev version of deepdiff 9 | git+https://github.com/seperman/deepdiff.git@dev 10 | 11 | # Use weekly astropy dev build 12 | git+https://github.com/astropy/asdf-astropy.git 13 | --extra-index-url https://pypi.anaconda.org/liberfa/simple pyerfa --pre 14 | --extra-index-url https://pypi.anaconda.org/astropy/simple 15 | astropy>=0.0.dev0 16 | photutils>=0.0.dev0 17 | 18 | # Use Bi-weekly numpy/scipy dev builds 19 | --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple 20 | numpy>=2.0.0.dev0 21 | scipy>=0.0.dev0 22 | -------------------------------------------------------------------------------- /romancal/__init__.py: -------------------------------------------------------------------------------- 1 | from importlib.metadata import version 2 | 3 | __version__ = version(__name__) 4 | -------------------------------------------------------------------------------- /romancal/assign_wcs/__init__.py: -------------------------------------------------------------------------------- 1 | from astropy.modeling.models import Mapping 2 | 3 | from .assign_wcs_step import AssignWcsStep 4 | 5 | __all__ = ["AssignWcsStep", "_distorted_to_undistorted"] 6 | 7 | 8 | def _distorted_to_undistorted(wcsobj): 9 | """ 10 | This is a convenience function for Build 17 which returns the transform 11 | from "detector" to "undistorted pixels". 12 | TODO: Add an 'undistorted' coordinate frame. The transform from detector to 13 | 'undistorted' is currently in the SIAF from 'detector' to 'Ideal' frame. 14 | The split is done in the code now. In the future this should be done in the 15 | distortion reference file and become part of the WCS. 16 | """ 17 | transform = wcsobj.get_transform("detector", "v2v3") 18 | shifts = transform[:4] 19 | poly = transform[5:7] 20 | return shifts | Mapping((0, 1, 0, 1)) | poly 21 | -------------------------------------------------------------------------------- /romancal/assign_wcs/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/assign_wcs/tests/__init__.py -------------------------------------------------------------------------------- /romancal/associations/__init__.py: -------------------------------------------------------------------------------- 1 | """Association Generator 2 | 3 | The Association Generator takes a list of items, an Association Pool, and 4 | creates sub-lists of those items depending on each item's attributes. How the 5 | sub-lists are created is defined by Association Rules. 6 | 7 | For more, see the :ref:`documentation overview `. 8 | 9 | """ 10 | 11 | # flake8: noqa: F401, F402, F403 12 | 13 | # Take version from the upstream package 14 | from .. import __version__ 15 | 16 | 17 | # Utility 18 | def libpath(filepath): 19 | """Return the full path to the module library.""" 20 | from os.path import abspath, dirname, join 21 | 22 | return join(dirname(abspath(__file__)), "lib", filepath) 23 | 24 | 25 | from .association import * 26 | from .association_io import * 27 | from .exceptions import * 28 | from .generate import * 29 | from .lib.process_list import * 30 | from .load_asn import load_asn 31 | from .main import * 32 | from .pool import * 33 | from .registry import * 34 | -------------------------------------------------------------------------------- /romancal/associations/config.py: -------------------------------------------------------------------------------- 1 | """Global configuration parameters""" 2 | 3 | # Debug mode 4 | # In conjunction with logging, this mode changes the behavior 5 | # of functions. Common use case is where associations would be 6 | # removed, they are no longer removed. This assists in determining 7 | # why they were an issue. 8 | DEBUG = False 9 | -------------------------------------------------------------------------------- /romancal/associations/exceptions.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "AssociationError", 3 | "AssociationNotAConstraint", 4 | "AssociationNotValidError", 5 | ] 6 | 7 | 8 | class AssociationError(Exception): 9 | """Basic errors related to Associations""" 10 | 11 | 12 | class AssociationNotAConstraint(AssociationError): 13 | """No matching constraint found""" 14 | 15 | 16 | class AssociationNotValidError(AssociationError): 17 | """Given data structure is not a valid association""" 18 | -------------------------------------------------------------------------------- /romancal/associations/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/associations/lib/__init__.py -------------------------------------------------------------------------------- /romancal/associations/lib/acid.py: -------------------------------------------------------------------------------- 1 | """Association Candidate Identifier""" 2 | 3 | import re 4 | from ast import literal_eval 5 | 6 | from .counter import Counter 7 | 8 | __all__ = ["ACID"] 9 | 10 | # Start of the discovered association ids. 11 | _DISCOVERED_ID_START = 3001 12 | 13 | 14 | class ACID: 15 | """Association Candidate Identifer 16 | 17 | Parameters 18 | ---------- 19 | input : str or 2-tuple 20 | The string representation or 2-tuple containing the 21 | candidate ID and TYPE. The string should be itself 22 | the 2-tuple representation when evaluated. The 2-tuple 23 | will have the form: 24 | (id, type) 25 | 26 | Attributes 27 | ---------- 28 | id : str 29 | The id number. 30 | 31 | type : str 32 | The type of candidate. Some, but not all 33 | possibilities include 'OBSERVATION', 34 | 'MOSAIC', 'DISCOVERED' 35 | 36 | __str__ : str 37 | The DMS-specified string representation of 38 | a candidate identifier. Possibilities include 39 | 'oXXX' for OBSERVATION, 'c1XXX' for 'MOSAIC' or 40 | other PPS-defined candidates, and 'a3XXX' for 41 | 'DISCOVERED' associations. 42 | """ 43 | 44 | def __init__(self, input): 45 | try: 46 | self.id, self.type = literal_eval(input) 47 | except (ValueError, SyntaxError): 48 | self.id, self.type = input 49 | 50 | def __str__(self): 51 | return self.id 52 | 53 | 54 | class ACIDMixin: 55 | """Enable ACID for rules""" 56 | 57 | def __init__(self, *args, **kwargs): 58 | # Initialize discovered association ID 59 | self.discovered_id = Counter(_DISCOVERED_ID_START) 60 | 61 | super().__init__(*args, **kwargs) 62 | 63 | def acid_from_constraints(self): 64 | """Determine ACID from constraints""" 65 | for constraint in self.constraints: 66 | if getattr(constraint, "is_acid", False): 67 | value = re.sub("\\\\", "", "-".join(constraint.found_values)) 68 | try: 69 | acid = ACID(value) 70 | except ValueError: 71 | pass 72 | else: 73 | break 74 | else: 75 | id = f"a{self.discovered_id.value:0>3}" 76 | acid = ACID((id, "DISCOVERED")) 77 | 78 | return acid 79 | -------------------------------------------------------------------------------- /romancal/associations/lib/association_rules.py: -------------------------------------------------------------------------------- 1 | """Association Definitions: DMS-specific 2 | 3 | Notes 4 | ----- 5 | These associations are specifically defined for use in DMS. 6 | """ 7 | 8 | from romancal.associations.lib import rules_level2 9 | from romancal.associations.registry import RegistryMarker 10 | 11 | RegistryMarker.mark(rules_level2) 12 | -------------------------------------------------------------------------------- /romancal/associations/lib/callback_registry.py: -------------------------------------------------------------------------------- 1 | """Callback registry""" 2 | 3 | from romancal.lib.signal_slot import Signal 4 | 5 | __all__ = ["CallbackRegistry"] 6 | 7 | 8 | class CallbackRegistry: 9 | """Callback registry""" 10 | 11 | def __init__(self): 12 | self.registry = dict() 13 | 14 | def add(self, event, callback): 15 | """Add a callback to an event""" 16 | try: 17 | signal = self.registry[event] 18 | except KeyError: 19 | signal = Signal() 20 | signal.connect(callback) 21 | self.registry[event] = signal 22 | 23 | def reduce(self, event, *args): 24 | """Perform a reduction on the event args 25 | 26 | Parameters 27 | ---------- 28 | args : [arg[,...]] 29 | The args to filter 30 | 31 | Returns 32 | ------- 33 | The reduced results. 34 | If no results can be determined, 35 | such as if no callbacks were registered, 36 | `None` is returned. 37 | 38 | Notes 39 | ----- 40 | Each function is given the results of the previous 41 | function. As such, if the data has more than one 42 | object, the return of each function should be a tuple that can 43 | then be passed as arguments to the next function. 44 | 45 | There is no guarantee on order which the registered 46 | callbacks are made. Currently, the callbacks are in a list. 47 | Hence, the callbacks will be called in the order registered. 48 | 49 | """ 50 | result = self.registry[event].reduce(*args) 51 | return result 52 | 53 | def add_decorator(self, event): 54 | """Add callbacks by decoration 55 | 56 | Parameters 57 | ---------- 58 | event : str 59 | The name of event to attach the object to. 60 | """ 61 | 62 | def decorator(func): 63 | self.add(event, func) 64 | return func 65 | 66 | return decorator 67 | 68 | __call__ = add_decorator 69 | -------------------------------------------------------------------------------- /romancal/associations/lib/counter.py: -------------------------------------------------------------------------------- 1 | class Counter: 2 | """Like itertools.count but access to the current value""" 3 | 4 | def __init__(self, start=0, step=1, end=None): 5 | self.value = start 6 | self.step = step 7 | self.end = end 8 | 9 | def __iter__(self): 10 | return self 11 | 12 | def __next__(self): 13 | if self.end is not None and abs(self.value) > abs(self.end): 14 | raise StopIteration 15 | self.value += self.step 16 | return self.value 17 | 18 | def next(self): 19 | """python2 compatibility""" 20 | return self.__next__() 21 | 22 | def set(self, value): 23 | """Set new value for counter""" 24 | self.value = value 25 | return self.value 26 | -------------------------------------------------------------------------------- /romancal/associations/lib/ioregistry.py: -------------------------------------------------------------------------------- 1 | """I/O Registry""" 2 | 3 | from .keyvalue_registry import KeyValueRegistry 4 | 5 | 6 | class IORegistry(KeyValueRegistry): 7 | """Association I/O Registry""" 8 | -------------------------------------------------------------------------------- /romancal/associations/lib/member.py: -------------------------------------------------------------------------------- 1 | """Association Member""" 2 | 3 | from collections import UserDict 4 | from copy import copy 5 | 6 | 7 | class Member(UserDict): 8 | """Member of an association 9 | 10 | Parameters 11 | ---------- 12 | initialdata: Dict-like or Member 13 | Initialization data. Any type of initialization that 14 | `collections.UserDict` allows or `Member` itself. 15 | 16 | item: obj 17 | The item to initialize with. This will override 18 | any `Member.item` given in `initialdata`. 19 | 20 | Attributes 21 | ---------- 22 | item: obj 23 | The original item that created this member. 24 | """ 25 | 26 | def __init__(self, initialdata=None, item=None): 27 | self.item = None 28 | 29 | if isinstance(initialdata, Member): 30 | self.data = copy(initialdata.data) 31 | self.item = copy(initialdata.item) 32 | else: 33 | super().__init__(initialdata) 34 | 35 | if item is not None: 36 | self.item = copy(item) 37 | -------------------------------------------------------------------------------- /romancal/associations/lib/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/associations/lib/tests/__init__.py -------------------------------------------------------------------------------- /romancal/associations/load_asn.py: -------------------------------------------------------------------------------- 1 | """Load an Association from a file or object""" 2 | 3 | from inspect import isclass 4 | 5 | from .association import Association 6 | from .registry import AssociationRegistry 7 | 8 | 9 | def load_asn( 10 | serialized, 11 | format=None, 12 | first=True, 13 | validate=True, 14 | registry=AssociationRegistry, 15 | **kwargs, 16 | ): 17 | """Load an Association from a file or object 18 | 19 | Parameters 20 | ---------- 21 | serialized : object 22 | The serialized form of the association. 23 | 24 | format : str or None 25 | The format to force. If None, try all available. 26 | 27 | validate : bool 28 | Validate against the class' defined schema, if any. 29 | 30 | first : bool 31 | A serialization potentially matches many rules. 32 | Only return the first succesful load. 33 | 34 | registry : AssociationRegistry or None 35 | The `AssociationRegistry` to use. 36 | If None, no registry is used. 37 | Can be passed just a registry class instead of instance. 38 | 39 | kwargs : dict 40 | Other arguments to pass to the `load` methods defined 41 | in the `Association.IORegistry` 42 | 43 | Returns 44 | ------- 45 | The Association object 46 | 47 | Raises 48 | ------ 49 | AssociationNotValidError 50 | Cannot create or validate the association. 51 | 52 | Notes 53 | ----- 54 | The `serialized` object can be in any format 55 | supported by the registered I/O routines. For example, for 56 | `json` and `yaml` formats, the input can be either a string or 57 | a file object containing the string. 58 | 59 | If no registry is specified, the default `Association.load` 60 | method is used. 61 | """ 62 | if registry is None: 63 | return Association.load(serialized, format=format, validate=validate) 64 | 65 | if isclass(registry): 66 | registry = registry() 67 | return registry.load( 68 | serialized, format=format, first=first, validate=validate, **kwargs 69 | ) 70 | -------------------------------------------------------------------------------- /romancal/associations/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/associations/tests/__init__.py -------------------------------------------------------------------------------- /romancal/associations/tests/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/associations/tests/data/__init__.py -------------------------------------------------------------------------------- /romancal/associations/tests/data/asn_level2.json: -------------------------------------------------------------------------------- 1 | { 2 | "asn_rule": "Asn_Lv2Image", 3 | "asn_pool": "jw82600_001_20160304T145416_pool", 4 | "program": "82600", 5 | "asn_type": "image2", 6 | "products": [ 7 | { 8 | "name": "test_lrs1_rate", 9 | "members": [ 10 | { 11 | "expname": "test_lrs1_rate.fits", 12 | "exptype": "science" 13 | } 14 | ] 15 | }, 16 | { 17 | "name": "test_lrs2_rate", 18 | "members": [ 19 | { 20 | "expname": "test_lrs2_rate.fits", 21 | "exptype": "science" 22 | } 23 | ] 24 | }, 25 | { 26 | "name": "test_lrs3bkg_rate", 27 | "members": [ 28 | { 29 | "expname": "test_lrs3bkg_rate.fits", 30 | "exptype": "science" 31 | } 32 | ] 33 | }, 34 | { 35 | "name": "test_lrs4bkg_rate", 36 | "members": [ 37 | { 38 | "expname": "test_lrs4bkg_rate.fits", 39 | "exptype": "science" 40 | } 41 | ] 42 | }, 43 | { 44 | "name": "test_lrs3_rate", 45 | "members": [ 46 | { 47 | "expname": "test_lrs3_rate.fits", 48 | "exptype": "science" 49 | }, 50 | { 51 | "expname": "test_lrs3bkg_rate.fits", 52 | "exptype": "background" 53 | } 54 | ] 55 | }, 56 | { 57 | "name": "test_lrs4_rate", 58 | "members": [ 59 | { 60 | "expname": "test_lrs4_rate.fits", 61 | "exptype": "science" 62 | }, 63 | { 64 | "expname": "test_lrs4bkg_rate.fits", 65 | "exptype": "background" 66 | } 67 | ] 68 | } 69 | ] 70 | } 71 | -------------------------------------------------------------------------------- /romancal/associations/tests/data/pool_001_candidates.csv: -------------------------------------------------------------------------------- 1 | FILENAME | OBS_ID | PROGRAM | OBS_NUM | EX_PLAN | PASS | VISIT | SEQUENCE| VISIT_ID | VISIT_GRP | VISIT_TYPE| SEQ_ID| ACT_ID| NEXPSUR| EXPCOUNT| EXP_TYPE| INSTRUME| DETECTOR| TARGNAME| TARGET_ID| OPT_ELEM| DITHER| SUBPIX_DITHER| ASN_CANDIDATE| TILE | N_TILES | EXPOSERR| CAT | SUBCAT 2 | r0099201005001001001_00101_0001_WFI_uncal.asdf|0099201005001001001P011010001|992| 1|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|1|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 3 | r0099201005001001001_00101_0002_WFI_uncal.asdf|0099201005001001001P011010001|992| 1|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|2|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 4 | r0099201005001001001_00101_0003_WFI_uncal.asdf|0099201005001001001P011010001|992| 1|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|3|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 5 | r0099201005001001001_00101_0001_WFI_uncal.asdf|0099201005001001001P011010001|992| 2|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|1|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 6 | r0099201005001001001_00101_0002_WFI_uncal.asdf|0099201005001001001P011010001|992| 2|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|2|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 7 | r0099201005001001001_00101_0003_WFI_uncal.asdf|0099201005001001001P011010001|992| 2|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|3|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 8 | -------------------------------------------------------------------------------- /romancal/associations/tests/data/pool_002_wfi_image.csv: -------------------------------------------------------------------------------- 1 | FILENAME | OBS_ID | PROGRAM | OBS_NUM | EX_PLAN | PASS | VISIT | SEQUENCE| VISIT_ID | VISIT_GRP | VISIT_TYPE| SEQ_ID| ACT_ID| NEXPSUR| EXPCOUNT| EXP_TYPE| INSTRUME| DETECTOR| TARGNAME| TARGET_ID| OPT_ELEM| DITHER| SUBPIX_DITHER| ASN_CANDIDATE| TILE | N_TILES | EXPOSERR| CAT | SUBCAT 2 | r0099201005001001001_00101_0001_WFI_uncal.asdf|0099201005001001001P011010001|992| 1|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|1|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o001'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 3 | r0099201005001001001_00101_0002_WFI_uncal.asdf|0099201005001001001P011010001|992| 1|1|5|1|1| 0099201005001001001|1|GBTDS|1|1|6|2|WFI_IMAGE|WFI|WFI|GBTDS|1|F158|BOXGAP4|NONE|[('o002'),('OBSERVATION')]|1|4|NULL|Survey|GBTDS 4 | -------------------------------------------------------------------------------- /romancal/associations/tests/data/rules_basic.py: -------------------------------------------------------------------------------- 1 | """Association Rules: Basic""" 2 | 3 | from romancal.associations import Association 4 | from romancal.associations.lib.constraint import ConstraintTrue 5 | from romancal.associations.registry import RegistryMarker 6 | 7 | 8 | @RegistryMarker.rule 9 | class Rule_1(Association): 10 | """Basic rule""" 11 | 12 | def __init__(self, version_id=None): 13 | self.constraints = ConstraintTrue() 14 | super().__init__(version_id=version_id) 15 | self.data["members"] = [] 16 | 17 | def _add(self, item): 18 | self.data["members"].append(item) 19 | 20 | def is_item_member(self, item): 21 | """Check if item is already a member of this association 22 | 23 | Parameters 24 | ---------- 25 | item: dict 26 | The item to add. 27 | 28 | Returns 29 | ------- 30 | is_item_member: bool 31 | True if item is a member. 32 | """ 33 | return item in self.data["members"] 34 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_generate.py: -------------------------------------------------------------------------------- 1 | """Test basic generate operations""" 2 | 3 | from romancal.associations import ( 4 | AssociationPool, 5 | AssociationRegistry, 6 | generate, 7 | load_asn, 8 | ) 9 | from romancal.associations.tests.helpers import t_path 10 | 11 | 12 | def test_simple(): 13 | """Test generate on simple registry""" 14 | registry = AssociationRegistry( 15 | [t_path("data/rules_basic.py")], include_default=False 16 | ) 17 | pool = AssociationPool() 18 | pool["value"] = ["row1", "row2"] 19 | 20 | asns = generate(pool, registry) 21 | assert len(asns) == 1 22 | assert len(asns[0]["members"]) == 2 23 | 24 | 25 | def test_unserialize(): 26 | """Test basic unserializing""" 27 | asn_file = t_path("data/asn_mosaic.json") 28 | with open(asn_file) as asn_fp: 29 | asn = load_asn(asn_fp) 30 | assert isinstance(asn, dict) 31 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_level2_basics.py: -------------------------------------------------------------------------------- 1 | """Test basic usage of Level2 associations""" 2 | 3 | from romancal.associations import generate, load_asn 4 | from romancal.associations.main import Main 5 | from romancal.associations.tests.helpers import ( 6 | combine_pools, 7 | registry_level2_only, 8 | t_path, 9 | ) 10 | 11 | # REGEX_LEVEL2 = r'(?P.+)(?P_rate(ints)?)(?P\..+)' 12 | # REGEX_LEVEL2 = r'(?P.+)(.*\_.*\_.*\_.*\_.*)' 13 | REGEX_LEVEL2 = r"(?P.+)(?P_cal?)(?P\..+)" 14 | 15 | 16 | def from_level2_schema(): 17 | with open(t_path("data/asn_level2.json")) as asn_file: 18 | asn = load_asn(asn_file) 19 | return [asn] 20 | 21 | 22 | def generate_from_pool(pool_path): 23 | """Generate associations from pools""" 24 | rules = registry_level2_only() 25 | pool = combine_pools(t_path(pool_path)) 26 | asns = generate(pool, rules) 27 | return asns 28 | 29 | 30 | def cmd_from_pool(pool_path, args): 31 | """Run commandline on pool 32 | 33 | Parameters 34 | --------- 35 | pool_path: str 36 | The pool to run on. 37 | 38 | args: [arg(, ...)] 39 | Additional command line arguments in the form `sys.argv` 40 | """ 41 | full_args = [ 42 | "--dry-run", 43 | "-D", 44 | "-r", 45 | t_path("../lib/rules_level2.py"), 46 | "--ignore-default", 47 | ] 48 | full_args.extend(args) 49 | result = Main(full_args, pool=pool_path) 50 | return result 51 | 52 | 53 | def test_level2_productname(): 54 | asns = generate_from_pool("data/pool_002_wfi_image.csv") 55 | for asn in asns: 56 | for product in asn["products"]: 57 | science = [ 58 | member 59 | for member in product["members"] 60 | if member["exptype"] == "science" or member["exptype"] == "wfi_image" 61 | ] 62 | if asn["asn_rule"] == "Asn_Lv2Image": 63 | assert len(science) == 2 64 | if asn["asn_rule"] == "Asn_Lv2FOV": 65 | assert len(science) == 18 66 | 67 | 68 | # match = re.match(REGEX_LEVEL2, science[0]['expname']) 69 | # pdb.set_trace() 70 | # assert match.groupdict()['path'] == product['name'] 71 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_level2_candidates.py: -------------------------------------------------------------------------------- 1 | """Test Level2 candidate operation""" 2 | 3 | import pytest 4 | 5 | from romancal.associations.main import Main 6 | from romancal.associations.tests.helpers import ( 7 | level2_rule_path, 8 | mkstemp_pool_file, 9 | t_path, 10 | ) 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "partial_args, n_asns", 15 | [ 16 | # Invalid ACID 17 | (["-i", "nosuchid"], 0), 18 | # Basic observation ACIDs 19 | (["-i", "o001"], 0), 20 | # Whole program 21 | ([], 5), 22 | # Discovered only 23 | (["--discover"], 0), 24 | # Candidates only 25 | (["--all-candidates"], 5), 26 | ], 27 | ) 28 | def test_candidate_observation(partial_args, n_asns): 29 | with mkstemp_pool_file(t_path("data/pool_001_candidates.csv")) as pool_path: 30 | cmd_args = [ 31 | pool_path, 32 | "--dry-run", 33 | "-r", 34 | level2_rule_path(), 35 | "--ignore-default", 36 | ] 37 | cmd_args.extend(partial_args) 38 | generated = Main.cli(cmd_args) 39 | assert len(generated.associations) == n_asns 40 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_mk_skycell_asn_from_skycell_list.py: -------------------------------------------------------------------------------- 1 | """Tests for mk_patchlist""" 2 | 3 | import pytest 4 | 5 | # from romancal.associations import Association, AssociationRegistry, load_asn 6 | from romancal.associations.mk_skycell_asn_from_skycell_list import _cli 7 | 8 | 9 | def test_cmdline_fails(): 10 | """Exercise the command line interface""" 11 | 12 | # No arguments 13 | with pytest.raises(SystemExit): 14 | _cli([]) 15 | 16 | # Only the association file argument 17 | with pytest.raises(SystemExit): 18 | _cli(["-o", "test_asn.json"]) 19 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_mk_skycell_list.py: -------------------------------------------------------------------------------- 1 | """Tests for mk_patchlist""" 2 | 3 | import pytest 4 | 5 | # from romancal.associations import Association, AssociationRegistry, load_asn 6 | from romancal.associations.mk_skycell_list import _cli 7 | 8 | 9 | def test_cmdline_fails(): 10 | """Exercise the command line interface""" 11 | 12 | # No arguments 13 | with pytest.raises(SystemExit): 14 | _cli([]) 15 | 16 | # Only the association file argument 17 | with pytest.raises(SystemExit): 18 | _cli(["-o", "test_asn.json"]) 19 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_pool.py: -------------------------------------------------------------------------------- 1 | from romancal.associations import AssociationPool 2 | from romancal.associations.tests.helpers import t_path 3 | 4 | POOL_FILE = t_path("data/jw93060_20150312T160130_pool.csv") 5 | 6 | 7 | def test_pool(tmp_path): 8 | pool = AssociationPool.read(POOL_FILE) 9 | assert len(pool) == 636 10 | 11 | file_path = tmp_path / __name__ 12 | file_path.mkdir() 13 | 14 | tmp_pool = str(file_path / "tmp_pool.csv") 15 | pool.write(tmp_pool) 16 | 17 | roundtrip = AssociationPool.read(tmp_pool) 18 | assert len(pool) == len(roundtrip) 19 | assert set(pool.colnames) == set(roundtrip.colnames) 20 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_registry.py: -------------------------------------------------------------------------------- 1 | """Test the Registry""" 2 | 3 | import pytest 4 | 5 | from romancal.associations.lib.keyvalue_registry import KeyValueRegistry 6 | 7 | 8 | @pytest.mark.skip(reason="Need to impliment") 9 | def test_registry_match(full_pool_rules): 10 | """Test the match method""" 11 | pool, rules, pool_fname = full_pool_rules 12 | 13 | assert len(rules.schemas) > 0 14 | matches = rules.match(pool[1]) 15 | assert isinstance(matches, tuple) 16 | asns = matches[0] 17 | reprocess_list = matches[1] 18 | assert isinstance(asns, list) 19 | assert isinstance(reprocess_list, list) 20 | assert len(asns) >= 1 21 | 22 | 23 | # Tests below for keyvalue_registry 24 | 25 | 26 | def test_dict_like(): 27 | """Test the basic to ensure similar to a dict""" 28 | data = {"a": 1, "b": 2} 29 | kvr = KeyValueRegistry(data) 30 | assert data.keys() == kvr.keys() 31 | assert set(data.values()) == set(kvr.values()) 32 | 33 | assert kvr.get("a") == 1 34 | assert kvr.get("c", 3) == 3 35 | 36 | keys, values = zip(*kvr.items(), strict=False) 37 | assert set(data.keys()) == set(keys) 38 | assert set(data.values()) == set(values) 39 | 40 | kvr_copy = kvr.copy() 41 | assert set(kvr_copy) == set(kvr) 42 | 43 | assert kvr.pop("a") == 1 44 | assert kvr.pop("a", 3) == 3 45 | assert kvr.popitem() == ("b", 2) 46 | with pytest.raises(KeyError): 47 | kvr.popitem() 48 | 49 | kvr = KeyValueRegistry() 50 | kvr.update(data) 51 | assert data.keys() == kvr.keys() 52 | assert set(data.values()) == set(kvr.values()) 53 | 54 | kvr.clear() 55 | assert len(kvr) == 0 56 | 57 | 58 | def test_default(): 59 | kvr = KeyValueRegistry(default={"a": 1}) 60 | assert kvr.default == "a" 61 | assert kvr[None] == 1 62 | 63 | 64 | def test_tuple(): 65 | data = ("a", 1) 66 | kvr = KeyValueRegistry(data) 67 | assert kvr["a"] == 1 68 | 69 | kvr = KeyValueRegistry(default=data) 70 | assert kvr["a"] == 1 71 | assert kvr[None] == 1 72 | 73 | 74 | def test_fn(): 75 | def fn(): 76 | return 1 77 | 78 | kvr = KeyValueRegistry(fn) 79 | assert kvr[fn.__name__] is fn 80 | 81 | kvr = KeyValueRegistry(default=fn) 82 | assert kvr[fn.__name__] is fn 83 | assert kvr[None] is fn 84 | 85 | 86 | def test_decorator(): 87 | kvr = KeyValueRegistry() 88 | 89 | @kvr 90 | def fn(): 91 | return 1 92 | 93 | assert kvr[fn.__name__] is fn 94 | assert kvr[fn.__name__]() is fn() 95 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_skycell_asn.py: -------------------------------------------------------------------------------- 1 | """Tests for skycell_asn""" 2 | 3 | import pytest 4 | 5 | # from romancal.associations import Association, AssociationRegistry, load_asn 6 | import romancal.associations.skycell_asn as skycell_asn 7 | from romancal.associations.skycell_asn import _cli 8 | 9 | 10 | def test_cmdline_fails(): 11 | """Exercise the command line interface""" 12 | 13 | # No arguments 14 | with pytest.raises(SystemExit): 15 | _cli([]) 16 | 17 | # Only the association file argument 18 | with pytest.raises(SystemExit): 19 | _cli(["-o", "test_asn.json"]) 20 | 21 | 22 | def test_parse_visitID(): 23 | filelist1 = [ 24 | "r0000101002003004005_0001_wfi10_cal.asdf", 25 | ] 26 | visitid_parts = skycell_asn.parse_visitID(filelist1[0][1:20]) 27 | assert visitid_parts["Program"] == "00001" 28 | assert visitid_parts["Execution"] == "01" 29 | assert visitid_parts["Pass"] == "002" # noqa: S105 30 | assert visitid_parts["Segment"] == "003" 31 | assert visitid_parts["Observation"] == "004" 32 | assert visitid_parts["Visit"] == "005" 33 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_update_path.py: -------------------------------------------------------------------------------- 1 | """Test utility update_path""" 2 | 3 | from romancal.associations.asn_from_list import asn_from_list 4 | from romancal.associations.lib.rules_elpp_base import DMS_ELPP_Base 5 | from romancal.associations.lib.update_path import update_path 6 | 7 | 8 | def test_update_path_level2(): 9 | members = ["a", "b", "c"] 10 | new_path = "new_path" 11 | product_name = "new_product" 12 | asn = asn_from_list(members, product_name=product_name, rule=DMS_ELPP_Base) 13 | update_path(asn, new_path) 14 | for product in asn["products"]: 15 | for member in product["members"]: 16 | assert member["expname"].startswith(new_path) 17 | 18 | 19 | def test_update_path_level3(): 20 | members = ["a", "b", "c"] 21 | new_path = "new_path" 22 | asn = asn_from_list(members, product_name="test") 23 | update_path(asn, new_path) 24 | for product in asn["products"]: 25 | for member in product["members"]: 26 | assert member["expname"].startswith(new_path) 27 | -------------------------------------------------------------------------------- /romancal/associations/tests/test_version.py: -------------------------------------------------------------------------------- 1 | """Test versioning consistency""" 2 | 3 | from romancal import __version__ 4 | from romancal.associations.asn_from_list import asn_from_list 5 | 6 | 7 | def test_asn_version(): 8 | """Test version in association is package version""" 9 | 10 | asn = asn_from_list(["a", "b", "c"], product_name="aproduct") 11 | 12 | assert asn["code_version"] == __version__ 13 | -------------------------------------------------------------------------------- /romancal/dark_current/__init__.py: -------------------------------------------------------------------------------- 1 | from .dark_current_step import DarkCurrentStep 2 | 3 | __all__ = ["DarkCurrentStep"] 4 | -------------------------------------------------------------------------------- /romancal/dark_current/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/dark_current/tests/__init__.py -------------------------------------------------------------------------------- /romancal/datamodels/__init__.py: -------------------------------------------------------------------------------- 1 | from .library import ModelLibrary 2 | 3 | __all__ = ["ModelLibrary"] 4 | -------------------------------------------------------------------------------- /romancal/datamodels/filetype.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | from pathlib import Path 4 | 5 | import roman_datamodels as rdm 6 | 7 | from romancal.datamodels import ModelLibrary 8 | 9 | 10 | def check(init: os.PathLike | Path | io.FileIO) -> str: 11 | """ 12 | Determine the type of a file and return it as a string 13 | 14 | Parameters 15 | ---------- 16 | 17 | init : str 18 | file path or file object 19 | 20 | Returns 21 | ------- 22 | file_type: str 23 | a string with the file type ("asdf", "asn", "DataModel", or "ModelLibrary") 24 | 25 | """ 26 | 27 | supported = ("asdf", "json", "DataModel") 28 | 29 | if isinstance(init, str | os.PathLike | Path): 30 | path, ext = os.path.splitext(init) 31 | ext = ext.strip(".") 32 | 33 | if not ext: 34 | raise ValueError(f"Input file path does not have an extension: {init}") 35 | 36 | if ext not in supported: # Could be the file is zipped; try splitting again 37 | path, ext = os.path.splitext(path) 38 | ext = ext.strip(".") 39 | 40 | if ext not in supported: 41 | raise ValueError(f"Unrecognized file type for: {init}") 42 | 43 | if ext == "json": # Assume json input is an association 44 | return "asn" 45 | 46 | return ext 47 | elif isinstance(init, rdm.DataModel): 48 | return "DataModel" 49 | 50 | elif isinstance(init, ModelLibrary): 51 | return "ModelLibrary" 52 | 53 | elif hasattr(init, "read") and hasattr(init, "seek"): 54 | magic = init.read(5) 55 | init.seek(0, 0) 56 | 57 | if not magic or len(magic) < 5: 58 | raise ValueError(f"Cannot get file type of {init!s}") 59 | 60 | if magic == b"#ASDF": 61 | return "asdf" 62 | 63 | return "asn" 64 | else: 65 | raise ValueError(f"Cannot get file type of {init!s}") 66 | -------------------------------------------------------------------------------- /romancal/datamodels/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/datamodels/tests/__init__.py -------------------------------------------------------------------------------- /romancal/datamodels/tests/data/empty.asdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/datamodels/tests/data/empty.asdf -------------------------------------------------------------------------------- /romancal/datamodels/tests/data/empty.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/datamodels/tests/data/empty.json -------------------------------------------------------------------------------- /romancal/datamodels/tests/data/example_schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "date" : { 3 | "title" : "[yyyy-mm-ddThh:mm:ss.ssssss] UTC date file created", 4 | "type" : "string", 5 | "sql_dtype" : "datetime2", 6 | "fits_keyword" : "DATE", 7 | "description" : "The UTC date and time when the HDU was created, in the form YYYY-MM-DDThh:mm:ss.ssssss, where YYYY shall be the four-digit calendar year number, MM the two-digit month number with January given by 01 and December by 12, and DD the two-digit day of the month. The literal T shall separate the date and time, hh shall be the two-digit hour in the day, mm the two-digit number of minutes after the hour, and ss.ssssss the number of seconds (two digits followed by a fraction accurate to microseconds) after the minute. Default values must not be given to any portion of the date/time string, and leading zeros must not be omitted.", 8 | "calculation" : "Operating system time in the format of YYYY-MM-DDThh:mm:ss.ssssss", 9 | "default_value" : "", 10 | "example" : "2015-01-01T00:00:00.000001", 11 | "units" : "", 12 | "sw_source" : "calculation", 13 | "source" : "Science Data Processing (SDP)", 14 | "destination" : ["ScienceCommon.date","GuideStar.date"], 15 | "level" : "1a", 16 | "si" : "Multiple", 17 | "section" : "Basic", 18 | "mode" : "All", 19 | "fits_hdu" : "PRIMARY", 20 | "misc" : "" 21 | }, 22 | 23 | "origin" : { 24 | "title" : "institution responsible for creating FITS file", 25 | "type" : "string", 26 | "sql_dtype" : "nvarchar(20)", 27 | "fits_keyword" : "ORIGIN", 28 | "description" : "Identifies the organization or institution responsible for creating the FITS file.", 29 | "calculation" : "", 30 | "default_value" : "STSCI", 31 | "example" : "STSCI", 32 | "units" : "", 33 | "sw_source" : "", 34 | "source" : "Science Data Processing (SDP)", 35 | "destination" : ["ScienceCommon.origin","GuideStar.origin"], 36 | "level" : "1a", 37 | "si" : "Multiple", 38 | "section" : "Basic", 39 | "mode" : "All", 40 | "fits_hdu" : "PRIMARY", 41 | "misc" : "" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /romancal/datamodels/tests/data/fake.asdf: -------------------------------------------------------------------------------- 1 | not actually an ASDF file 2 | -------------------------------------------------------------------------------- /romancal/datamodels/tests/data/fake.json: -------------------------------------------------------------------------------- 1 | not actually a JSON file 2 | -------------------------------------------------------------------------------- /romancal/datamodels/tests/data/pluto.asdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/datamodels/tests/data/pluto.asdf -------------------------------------------------------------------------------- /romancal/datamodels/tests/test_filetype.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | import roman_datamodels as rdm 5 | 6 | from romancal.datamodels import ModelLibrary, filetype 7 | 8 | DATA_DIRECTORY = Path(__file__).parent / "data" 9 | 10 | 11 | def test_filetype(): 12 | file_1 = filetype.check(DATA_DIRECTORY / "empty.json") 13 | file_2 = filetype.check(DATA_DIRECTORY / "example_schema.json") 14 | with open(DATA_DIRECTORY / "fake.json") as file_h: 15 | file_3 = filetype.check(file_h) 16 | file_4 = filetype.check(DATA_DIRECTORY / "empty.asdf") 17 | file_5 = filetype.check(DATA_DIRECTORY / "pluto.asdf") 18 | with open(DATA_DIRECTORY / "pluto.asdf", "rb") as file_h: 19 | file_6 = filetype.check(file_h) 20 | file_7 = filetype.check(DATA_DIRECTORY / "fake.asdf") 21 | with open(DATA_DIRECTORY / "fake.json") as file_h: 22 | file_8 = filetype.check(file_h) 23 | file_9 = filetype.check(str(DATA_DIRECTORY / "pluto.asdf")) 24 | im1 = rdm.datamodels.ImageModel.create_fake_data(shape=(20, 20)) 25 | file_11 = filetype.check(im1) 26 | model_library = ModelLibrary([im1]) 27 | file_10 = filetype.check(model_library) 28 | 29 | assert file_1 == "asn" 30 | assert file_2 == "asn" 31 | assert file_3 == "asn" 32 | assert file_4 == "asdf" 33 | assert file_5 == "asdf" 34 | assert file_6 == "asdf" 35 | assert file_7 == "asdf" 36 | assert file_8 == "asn" 37 | assert file_9 == "asdf" 38 | assert file_10 == "ModelLibrary" 39 | assert file_11 == "DataModel" 40 | 41 | with pytest.raises(ValueError): 42 | filetype.check(DATA_DIRECTORY / "empty.txt") 43 | 44 | with pytest.raises(ValueError): 45 | filetype.check(2) 46 | 47 | with pytest.raises(ValueError): 48 | filetype.check("test") 49 | -------------------------------------------------------------------------------- /romancal/dq_init/__init__.py: -------------------------------------------------------------------------------- 1 | from .dq_init_step import DQInitStep 2 | 3 | __all__ = ["DQInitStep"] 4 | -------------------------------------------------------------------------------- /romancal/dq_init/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/dq_init/tests/__init__.py -------------------------------------------------------------------------------- /romancal/flatfield/__init__.py: -------------------------------------------------------------------------------- 1 | from .flat_field_step import FlatFieldStep 2 | 3 | __all__ = ["FlatFieldStep"] 4 | -------------------------------------------------------------------------------- /romancal/flatfield/flat_field_step.py: -------------------------------------------------------------------------------- 1 | """ 2 | Flat-field a science image 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | from typing import TYPE_CHECKING 8 | 9 | import roman_datamodels as rdm 10 | 11 | from ..stpipe import RomanStep 12 | from . import flat_field 13 | 14 | if TYPE_CHECKING: 15 | from typing import ClassVar 16 | 17 | __all__ = ["FlatFieldStep"] 18 | 19 | 20 | class FlatFieldStep(RomanStep): 21 | """Flat-field a science image using a flatfield reference image.""" 22 | 23 | class_alias = "flat_field" 24 | spec = """ 25 | include_var_flat = boolean(default=False) # include flat field variance 26 | """ 27 | 28 | reference_file_types: ClassVar = ["flat"] 29 | 30 | def process(self, input_model): 31 | if not isinstance(input_model, rdm.DataModel): 32 | input_model = rdm.open(input_model) 33 | 34 | reference_file_name = self.get_reference_file(input_model, "flat") 35 | 36 | # Check for a valid reference file 37 | if reference_file_name == "N/A": 38 | self.log.warning("No FLAT reference file found") 39 | self.log.warning("Flat Field step will be skipped") 40 | reference_file_name = None 41 | 42 | if reference_file_name is not None: 43 | reference_file_model = rdm.open(reference_file_name) 44 | self.log.debug(f"Using FLAT ref file: {reference_file_name}") 45 | else: 46 | reference_file_model = None 47 | self.log.debug("Using no FLAT ref file") 48 | 49 | # Do the flat-field correction 50 | output_model = flat_field.do_correction( 51 | input_model, reference_file_model, include_var_flat=self.include_var_flat 52 | ) 53 | 54 | # Close reference file 55 | if reference_file_model is not None: 56 | reference_file_model.close() 57 | 58 | if self.save_results: 59 | try: 60 | self.suffix = "flat" 61 | except AttributeError: 62 | self["suffix"] = "flat" 63 | 64 | return output_model 65 | -------------------------------------------------------------------------------- /romancal/flatfield/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/flatfield/tests/__init__.py -------------------------------------------------------------------------------- /romancal/flux/__init__.py: -------------------------------------------------------------------------------- 1 | from .flux_step import FluxStep 2 | 3 | __all__ = ["FluxStep"] 4 | -------------------------------------------------------------------------------- /romancal/flux/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/flux/tests/__init__.py -------------------------------------------------------------------------------- /romancal/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/lib/__init__.py -------------------------------------------------------------------------------- /romancal/lib/dqflags.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from roman_datamodels.dqflags import group, pixel 4 | 5 | __all__ = ["group", "pixel"] 6 | 7 | warnings.warn( 8 | "romancal.dqflags is deprecated. Please use roman_datamodels.dqflags instead.", 9 | DeprecationWarning, 10 | stacklevel=2, 11 | ) 12 | -------------------------------------------------------------------------------- /romancal/lib/progress.py: -------------------------------------------------------------------------------- 1 | """Provide a visual progress bar facility 2 | 3 | The actual functionality is provided by the external library `progress` 4 | 5 | https://pypi.org/project/progress/ 6 | 7 | If the module is not available, then stub it out. 8 | """ 9 | 10 | import logging 11 | 12 | __all__ = ["Bar"] 13 | 14 | # Configure logging 15 | logger = logging.getLogger(__name__) 16 | logger.addHandler(logging.NullHandler()) 17 | 18 | 19 | class _BarStub: 20 | """Stub the Bar functionality""" 21 | 22 | def __init__(self, *args, **kwargs): 23 | pass 24 | 25 | def __enter__(self): 26 | return self 27 | 28 | def __exit__(self, *args, **kwargs): 29 | pass 30 | 31 | def next(self): 32 | pass 33 | 34 | 35 | # Stub the Bar functionality if the actual module does not exist. 36 | try: 37 | from progress.bar import Bar as _Bar 38 | except ModuleNotFoundError: 39 | _Bar = _BarStub 40 | 41 | 42 | def Bar(*args, log_level=logging.INFO, log_cutoff=logging.INFO, **kwargs): 43 | """Actually use Bar only if logging level is appropriate 44 | 45 | Parameters 46 | ---------- 47 | *args : *args 48 | Positional arguments for the `progress.Bar` class 49 | 50 | log_level : int 51 | The current log level. Only produce a Bar if the `log_level` 52 | is less than or equal to `log_cutoff` 53 | 54 | log_cutoff : int 55 | Maximum logging level to allow Bar to be created. 56 | 57 | **kwargs : **kwargs 58 | Keyword arguments for `progress.Bar` class 59 | """ 60 | if log_level <= log_cutoff: 61 | return _Bar(*args, **kwargs) 62 | return _BarStub(*args, **kwargs) 63 | -------------------------------------------------------------------------------- /romancal/lib/save_wcs.py: -------------------------------------------------------------------------------- 1 | """Step-level utility to create/save WfiWcsModel""" 2 | 3 | import logging 4 | 5 | from roman_datamodels.datamodels import WfiWcsModel 6 | 7 | # Define logging 8 | log = logging.getLogger(__name__) 9 | log.setLevel(logging.DEBUG) 10 | 11 | 12 | def save_wfiwcs(step, lib, force=False): 13 | """Create and save the WfiWcs products 14 | 15 | Parameters 16 | ---------- 17 | lib : ModelLibrary 18 | The final L2 models 19 | 20 | force : boolean 21 | Regardless of whether ``save_results`` is `False` 22 | and no ``output_file`` is specified, try saving. 23 | """ 24 | log.info("Writing the WCS files...") 25 | with lib: 26 | for model in lib: 27 | try: 28 | wfiwcs = WfiWcsModel.from_model_with_wcs(model) 29 | except ValueError: 30 | log.info( 31 | f"No WCS information for model {model}. Now `_wcs` product will be created." 32 | ) 33 | lib.shelve(model) 34 | continue 35 | step.finalize_result(wfiwcs, []) 36 | step.save_model(wfiwcs, suffix="wcs", force=force) 37 | lib.shelve(model) 38 | -------------------------------------------------------------------------------- /romancal/lib/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/lib/tests/__init__.py -------------------------------------------------------------------------------- /romancal/lib/tests/test_basic_utils.py: -------------------------------------------------------------------------------- 1 | """Test basic utils""" 2 | 3 | import numpy as np 4 | import pytest 5 | from astropy.table import Table 6 | 7 | from romancal.lib.basic_utils import bytes2human, recarray_to_ndarray 8 | 9 | test_data = [ 10 | (1000, "1000B"), 11 | (1024, "1.0K"), 12 | (1024 * 10, "10.0K"), 13 | (100001221, "95.4M"), 14 | ] 15 | 16 | 17 | @pytest.mark.parametrize("input_data, result", test_data) 18 | def test_bytes2human(input_data, result): 19 | """test the basic conversion""" 20 | 21 | assert bytes2human(input_data) == result 22 | 23 | 24 | def test_structured_array_utils(): 25 | arrays = [np.arange(0, 10), np.arange(10, 20), np.arange(30, 40)] 26 | names = "a, b, c" 27 | 28 | recarr0 = np.rec.fromarrays( 29 | arrays, names=names, formats=[arr.dtype for arr in arrays] 30 | ) 31 | round_tripped = recarray_to_ndarray(recarr0) 32 | assert np.all(round_tripped == np.column_stack(arrays).astype(" RampModel: 23 | """ 24 | Organize the steps to run the reference pixel correction. 25 | """ 26 | 27 | # Read in the data from the datamodels 28 | log.debug("Reading data from datamodel into single array") 29 | coeffs = Coefficients.from_ref(refs) 30 | standard = StandardView.from_datamodel(datamodel) 31 | 32 | # Remove offset from the data 33 | if remove_offset: 34 | standard = standard.remove_offset() 35 | log.debug("Removed the general offset from data, to be re-applied later.") 36 | 37 | # Convert to channel view 38 | channel = standard.channels 39 | 40 | # Remove the boundary trends 41 | if remove_trends: 42 | channel = channel.remove_trends() 43 | log.debug("Removed boundary trends (in time) from data.") 44 | 45 | # Cosine interpolate the the data 46 | if cosine_interpolate: 47 | channel = channel.cosine_interpolate() 48 | log.debug("Cosine interpolated the reference pixels.") 49 | 50 | # FFT interpolate the data 51 | if fft_interpolate: 52 | channel = channel.fft_interpolate() 53 | log.debug("FFT interpolated the reference pixel pads.") 54 | 55 | # Perform the reference pixel correction 56 | standard = channel.apply_correction(coeffs) 57 | log.debug("Applied reference pixel correction") 58 | 59 | # Re-apply the offset (if necessary) 60 | standard.apply_offset() 61 | log.debug("Re-applied the general offset (if removed) to the data.") 62 | 63 | # Write the data back to the datamodel 64 | standard.update(datamodel) 65 | log.debug("Updated the datamodel with the corrected data.") 66 | 67 | return datamodel 68 | -------------------------------------------------------------------------------- /romancal/refpix/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/refpix/tests/__init__.py -------------------------------------------------------------------------------- /romancal/refpix/tests/test_refpix.py: -------------------------------------------------------------------------------- 1 | from romancal.refpix.data import StandardView 2 | from romancal.refpix.refpix import run_steps 3 | 4 | from . import reference_utils 5 | 6 | 7 | def test_run_steps_regression(datamodel, ref_pix_ref): 8 | regression = StandardView.from_datamodel(datamodel).data.copy() 9 | regression_out = reference_utils.apply_correction( 10 | regression, ref_pix_ref.alpha, ref_pix_ref.gamma, ref_pix_ref.zeta 11 | ) 12 | 13 | result = run_steps(datamodel, ref_pix_ref, True, True, True, True) 14 | 15 | assert (result.data == regression_out).all() 16 | # regression_out does not return amp33 data 17 | -------------------------------------------------------------------------------- /romancal/refpix/tests/test_step.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from romancal.refpix import RefPixStep 4 | from romancal.refpix.refpix import run_steps 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "remove_offset, remove_trends, cosine_interpolate, fft_interpolate", 9 | [(True, True, True, False), (True, True, True, True)], 10 | ) 11 | def test_refpix_step( 12 | datamodel, 13 | ref_pix_ref, 14 | remove_offset, 15 | remove_trends, 16 | cosine_interpolate, 17 | fft_interpolate, 18 | ): 19 | # Setup and run the correction outside of the step, this is 20 | # already regression tested. 21 | regression_datamodel = datamodel.copy() 22 | regression_ref_pix_ref = ref_pix_ref.copy() 23 | regression = run_steps( 24 | regression_datamodel, 25 | regression_ref_pix_ref, 26 | remove_offset, 27 | remove_trends, 28 | cosine_interpolate, 29 | fft_interpolate, 30 | ) 31 | 32 | # Run the step 33 | result = RefPixStep.call( 34 | datamodel, 35 | override_refpix=ref_pix_ref, 36 | remove_offset=remove_offset, 37 | remove_trends=remove_trends, 38 | cosine_interpolate=cosine_interpolate, 39 | fft_interpolate=fft_interpolate, 40 | ) 41 | 42 | # Check the outputs are distinct 43 | assert result is not regression 44 | 45 | # Check the data 46 | assert (result.data == regression.data).all() 47 | # Check the amp33 48 | assert (result.amp33 == regression.amp33).all() 49 | # Check left ref pix 50 | assert (result.border_ref_pix_left == regression.border_ref_pix_left).all() 51 | # Check right ref pix 52 | assert (result.border_ref_pix_right == regression.border_ref_pix_right).all() 53 | # 54 | # Run the step with reffile = N/A 55 | result = RefPixStep.call( 56 | datamodel, 57 | override_refpix="N/A", 58 | remove_offset=remove_offset, 59 | remove_trends=remove_trends, 60 | cosine_interpolate=cosine_interpolate, 61 | fft_interpolate=fft_interpolate, 62 | ) 63 | assert result.meta.cal_step.refpix == "SKIPPED" 64 | -------------------------------------------------------------------------------- /romancal/regtest/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/regtest/__init__.py -------------------------------------------------------------------------------- /romancal/regtest/test_linearity.py: -------------------------------------------------------------------------------- 1 | """Regression test for the linearity correction step.""" 2 | 3 | import pytest 4 | 5 | from romancal.stpipe import RomanStep 6 | 7 | from .regtestdata import compare_asdf 8 | 9 | 10 | @pytest.mark.bigdata 11 | def test_linearity_step(rtdata, ignore_asdf_paths, resource_tracker, request): 12 | """Function to run and compare linearity correction files.""" 13 | input_file = "r0000101001001001001_0001_wfi01_f158_refpix.asdf" 14 | rtdata.get_data(f"WFI/image/{input_file}") 15 | rtdata.input = input_file 16 | 17 | args = ["romancal.step.LinearityStep", rtdata.input] 18 | with resource_tracker.track(log=request): 19 | RomanStep.from_cmdline(args) 20 | output = "r0000101001001001001_0001_wfi01_f158_linearity.asdf" 21 | rtdata.output = output 22 | rtdata.get_truth(f"truth/WFI/image/{output}") 23 | diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) 24 | assert diff.identical, diff.report() 25 | 26 | 27 | @pytest.mark.bigdata 28 | def test_linearity_outfile_step(rtdata, ignore_asdf_paths, resource_tracker, request): 29 | """Function to run and linearity correction files. Here the 30 | test is for renaming the output file.""" 31 | input_file = "r0000101001001001001_0001_wfi01_f158_refpix.asdf" 32 | rtdata.get_data(f"WFI/image/{input_file}") 33 | rtdata.input = input_file 34 | 35 | args = ["romancal.step.LinearityStep", rtdata.input, "--output_file=Test_linearity"] 36 | with resource_tracker.track(log=request): 37 | RomanStep.from_cmdline(args) 38 | output = "Test_linearity.asdf" 39 | rtdata.output = output 40 | rtdata.get_truth(f"truth/WFI/image/{output}") 41 | diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) 42 | assert diff.identical, diff.report() 43 | -------------------------------------------------------------------------------- /romancal/regtest/test_mos_skycell_pipeline.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import roman_datamodels as rdm 3 | 4 | from romancal.lib.wcsinfo_to_wcs import wcsinfo_to_wcs 5 | from romancal.pipeline.mosaic_pipeline import MosaicPipeline 6 | 7 | from . import util 8 | from .regtestdata import compare_asdf 9 | 10 | # mark all tests in this module 11 | pytestmark = [pytest.mark.bigdata, pytest.mark.soctests] 12 | 13 | 14 | @pytest.fixture(scope="module") 15 | def run_mos(rtdata_module, resource_tracker): 16 | rtdata = rtdata_module 17 | 18 | # Test Pipeline 19 | rtdata.get_asn("WFI/image/L3_mosaic_asn.json") 20 | output = "r00001_p_v01001001001001_270p65x49y70_f158_coadd.asdf" 21 | rtdata.output = output 22 | args = [ 23 | "roman_mos", 24 | rtdata.input, 25 | ] 26 | with resource_tracker.track(): 27 | MosaicPipeline.from_cmdline(args) 28 | rtdata.get_truth(f"truth/WFI/image/{output}") 29 | return rtdata 30 | 31 | 32 | @pytest.fixture(scope="module") 33 | def output_filename(run_mos): 34 | return run_mos.output 35 | 36 | 37 | @pytest.fixture(scope="module") 38 | def output_model(output_filename): 39 | with rdm.open(output_filename) as model: 40 | yield model 41 | 42 | 43 | @pytest.fixture(scope="module") 44 | def truth_filename(run_mos): 45 | return run_mos.truth 46 | 47 | 48 | def test_log_tracked_resources(log_tracked_resources, run_mos): 49 | log_tracked_resources() 50 | 51 | 52 | def test_output_matches_truth(output_filename, truth_filename, ignore_asdf_paths): 53 | diff = compare_asdf(output_filename, truth_filename, **ignore_asdf_paths) 54 | assert diff.identical, diff.report() 55 | 56 | 57 | def test_resample_ran(output_model): 58 | # DMS373 test output is resampled to a skycell 59 | # FIXME this doesn't test if the output is a skycell 60 | assert output_model.meta.cal_step.resample == "COMPLETE" 61 | 62 | 63 | def test_location_name(output_model): 64 | # test that the location_name matches the skycell selected 65 | assert output_model.meta.basic.location_name == "270p65x49y70" 66 | 67 | 68 | def test_wcsinfo_wcs_roundtrip(output_model): 69 | """Test that the contents of wcsinfo reproduces the wcs""" 70 | gwcs_from_wcsinfo = wcsinfo_to_wcs(output_model.meta.wcsinfo) 71 | 72 | ra_mad, dec_mad = util.comp_wcs_grids_arcs(output_model.meta.wcs, gwcs_from_wcsinfo) 73 | assert (ra_mad + dec_mad) / 2.0 < 1.0e-4 74 | -------------------------------------------------------------------------------- /romancal/regtest/test_refpix.py: -------------------------------------------------------------------------------- 1 | """Regression tests for the Reference Pixel Correction step""" 2 | 3 | import pytest 4 | 5 | from romancal.stpipe import RomanStep 6 | 7 | from .regtestdata import compare_asdf 8 | 9 | 10 | @pytest.mark.bigdata 11 | def test_refpix_step(rtdata, ignore_asdf_paths, resource_tracker, request): 12 | input_datafile = "r0000101001001001001_0001_wfi01_f158_saturation.asdf" 13 | rtdata.get_data(f"WFI/image/{input_datafile}") 14 | rtdata.input = input_datafile 15 | 16 | args = ["romancal.step.RefPixStep", rtdata.input] 17 | with resource_tracker.track(log=request): 18 | RomanStep.from_cmdline(args) 19 | 20 | output = "r0000101001001001001_0001_wfi01_f158_refpix.asdf" 21 | rtdata.output = output 22 | rtdata.get_truth(f"truth/WFI/image/{output}") 23 | 24 | diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) 25 | assert diff.identical, diff.report() 26 | -------------------------------------------------------------------------------- /romancal/regtest/test_resource_tracker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | from romancal.regtest.resource_tracker import ( 5 | ResourceTracker, 6 | TrackPeakMemory, 7 | TrackRuntime, 8 | ) 9 | 10 | 11 | class FakeNode: 12 | def __init__(self): 13 | self.user_properties = [] 14 | 15 | 16 | class FakeRequest: 17 | def __init__(self): 18 | self.node = FakeNode() 19 | 20 | 21 | def test_runtime(): 22 | tracker = TrackRuntime() 23 | with tracker: 24 | time.sleep(1.0) 25 | # a 1 second sleep is sometimes too much to ask 26 | # of CI runners. Use a wide margin to make this test 27 | # less brittle in those cases. 28 | threshold = 0.5 if "CI" in os.environ else 0.1 29 | assert abs(tracker.log()[1] - 1.0) < threshold 30 | 31 | 32 | def test_memory(): 33 | tracker = TrackPeakMemory() 34 | N = 1024 * 1024 35 | with tracker: 36 | b = b"0" * N # noqa: F841 37 | assert abs(tracker.log()[1] - N) / N < 0.01 38 | 39 | 40 | def test_resource_tracker(): 41 | tracker = ResourceTracker() 42 | with tracker.track(): 43 | pass 44 | fake_request = FakeRequest() 45 | tracker.log(fake_request) 46 | keys = {log[0] for log in fake_request.node.user_properties} 47 | assert keys == {"tracked-time", "tracked-peakmem"} 48 | 49 | 50 | def test_log(): 51 | tracker = ResourceTracker() 52 | fake_request = FakeRequest() 53 | with tracker.track(log=fake_request): 54 | pass 55 | keys = {log[0] for log in fake_request.node.user_properties} 56 | assert keys == {"tracked-time", "tracked-peakmem"} 57 | -------------------------------------------------------------------------------- /romancal/regtest/test_wfi_image_16resultants.py: -------------------------------------------------------------------------------- 1 | """Roman tests for flat field correction""" 2 | 3 | import pytest 4 | import roman_datamodels as rdm 5 | 6 | from romancal.pipeline.exposure_pipeline import ExposurePipeline 7 | 8 | # mark all tests in this module 9 | pytestmark = [pytest.mark.bigdata, pytest.mark.soctests] 10 | 11 | 12 | @pytest.fixture(scope="module") 13 | def run_elp(rtdata_module, resource_tracker): 14 | rtdata = rtdata_module 15 | 16 | input_data = "r0000101001001001001_0004_wfi01_f158_uncal.asdf" 17 | rtdata.get_data(f"WFI/image/{input_data}") 18 | rtdata.input = input_data 19 | 20 | # Test Pipeline 21 | output = "r0000101001001001001_0004_wfi01_f158_cal.asdf" 22 | rtdata.output = output 23 | args = [ 24 | "roman_elp", 25 | rtdata.input, 26 | ] 27 | with resource_tracker.track(): 28 | ExposurePipeline.from_cmdline(args) 29 | return rtdata 30 | 31 | 32 | @pytest.fixture(scope="module") 33 | def output_filename(run_elp): 34 | return run_elp.output 35 | 36 | 37 | @pytest.fixture(scope="module") 38 | def output_model(output_filename): 39 | with rdm.open(output_filename) as model: 40 | yield model 41 | 42 | 43 | @pytest.fixture(scope="module") 44 | def input_filename(run_elp): 45 | return run_elp.input 46 | 47 | 48 | @pytest.fixture(scope="module") 49 | def input_model(input_filename): 50 | with rdm.open(input_filename) as model: 51 | yield model 52 | 53 | 54 | def test_log_tracked_resources(log_tracked_resources, run_elp): 55 | log_tracked_resources() 56 | 57 | 58 | def test_output_is_image_model(output_model): 59 | # DMS413 60 | assert isinstance(output_model, rdm.datamodels.ImageModel) 61 | 62 | 63 | def test_input_has_16_resultants(input_model): 64 | # DMS413 65 | assert len(input_model.meta.exposure.read_pattern) == 16 66 | 67 | 68 | def test_output_has_16_resultants(output_model): 69 | # DMS413 70 | assert len(output_model.meta.exposure.read_pattern) == 16 71 | 72 | 73 | @pytest.mark.parametrize( 74 | "step_name", 75 | ( 76 | "assign_wcs", 77 | "flat_field", 78 | "dark", 79 | "dq_init", 80 | "linearity", 81 | "ramp_fit", 82 | "saturation", 83 | "photom", 84 | ), 85 | ) 86 | def test_steps_ran(output_model, step_name): 87 | assert getattr(output_model.meta.cal_step, step_name) == "COMPLETE" 88 | -------------------------------------------------------------------------------- /romancal/regtest/util.py: -------------------------------------------------------------------------------- 1 | """Test utilities""" 2 | 3 | import numpy as np 4 | from astropy.stats import mad_std 5 | 6 | 7 | def comp_wcs_grids_arcs(wcs_a, wcs_b, npix=4088, interval=10): 8 | """Compare world grids produced by the two wcs 9 | 10 | Parameters 11 | ---------- 12 | wcs_a, wcs_b : gwcs.WCS 13 | The wcs object to compare. 14 | 15 | npix : int 16 | The size of the grid to produce. 17 | 18 | interval : int 19 | The interval to check over. 20 | 21 | Returns 22 | ------- 23 | mad_std : float 24 | The numpy MAD_STD in arcseconds 25 | """ 26 | xx, yy = np.meshgrid(np.linspace(0, npix, interval), np.linspace(0, npix, interval)) 27 | ra_a, dec_a = wcs_a(xx, yy, with_bounding_box=False) 28 | ra_b, dec_b = wcs_b(xx, yy, with_bounding_box=False) 29 | 30 | ra_mad = mad_std(ra_a - ra_b, ignore_nan=True) * 60.0 * 60.0 * 1000.0 31 | dec_mad = mad_std(dec_a - dec_b, ignore_nan=True) * 60.0 * 60.0 * 1000.0 32 | 33 | return ra_mad, dec_mad 34 | -------------------------------------------------------------------------------- /romancal/resample/__init__.py: -------------------------------------------------------------------------------- 1 | from .resample_step import ResampleStep 2 | 3 | __all__ = ["ResampleStep"] 4 | -------------------------------------------------------------------------------- /romancal/resample/exptime_resampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from drizzle.resample import Drizzle 3 | from roman_datamodels import dqflags 4 | from stcal.resample.utils import ( 5 | build_driz_weight, 6 | ) 7 | 8 | 9 | class ExptimeResampler: 10 | def __init__(self, out_wcs, out_shape, good_bits, kernel): 11 | self.out_wcs = out_wcs 12 | self.out_shape = out_shape 13 | self.good_bits = good_bits 14 | 15 | self.out_img = np.zeros(out_shape, dtype="f4") 16 | self.out_wht = np.zeros(out_shape, dtype="f4") 17 | self.exptime_total = np.zeros(out_shape, dtype="f4") 18 | 19 | self._driz = Drizzle( 20 | kernel=kernel, 21 | fillval=0, 22 | out_img=self.out_img, 23 | out_wht=self.out_wht, 24 | disable_ctx=True, 25 | ) 26 | 27 | def add_image(self, model, pixmap, xmin, xmax, ymin, ymax): 28 | data = np.full(model["data"].shape, model["effective_exposure_time"]) 29 | 30 | # create a unit weight map for all the input pixels with science data 31 | inwht = build_driz_weight( 32 | { 33 | "data": model["data"], 34 | "dq": model["dq"], 35 | }, 36 | weight_type=None, 37 | good_bits=self.good_bits, 38 | flag_name_map=dqflags.pixel, 39 | ) 40 | 41 | self._driz.add_image( 42 | data, 43 | pixmap=pixmap, 44 | exptime=1.0, 45 | scale=1.0, 46 | weight_map=inwht, 47 | wht_scale=1.0, 48 | pixfrac=1.0, 49 | in_units="cps", 50 | xmin=xmin, 51 | xmax=xmax, 52 | ymin=ymin, 53 | ymax=ymax, 54 | ) 55 | 56 | self.exptime_total += self.out_img 57 | 58 | # zero out arrays for next image 59 | self.out_img[:] = 0 60 | self.out_wht[:] = 0 61 | 62 | def finalize(self): 63 | return self.exptime_total 64 | -------------------------------------------------------------------------------- /romancal/resample/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/resample/tests/__init__.py -------------------------------------------------------------------------------- /romancal/saturation/__init__.py: -------------------------------------------------------------------------------- 1 | from .saturation_step import SaturationStep 2 | 3 | __all__ = ["SaturationStep"] 4 | -------------------------------------------------------------------------------- /romancal/saturation/saturation.py: -------------------------------------------------------------------------------- 1 | # Module for 2d saturation 2 | # 3 | import logging 4 | 5 | import numpy as np 6 | from roman_datamodels.dqflags import pixel 7 | from stcal.saturation.saturation import flag_saturated_pixels 8 | 9 | log = logging.getLogger(__name__) 10 | log.setLevel(logging.DEBUG) 11 | 12 | ATOD_LIMIT = 65535.0 # Hard DN limit of 16-bit A-to-D converter 13 | 14 | 15 | def flag_saturation(input_model, ref_model): 16 | """ 17 | Short Summary 18 | ------------- 19 | Function to call stcal for flagging saturated pixels. 20 | 21 | Parameters 22 | ---------- 23 | input_model : `~roman_datamodels.datamodels.RampModel` 24 | The input science data to be corrected 25 | 26 | ref_model : `~roman_datamodels.datamodels.SaturationRefModel` 27 | Saturation reference file data model 28 | 29 | Returns 30 | ------- 31 | output_model : `~roman_datamodels.datamodels.RampModel` 32 | Data model with saturation, A/D floor, and do not use flags set in 33 | the GROUPDQ array 34 | The input model is modified in place and returned as the output model. 35 | """ 36 | 37 | data = input_model.data[np.newaxis, :] 38 | 39 | # Modify input_model in place. 40 | gdq = input_model.groupdq[np.newaxis, :] 41 | pdq = input_model.pixeldq[np.newaxis, :] 42 | 43 | # Copy information from saturation reference file 44 | sat_thresh = ref_model.data.copy() 45 | sat_dq = ref_model.dq.copy() 46 | 47 | # Obtain dq arrays updated for saturation 48 | # The third variable is the processed ZEROFRAME, which is not 49 | # used in romancal, so is always None. 50 | gdq_new, pdq_new, _ = flag_saturated_pixels( 51 | data, 52 | gdq, 53 | pdq, 54 | sat_thresh, 55 | sat_dq, 56 | ATOD_LIMIT, 57 | pixel, 58 | n_pix_grow_sat=0, 59 | read_pattern=input_model.meta.exposure.read_pattern, 60 | ) 61 | 62 | # Save the flags in the output GROUPDQ array 63 | input_model.groupdq = gdq_new[0, :] 64 | 65 | # Save the NO_SAT_CHECK flags in the output PIXELDQ array 66 | input_model.pixeldq = pdq_new[0, :] 67 | 68 | return input_model 69 | -------------------------------------------------------------------------------- /romancal/saturation/saturation_step.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | from __future__ import annotations 3 | 4 | from typing import TYPE_CHECKING 5 | 6 | import roman_datamodels as rdm 7 | from roman_datamodels.datamodels import SaturationRefModel 8 | 9 | from romancal.saturation import saturation 10 | from romancal.stpipe import RomanStep 11 | 12 | if TYPE_CHECKING: 13 | from typing import ClassVar 14 | 15 | __all__ = ["SaturationStep"] 16 | 17 | 18 | class SaturationStep(RomanStep): 19 | """ 20 | This Step sets saturation flags. 21 | """ 22 | 23 | class_alias = "saturation" 24 | 25 | reference_file_types: ClassVar = ["saturation"] 26 | 27 | def process(self, input): 28 | if isinstance(input, rdm.DataModel): 29 | input_model = input 30 | else: 31 | # Open the input data model 32 | input_model = rdm.open(input) 33 | 34 | # Get the name of the saturation reference file 35 | self.ref_name = self.get_reference_file(input_model, "saturation") 36 | 37 | # Check for a valid reference file 38 | if self.ref_name == "N/A": 39 | self.log.warning("No SATURATION reference file found") 40 | self.log.warning("Saturation step will be skipped") 41 | result = input_model.copy() 42 | result.meta.cal_step.saturation = "SKIPPED" 43 | return result 44 | 45 | # Open the reference file data model 46 | # Test for reference file 47 | self.log.info("Using SATURATION reference file: %s", self.ref_name) 48 | ref_model = SaturationRefModel(self.ref_name) 49 | 50 | # Perform saturation check 51 | sat = saturation.flag_saturation(input_model, ref_model) 52 | 53 | # Close the reference file and update the step status 54 | ref_model.close() 55 | sat.meta.cal_step.saturation = "COMPLETE" 56 | 57 | if self.save_results: 58 | try: 59 | self.suffix = "saturation" 60 | except AttributeError: 61 | self["suffix"] = "saturation" 62 | 63 | return sat 64 | -------------------------------------------------------------------------------- /romancal/saturation/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/saturation/tests/__init__.py -------------------------------------------------------------------------------- /romancal/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/scripts/__init__.py -------------------------------------------------------------------------------- /romancal/scripts/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/scripts/tests/__init__.py -------------------------------------------------------------------------------- /romancal/scripts/tests/test_scripts.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | import pytest 4 | 5 | SCRIPTS = [ 6 | "verify_install_requires", 7 | ] 8 | 9 | 10 | @pytest.mark.parametrize("script", SCRIPTS) 11 | def test_script_installed(script): 12 | assert shutil.which(script) is not None, f"`{script}` not installed" 13 | -------------------------------------------------------------------------------- /romancal/skycell/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/skycell/__init__.py -------------------------------------------------------------------------------- /romancal/skycell/tests/data/L3_mosaic_asn.json: -------------------------------------------------------------------------------- 1 | { 2 | "asn_type": "None", 3 | "asn_rule": "DMS_ELPP_Base", 4 | "version_id": null, 5 | "code_version": "0.18.1.dev102+g74d0e38d", 6 | "target": "225p90x49y67", 7 | "degraded_status": "No known degraded exposures in association.", 8 | "program": "noprogram", 9 | "constraints": "No constraints", 10 | "asn_id": "a3001", 11 | "asn_pool": "none", 12 | "skycell_wcs_info": "none", 13 | "products": [ 14 | { 15 | "name": "r00001_p_v01001001001001_225p90x49y67_f158", 16 | "members": [ 17 | { 18 | "expname": "r0000101001001001001_0001_wfi01_f158_cal.asdf", 19 | "exptype": "science" 20 | }, 21 | { 22 | "expname": "r0000101001001001001_0002_wfi01_f158_cal.asdf", 23 | "exptype": "science" 24 | }, 25 | { 26 | "expname": "r0000101001001001001_0003_wfi01_f158_cal.asdf", 27 | "exptype": "science" 28 | } 29 | ] 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /romancal/skycell/tests/data/L3_mosaic_wcs_info_asn.json: -------------------------------------------------------------------------------- 1 | { 2 | "asn_type": "None", 3 | "asn_rule": "DMS_ELPP_Base", 4 | "version_id": null, 5 | "code_version": "0.18.1.dev102+g74d0e38d", 6 | "target": "None", 7 | "degraded_status": "No known degraded exposures in association.", 8 | "program": "noprogram", 9 | "constraints": "No constraints", 10 | "asn_id": "a3001", 11 | "asn_pool": "none", 12 | "skycell_wcs_info": { 13 | "name": "225p90x49y67", 14 | "pixel_scale": 1.5277777777777777e-5, 15 | "ra_projection_center": 225.0, 16 | "dec_projection_center": 90.0, 17 | "x0_projection": 7299.5, 18 | "y0_projection": -79100.5, 19 | "ra_center": 228.3657197782, 20 | "dec_center": 88.7513616876, 21 | "nx": 5000, 22 | "ny": 5000, 23 | "orientat": -3.365719795227051, 24 | "orientat_projection_center": 180.0 25 | }, 26 | "products": [ 27 | { 28 | "name": "r00001_p_v01001001001001_225p90x49y67_f158", 29 | "members": [ 30 | { 31 | "expname": "r0000101001001001001_0001_wfi01_f158_cal.asdf", 32 | "exptype": "science" 33 | }, 34 | { 35 | "expname": "r0000101001001001001_0002_wfi01_f158_cal.asdf", 36 | "exptype": "science" 37 | }, 38 | { 39 | "expname": "r0000101001001001001_0003_wfi01_f158_cal.asdf", 40 | "exptype": "science" 41 | } 42 | ] 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /romancal/skycell/tests/data/L3_regtest_asn.json: -------------------------------------------------------------------------------- 1 | { 2 | "asn_type": "None", 3 | "asn_rule": "DMS_ELPP_Base", 4 | "version_id": null, 5 | "code_version": "0.18.1.dev102+g74d0e38d", 6 | "target": "None", 7 | "degraded_status": "No known degraded exposures in association.", 8 | "program": "noprogram", 9 | "constraints": "No constraints", 10 | "asn_id": "a3001", 11 | "asn_pool": "none", 12 | "skycell_wcs_info": "none", 13 | "products": [ 14 | { 15 | "name": "r0000101001001001001_f158", 16 | "members": [ 17 | { 18 | "expname": "r0000101001001001001_0001_wfi01_f158_cal.asdf", 19 | "exptype": "science" 20 | }, 21 | { 22 | "expname": "r0000101001001001001_0002_wfi01_f158_cal.asdf", 23 | "exptype": "science" 24 | }, 25 | { 26 | "expname": "r0000101001001001001_0003_wfi01_f158_cal.asdf", 27 | "exptype": "science" 28 | } 29 | ] 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /romancal/skycell/tests/data/L3_skycell_mbcat_asn.json: -------------------------------------------------------------------------------- 1 | { 2 | "asn_type": "None", 3 | "asn_rule": "DMS_ELPP_Base", 4 | "version_id": null, 5 | "code_version": "0.18.1.dev102+g74d0e38d", 6 | "target": "None", 7 | "degraded_status": "No known degraded exposures in association.", 8 | "program": "noprogram", 9 | "constraints": "No constraints", 10 | "asn_id": "a3001", 11 | "asn_pool": "none", 12 | "skycell_wcs_info": "none", 13 | "products": [ 14 | { 15 | "name": "r00001_p_v01001001001001_225p90x49y67_f158_mbcat", 16 | "members": [ 17 | { 18 | "expname": "r00001_p_v01001001001001_225p90x49y67_f158_coadd.asdf", 19 | "exptype": "science" 20 | } 21 | ] 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /romancal/skycell/tests/data/skymap_subset.asdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/skycell/tests/data/skymap_subset.asdf -------------------------------------------------------------------------------- /romancal/skycell/tests/skymap_subset.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import romancal.skycell.skymap as sc 4 | 5 | LAST_PROJREGION_INDEX = 2 6 | DATA_DIRECTORY = Path(__file__).parent / "data" 7 | 8 | if __name__ == "__main__": 9 | skymap_subset = sc.SKYMAP.data.copy() 10 | 11 | # to maintain the proper indices, the subset must contain all the previous projection regions up to the specified index 12 | skymap_subset["roman"]["projection_regions"] = sc.SKYMAP.projection_regions[ 13 | : LAST_PROJREGION_INDEX + 1 14 | ].copy() 15 | skymap_subset["roman"]["skycells"] = sc.SKYMAP.skycells[ 16 | : skymap_subset["roman"]["projection_regions"][-1]["skycell_end"] + 1 17 | ].copy() 18 | 19 | skymap_subset.write_to(DATA_DIRECTORY / "skymap_subset.asdf") 20 | -------------------------------------------------------------------------------- /romancal/skymatch/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package provides support for sky background subtraction and equalization 3 | (matching). 4 | """ 5 | 6 | import logging 7 | 8 | from .skymatch_step import SkyMatchStep 9 | 10 | log = logging.getLogger(__name__) 11 | log.setLevel(logging.DEBUG) 12 | 13 | __all__ = ["SkyMatchStep"] 14 | -------------------------------------------------------------------------------- /romancal/skymatch/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/skymatch/tests/__init__.py -------------------------------------------------------------------------------- /romancal/source_catalog/__init__.py: -------------------------------------------------------------------------------- 1 | from .source_catalog_step import SourceCatalogStep 2 | 3 | __all__ = ["SourceCatalogStep"] 4 | -------------------------------------------------------------------------------- /romancal/source_catalog/_wcs_helpers.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | # (taken from photutils: should probably migrate into astropy.wcs) 3 | """ 4 | This module provides WCS helper tools. 5 | """ 6 | 7 | import astropy.units as u 8 | import numpy as np 9 | 10 | 11 | def pixel_scale_angle_at_skycoord(skycoord, wcs, offset=1 * u.arcsec): 12 | """ 13 | Calculate the pixel coordinate scale and WCS rotation angle at 14 | the position of a SkyCoord coordinate. 15 | 16 | Parameters 17 | ---------- 18 | skycoord : `~astropy.coordinates.SkyCoord` 19 | The SkyCoord coordinate. 20 | 21 | wcs : WCS object 22 | A world coordinate system (WCS) transformation that 23 | supports the `astropy shared interface for WCS 24 | `_ (e.g., 25 | `astropy.wcs.WCS`, `gwcs.wcs.WCS`). 26 | 27 | offset : `~astropy.units.Quantity` 28 | A small angular offset to use to compute the pixel scale and 29 | position angle. 30 | 31 | Returns 32 | ------- 33 | xypos : tuple of float 34 | The (x, y) pixel coordinate. 35 | 36 | scale : `~astropy.units.Quantity` 37 | The pixel scale in arcsec. 38 | 39 | angle : `~astropy.units.Quantity` 40 | The angle (in degrees) measured counterclockwise from the 41 | positive x axis to the "North" axis of the celestial coordinate 42 | system. 43 | 44 | Notes 45 | ----- 46 | If distortions are present in the image, the x and y pixel scales 47 | likely differ. This function computes a single pixel scale along the 48 | North/South axis. 49 | """ 50 | # Convert to pixel coordinates 51 | xpos, ypos = wcs.world_to_pixel(skycoord) 52 | 53 | # We take a point directly North (i.e., latitude offset) the 54 | # input sky coordinate and convert it to pixel coordinates, 55 | # then we use the pixel deltas between the input and offset sky 56 | # coordinate to calculate the pixel scale and angle. 57 | skycoord_offset = skycoord.directional_offset_by(0.0, offset) 58 | x_offset, y_offset = wcs.world_to_pixel(skycoord_offset) 59 | 60 | dx = x_offset - xpos 61 | dy = y_offset - ypos 62 | scale = offset.to(u.arcsec) / np.hypot(dx, dy) 63 | angle = (np.arctan2(dy, dx) * u.radian).to(u.deg) 64 | 65 | return (xpos, ypos), scale, angle 66 | -------------------------------------------------------------------------------- /romancal/source_catalog/neighbors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to compute nearest neighbors for a catalog of sources. 3 | """ 4 | 5 | import numpy as np 6 | from astropy.utils.decorators import lazyproperty 7 | from scipy.spatial import KDTree 8 | 9 | 10 | class NNCatalog: 11 | """ 12 | Class to compute nearest neighbors for a catalog of sources. 13 | """ 14 | 15 | def __init__(self, label, xypos, xypos_finite, pixel_scale): 16 | self.label = label 17 | self.xypos = xypos 18 | self.xypos_finite = xypos_finite 19 | self.pixel_scale = pixel_scale 20 | 21 | self.nonfinite_mask = ~np.isfinite(xypos).all(axis=1) 22 | 23 | self.names = ["nn_label", "nn_distance"] 24 | 25 | def __len__(self): 26 | """ 27 | Return the number of sources in the catalog. 28 | """ 29 | return len(self.label) 30 | 31 | @lazyproperty 32 | def _kdtree_query(self): 33 | """ 34 | The distance in pixels to the nearest neighbor and its index. 35 | """ 36 | if len(self) == 1: 37 | return [np.nan], [np.nan] 38 | 39 | # non-finite xypos causes memory errors on linux, but not MacOS 40 | tree = KDTree(self.xypos_finite) 41 | qdist, qidx = tree.query(self.xypos_finite, k=[2]) 42 | return np.transpose(qdist)[0], np.transpose(qidx)[0] 43 | 44 | @lazyproperty 45 | def nn_label(self): 46 | """ 47 | The label number of the nearest neighbor. 48 | 49 | A label value of -1 is returned if there is only one detected 50 | source and for sources with a non-finite centroid. 51 | """ 52 | if len(self) == 1: 53 | return np.int32(-1) 54 | 55 | nn_label = self.label[self._kdtree_query[1]].astype(np.int32) 56 | # assign a label of -1 for non-finite xypos 57 | nn_label[self.nonfinite_mask] = -1 58 | 59 | return nn_label 60 | 61 | @lazyproperty 62 | def nn_distance(self): 63 | """ 64 | The distance in arcsec to the nearest neighbor. 65 | 66 | NaN is returned for non-finite centroid positions or when 67 | the catalog contains only one source. 68 | """ 69 | nn_distance = self._kdtree_query[0] 70 | if len(self) != 1: 71 | # assign a distance of np.nan for non-finite xypos 72 | nn_distance[self.nonfinite_mask] = np.nan 73 | 74 | return (nn_distance * self.pixel_scale).astype(np.float32) 75 | -------------------------------------------------------------------------------- /romancal/source_catalog/save_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import numpy as np 4 | from roman_datamodels.datamodels import ( 5 | ImageSourceCatalogModel, 6 | MosaicSegmentationMapModel, 7 | SegmentationMapModel, 8 | ) 9 | 10 | 11 | def save_segment_image(self, segment_img, source_catalog_model, output_filename): 12 | """ 13 | Save the segmentation image to an ASDF file. 14 | 15 | Parameters 16 | ---------- 17 | self : `romancal.source_catalog.SourceCatalogStep` or `romancal.multiband_catalog.MultiBandCatalogStep` 18 | The step instance that is calling this function. 19 | 20 | segment_img : `photutils.segmentation.SegmentationImage` 21 | The segmentation image to save. 22 | 23 | source_catalog_model : `roman_datamodels.datamodels.SourceCatalogModel` or `roman_datamodels.datamodels.MosaicSourceCatalogModel` 24 | The source catalog model to use for metadata. 25 | 26 | output_filename : str 27 | The output file name. 28 | """ 29 | if segment_img is None: 30 | logging.warning("No segmentation image to save.") 31 | return 32 | 33 | # Define the segmentation model based on the source catalog model 34 | # type 35 | if isinstance(source_catalog_model, ImageSourceCatalogModel): 36 | segm_model = SegmentationMapModel 37 | else: 38 | segm_model = MosaicSegmentationMapModel 39 | segmentation_model = segm_model.create_minimal({"meta": source_catalog_model.meta}) 40 | 41 | # Set the data and detection image 42 | segmentation_model.data = segment_img.data.astype(np.uint32) 43 | segmentation_model["detection_image"] = segment_img.detection_image 44 | 45 | # Save the segmentation image to the output file 46 | self.output_ext = "asdf" 47 | self.save_model( 48 | segmentation_model, 49 | output_file=output_filename, 50 | suffix="segm", 51 | force=True, 52 | ) 53 | -------------------------------------------------------------------------------- /romancal/source_catalog/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/source_catalog/tests/__init__.py -------------------------------------------------------------------------------- /romancal/step.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module collects all of the stpipe.Step subclasses 3 | made available by this package. 4 | """ 5 | 6 | from .assign_wcs.assign_wcs_step import AssignWcsStep 7 | from .dark_current.dark_current_step import DarkCurrentStep 8 | from .dq_init.dq_init_step import DQInitStep 9 | from .flatfield.flat_field_step import FlatFieldStep 10 | from .flux import FluxStep 11 | from .linearity.linearity_step import LinearityStep 12 | from .multiband_catalog.multiband_catalog_step import MultibandCatalogStep 13 | from .outlier_detection.outlier_detection_step import OutlierDetectionStep 14 | from .photom.photom_step import PhotomStep 15 | from .ramp_fitting.ramp_fit_step import RampFitStep 16 | from .refpix.refpix_step import RefPixStep 17 | from .resample.resample_step import ResampleStep 18 | from .saturation.saturation_step import SaturationStep 19 | from .skymatch.skymatch_step import SkyMatchStep 20 | from .source_catalog.source_catalog_step import SourceCatalogStep 21 | from .tweakreg.tweakreg_step import TweakRegStep 22 | 23 | __all__ = [ 24 | "AssignWcsStep", 25 | "DQInitStep", 26 | "DarkCurrentStep", 27 | "FlatFieldStep", 28 | "FluxStep", 29 | "LinearityStep", 30 | "MultibandCatalogStep", 31 | "OutlierDetectionStep", 32 | "PhotomStep", 33 | "RampFitStep", 34 | "RefPixStep", 35 | "ResampleStep", 36 | "SaturationStep", 37 | "SkyMatchStep", 38 | "SourceCatalogStep", 39 | "TweakRegStep", 40 | ] 41 | -------------------------------------------------------------------------------- /romancal/stpipe/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import RomanPipeline, RomanStep 2 | 3 | __all__ = ["RomanPipeline", "RomanStep"] 4 | -------------------------------------------------------------------------------- /romancal/stpipe/integration.py: -------------------------------------------------------------------------------- 1 | """ 2 | Entry point implementations. 3 | """ 4 | 5 | 6 | def get_steps(): 7 | """ 8 | Return tuples describing the stpipe.Step subclasses provided 9 | by this package. This method is registered with the stpipe.steps 10 | entry point. 11 | 12 | Returns 13 | ------- 14 | list of tuple (str, str, bool) 15 | The first element each tuple is a fully-qualified Step 16 | subclass name. The second element is an optional class 17 | alias. The third element indicates that the class 18 | is a subclass of Pipeline. 19 | """ 20 | # Unit tests ensure that this list is kept in sync with the actual 21 | # class definitions. We need to avoid importing romancal.pipeline and 22 | # romancal.step to keep the CLI snappy. 23 | return [ 24 | ("romancal.pipeline.ExposurePipeline", "roman_elp", True), 25 | ("romancal.pipeline.MosaicPipeline", "roman_mos", True), 26 | ("romancal.step.DarkCurrentStep", "dark", False), 27 | ("romancal.step.DQInitStep", "dq_init", False), 28 | ("romancal.step.FlatFieldStep", "flat_field", False), 29 | ("romancal.step.FluxStep", "flux", False), 30 | ("romancal.step.LinearityStep", "linearity", False), 31 | ("romancal.step.PhotomStep", "photom", False), 32 | ("romancal.step.RampFitStep", "ramp_fit", False), 33 | ("romancal.step.RefPixStep", "refpix", False), 34 | ("romancal.step.SaturationStep", "saturation", False), 35 | ("romancal.step.AssignWcsStep", "assign_wcs", False), 36 | ("romancal.step.OutlierDetectionStep", "outlier_detection", False), 37 | ("romancal.step.SkyMatchStep", "skymatch", False), 38 | ("romancal.step.TweakRegStep", "tweakreg", False), 39 | ("romancal.step.ResampleStep", "resample", False), 40 | ("romancal.step.SourceCatalogStep", "source_catalog", False), 41 | ("romancal.step.MultibandCatalogStep", "multiband_catalog", False), 42 | ] 43 | -------------------------------------------------------------------------------- /romancal/stpipe/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/stpipe/tests/__init__.py -------------------------------------------------------------------------------- /romancal/stpipe/tests/test_integration.py: -------------------------------------------------------------------------------- 1 | from stpipe import entry_points 2 | from stpipe.utilities import import_class 3 | 4 | import romancal.pipeline 5 | import romancal.step 6 | from romancal.stpipe import RomanPipeline, RomanStep 7 | from romancal.stpipe.integration import get_steps 8 | 9 | 10 | def test_get_steps(): 11 | tuples = get_steps() 12 | 13 | assert {t[0].split(".")[-1] for t in tuples} == set( 14 | romancal.step.__all__ + romancal.pipeline.__all__ 15 | ) 16 | 17 | for class_name, class_alias, is_pipeline in tuples: 18 | step_class = import_class(class_name) 19 | assert issubclass(step_class, RomanStep) 20 | assert step_class.class_alias == class_alias 21 | if is_pipeline: 22 | assert issubclass(step_class, RomanPipeline) 23 | 24 | 25 | def test_entry_point(): 26 | all_steps = entry_points.get_steps() 27 | roman_steps = [s for s in all_steps if s.package_name == "romancal"] 28 | tuples = {(s.class_name, s.class_alias, s.is_pipeline) for s in roman_steps} 29 | assert tuples == set(get_steps()) 30 | -------------------------------------------------------------------------------- /romancal/stpipe/utilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities 3 | """ 4 | 5 | import inspect 6 | import logging 7 | from importlib import import_module 8 | from pkgutil import walk_packages 9 | 10 | # Configure logging 11 | logger = logging.getLogger(__name__) 12 | logger.addHandler(logging.NullHandler()) 13 | 14 | # Step classes that are not user-api steps 15 | NON_STEPS = [ 16 | "EngDBLogStep", 17 | "FunctionWrapper", 18 | "RomancalPipeline", 19 | "RomanPipeline", 20 | "ExposurePipeline", 21 | "MosaicPipeline", 22 | "RomanStep", 23 | "Pipeline", 24 | "Step", 25 | "SystemCall", 26 | ] 27 | 28 | 29 | def all_steps(): 30 | """List all classes subclassed from Step 31 | 32 | Returns 33 | ------- 34 | steps : dict 35 | Key is the classname, value is the class 36 | """ 37 | from romancal.stpipe import RomanStep as Step 38 | 39 | romancal = import_module("romancal") 40 | 41 | steps = {} 42 | for module in load_sub_modules(romancal): 43 | more_steps = { 44 | klass_name: klass 45 | for klass_name, klass in inspect.getmembers( 46 | module, lambda o: inspect.isclass(o) and issubclass(o, Step) 47 | ) 48 | if klass_name not in NON_STEPS 49 | } 50 | steps.update(more_steps) 51 | 52 | return steps 53 | 54 | 55 | def load_sub_modules(module): 56 | """ 57 | Recursively loads all submodules of a module (this is not a local import). 58 | 59 | Parameters 60 | ---------- 61 | module : module 62 | A python module to walk, load 63 | 64 | Returns 65 | ------- 66 | generator 67 | A generator of all submodules of module recursively until no more sub modules are found 68 | """ 69 | 70 | for package_info in walk_packages(module.__path__): 71 | if package_info.module_finder.path.startswith(module.__path__[0]): 72 | package = import_module(f"{module.__name__}.{package_info.name}") 73 | 74 | if package_info.ispkg: 75 | yield from load_sub_modules(package) 76 | 77 | yield package 78 | -------------------------------------------------------------------------------- /romancal/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under a 3-clause BSD style license - see LICENSE.rst 2 | """ 3 | This module contains package tests. 4 | """ 5 | -------------------------------------------------------------------------------- /romancal/tests/test_dms_requirements.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | from pathlib import Path 4 | 5 | TEST_DIRECTORY = Path(__file__).parent.parent 6 | TEST_REQUIREMENTS_FILENAME = TEST_DIRECTORY / "tests" / "dms_requirement_tests.json" 7 | 8 | 9 | def test_requirements(): 10 | test_requirements_filename = TEST_REQUIREMENTS_FILENAME.expanduser().absolute() 11 | test_directory = TEST_DIRECTORY.expanduser().absolute() 12 | 13 | with open(test_requirements_filename) as test_requirements_file: 14 | requirements = json.load(test_requirements_file) 15 | 16 | required_tests = { 17 | test 18 | for requirement_tests in requirements.values() 19 | for test in requirement_tests 20 | } 21 | 22 | existing_tests = set() 23 | test_regex = re.compile(r"def (test_[^\(]+)\(") 24 | for test_filename in test_directory.glob("**/test_*.py"): 25 | with open(test_filename) as test_file: 26 | test_file_contents = test_file.read() 27 | 28 | for match in re.finditer(test_regex, test_file_contents): 29 | test = f"{test_directory.stem}.{str(test_filename.relative_to(test_directory).parent).replace('/', '.')}.{test_filename.stem}.{match.group(1)}" 30 | if test in required_tests: 31 | existing_tests.add(test) 32 | 33 | missing_tests = required_tests - existing_tests 34 | assert not missing_tests, ( 35 | f"could not find the following tests correlated with DMS requirements: {missing_tests}" 36 | ) 37 | -------------------------------------------------------------------------------- /romancal/tests/test_import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This test file is a bit different in that pytest should 4 | not be imported within this file. It is designed to check 5 | that all submodules are importable (without silently 6 | depending on pytest or it's dependencies) and is run 7 | in a separate tox environment where pytest is not installed. 8 | """ 9 | 10 | import importlib 11 | import pkgutil 12 | 13 | import romancal 14 | 15 | 16 | def dependencies(package, exclude: [str]): 17 | return [ 18 | module[1] 19 | for module in pkgutil.walk_packages( 20 | package.__path__, prefix=package.__name__ + "." 21 | ) 22 | if not any(exclude_module in module[1] for exclude_module in exclude) 23 | ] 24 | 25 | 26 | # dqflags is deprecated 27 | MODULES = dependencies(romancal, exclude=["test", "time", "static_preview", "dqflags"]) 28 | 29 | 30 | def test_modules_import(): 31 | for module in MODULES: 32 | importlib.import_module(module) 33 | 34 | 35 | if __name__ == "__main__": 36 | test_modules_import() 37 | -------------------------------------------------------------------------------- /romancal/tweakreg/__init__.py: -------------------------------------------------------------------------------- 1 | from .tweakreg_step import TweakRegStep 2 | 3 | __all__ = ["TweakRegStep"] 4 | -------------------------------------------------------------------------------- /romancal/tweakreg/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spacetelescope/romancal/1ca5db722ec95b0a65c9b5dd0ceacfb7a2345638/romancal/tweakreg/tests/__init__.py --------------------------------------------------------------------------------